Saboorhsn commited on
Commit
24b4364
·
verified ·
1 Parent(s): 308ef5a

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +9 -4
app.py CHANGED
@@ -3,6 +3,7 @@ from datasets import load_dataset, concatenate_datasets
3
  import json
4
  import os
5
  import base64
 
6
 
7
  def load_and_combine_datasets():
8
  python_codes_dataset = load_dataset('flytech/python-codes-25k', split='train')
@@ -13,10 +14,14 @@ def load_and_combine_datasets():
13
 
14
  return combined_dataset
15
 
 
 
 
 
16
  def save_combined_dataset_as_jsonl(combined_dataset, file_path):
17
  with open(file_path, 'w', encoding='utf-8') as f:
18
  for example in combined_dataset:
19
- json.dump(example, f, ensure_ascii=False)
20
  f.write('\n')
21
 
22
  def main():
@@ -42,8 +47,8 @@ def main():
42
  file_path = os.path.join(os.getcwd(), f"{output_dataset_name}.jsonl")
43
  save_combined_dataset_as_jsonl(combined_dataset, file_path)
44
  st.write("Download the combined dataset as JSONL file:")
45
- with open(file_path, "r") as f:
46
- bytes_data = f.read().encode()
47
  b64 = base64.b64encode(bytes_data).decode()
48
  href = f'<a href="data:file/jsonl;base64,{b64}" download="{output_dataset_name}.jsonl">Download JSONL File</a>'
49
  st.markdown(href, unsafe_allow_html=True)
@@ -55,4 +60,4 @@ def main():
55
  mime="application/jsonl")
56
 
57
  if __name__ == "__main__":
58
- main()
 
3
  import json
4
  import os
5
  import base64
6
+ from datetime import datetime
7
 
8
  def load_and_combine_datasets():
9
  python_codes_dataset = load_dataset('flytech/python-codes-25k', split='train')
 
14
 
15
  return combined_dataset
16
 
17
+ def datetime_serializer(o):
18
+ if isinstance(o, datetime):
19
+ return o.strftime('%Y-%m-%d %H:%M:%S')
20
+
21
  def save_combined_dataset_as_jsonl(combined_dataset, file_path):
22
  with open(file_path, 'w', encoding='utf-8') as f:
23
  for example in combined_dataset:
24
+ json.dump(example, f, ensure_ascii=False, default=datetime_serializer)
25
  f.write('\n')
26
 
27
  def main():
 
47
  file_path = os.path.join(os.getcwd(), f"{output_dataset_name}.jsonl")
48
  save_combined_dataset_as_jsonl(combined_dataset, file_path)
49
  st.write("Download the combined dataset as JSONL file:")
50
+ with open(file_path, "rb") as f:
51
+ bytes_data = f.read()
52
  b64 = base64.b64encode(bytes_data).decode()
53
  href = f'<a href="data:file/jsonl;base64,{b64}" download="{output_dataset_name}.jsonl">Download JSONL File</a>'
54
  st.markdown(href, unsafe_allow_html=True)
 
60
  mime="application/jsonl")
61
 
62
  if __name__ == "__main__":
63
+ main()