Update app.py

app.py

@@ -1,11 +1,11 @@
+import json
 from huggingface_hub import InferenceClient
 import gradio as gr
 import random
 import pandas as pd
-from io import BytesIO 
-import csv
+from io import BytesIO
 import os
-import io 
+import io
 import tempfile
 import re
 
@@ -13,24 +13,20 @@ client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
 
 def extract_sentences_from_excel(file):
     df = pd.read_excel(file)
-    text = ' '.join(df['Unnamed: 1'].astype(str))
+    text = ' '.join(df['Column_Name'].astype(str))
     sentences = text.split('.')
     sentences = [s.strip() for s in sentences if s.strip()]
     return sentences
 
-def save_to_csv(sentence, output, filename="synthetic_data.csv"):
-    with open(filename, mode='a', newline='', encoding='utf-8') as file:
-        writer = csv.writer(file)
-        writer.writerow([sentence, output])
+def save_to_json(data, filename="synthetic_data.json"):
+    with open(filename, mode='a', encoding='utf-8') as file:
+        json.dump(data, file, indent=4, ensure_ascii=False)
 
 def generate(file, prompt, temperature, max_new_tokens, top_p, repetition_penalty):
     sentences = extract_sentences_from_excel(file)
+    data = []
 
-    with tempfile.NamedTemporaryFile(mode='w', newline='', delete=False, suffix='.csv') as tmp:
-        fieldnames = ['Original Sentence', 'Generated Sentence']
-        writer = csv.DictWriter(tmp, fieldnames=fieldnames)
-        writer.writeheader()  
-
+    with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json') as tmp:
         for sentence in sentences:
             sentence = sentence.strip()
             if not sentence:
@@ -46,25 +42,26 @@ def generate(file, prompt, temperature, max_new_tokens, top_p, repetition_penalt
             }
 
             try:
-                stream = client.text_generation(f"{prompt} {sentence}", **generate_kwargs, stream=True, details=True, return_full_text=False)
+                stream = client.text_generation(f"{prompt} Output the response in JSON format.", **generate_kwargs, stream=True, details=True, return_full_text=False)
                 output = ""
                 for response in stream:
                     output += response.token.text
 
-                generated_sentences = re.split(r'(?<=[\.\!\?:])[\s\n]+', output)
-                generated_sentences = [s.strip() for s in generated_sentences if s.strip() and s != '.']
-
-                for generated_sentence in generated_sentences:
-                    writer.writerow({'Original Sentence': sentence, 'Generated Sentence': generated_sentence})
+                try:
+                    json_output = json.loads(output)
+                    data.append({"original_sentence": sentence, "generated_sentence": json_output})
+                except json.JSONDecodeError:
+                    print(f"Error decoding JSON for sentence '{sentence}': {output}")
 
             except Exception as e:
                 print(f"Error generating data for sentence '{sentence}': {e}")
 
+        save_to_json(data, tmp.name)
         tmp_path = tmp.name
 
     return tmp_path
 
-gr.Interface( 
+gr.Interface(
     fn=generate,
     inputs=[
         gr.File(label="Upload Excel File", file_count="single", file_types=[".xlsx"]),