Update app.py

app.py

@@ -2,62 +2,69 @@ from huggingface_hub import InferenceClient
 import gradio as gr
 import random
 import pandas as pd
-from io import BytesIO
+from io import BytesIO 
+import csv
+import os
+import io 
+import tempfile
+import re
 
 client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
 
 def extract_text_from_excel(file):
     df = pd.read_excel(file)
-    text = ' '.join(df['data'].astype(str))
+    text = ' '.join(df['Unnamed: 1'].astype(str))
     return text
 
-def generate_sentences(text, temperature, max_new_tokens, top_p, repetition_penalty):
+def save_to_csv(sentence, output, filename="synthetic_data.csv"):
+    with open(filename, mode='a', newline='', encoding='utf-8') as file:
+        writer = csv.writer(file)
+        writer.writerow([sentence, output])
+
+def generate(file, temperature, max_new_tokens, top_p, repetition_penalty):
+    text = extract_text_from_excel(file)
     sentences = text.split('.')
     random.shuffle(sentences)  # Shuffle sentences
 
-    generated_data = []
+    with tempfile.NamedTemporaryFile(mode='w', newline='', delete=False, suffix='.csv') as tmp:
+        fieldnames = ['Original Sentence', 'Generated Sentence']
+        writer = csv.DictWriter(tmp, fieldnames=fieldnames)
+        writer.writeheader()  
 
-    for sentence in sentences:
-        sentence = sentence.strip()
-        if not sentence:
-            continue
+        for sentence in sentences:
+            sentence = sentence.strip()
+            if not sentence:
+                continue
 
-        generate_kwargs = {
-            "temperature": temperature,
-            "max_new_tokens": max_new_tokens,
-            "top_p": top_p,
-            "repetition_penalty": repetition_penalty,
-            "do_sample": True,
-            "seed": 42,
-        }
+            generate_kwargs = {
+                "temperature": temperature,
+                "max_new_tokens": max_new_tokens,
+                "top_p": top_p,
+                "repetition_penalty": repetition_penalty,
+                "do_sample": True,
+                "seed": 42,
+            }
 
-        try:
-            stream = client.text_generation(sentence, **generate_kwargs, stream=True, details=True, return_full_text=False)
-            output = ""
-            for response in stream:
-                output += response.token.text
+            try:
+                stream = client.text_generation(sentence, **generate_kwargs, stream=True, details=True, return_full_text=False)
+                output = ""
+                for response in stream:
+                    output += response.token.text
 
-            generated_sentences = [s.strip() for s in output.split('.') if s.strip()]
-            generated_data.extend([(sentence, generated_sentence) for generated_sentence in generated_sentences])
+                generated_sentences = re.split(r'(?<=[\.\!\?:])[\s\n]+', output)
+                generated_sentences = [s.strip() for s in generated_sentences if s.strip() and s != '.']
 
-        except Exception as e:
-            print(f"Error generating data for sentence '{sentence}': {e}")
+                for generated_sentence in generated_sentences:
+                    writer.writerow({'Original Sentence': sentence, 'Generated Sentence': generated_sentence})
 
-    return generated_data
+            except Exception as e:
+                print(f"Error generating data for sentence '{sentence}': {e}")
 
-def save_to_csv(data, filename="synthetic_data.csv"):
-    with open(filename, mode='w', newline='', encoding='utf-8') as file:
-        writer = csv.writer(file)
-        writer.writerow(['Original Sentence', 'Generated Sentence'])
-        writer.writerows(data)
+        tmp_path = tmp.name
 
-def generate(file, temperature, max_new_tokens, top_p, repetition_penalty):
-    text = extract_text_from_excel(file)
-    data = generate_sentences(text, temperature, max_new_tokens, top_p, repetition_penalty)
-    save_to_csv(data)
-    return gr.File.update(value=filename, visible=True)
+    return tmp_path
 
-gr.Interface(
+gr.Interface( 
     fn=generate,
     inputs=[
         gr.File(label="Upload Excel File", file_count="single", file_types=[".xlsx"]),
@@ -66,7 +73,7 @@ gr.Interface(
         gr.Slider(label="Top-p (nucleus sampling)", value=0.95, minimum=0.0, maximum=1, step=0.05, interactive=True, info="Higher values sample more low-probability tokens"),
         gr.Slider(label="Repetition penalty", value=1.0, minimum=1.0, maximum=2.0, step=0.1, interactive=True, info="Penalize repeated tokens"),
     ],
-    outputs=gr.File(label="Synthetic Data"),
+    outputs=gr.File(label="Synthetic Data "),
     title="SDG",
     description="AYE QABIL.",
     allow_flagging="never",