fix data lost

app.py

@@ -3,7 +3,7 @@ from transformers import pipeline, AutoModelForSeq2SeqLM, AutoTokenizer
 import os
 import torch
 import spaces
-from datasets import Dataset
+from datasets import Dataset, load_dataset
 import time
 import datetime
 
@@ -65,7 +65,7 @@ def preload_models():
     return nano_large_models, ultra_supreme_models
 
 def push_to_hf_dataset():
-    """ Save translations in HF dataset for monitoring """
+    """ Save translations in HF dataset for monitoring, preserving previous data """
     global translations_buffer, last_push_time
     
     if not translations_buffer:
@@ -75,18 +75,31 @@ def push_to_hf_dataset():
         print(f"[INFO] Pushing {len(translations_buffer)} translations to Hugging Face dataset...")
         
         # Create dataset from buffer
-        ds = Dataset.from_dict({
+        new_data = Dataset.from_dict({
             "source_text": [item["source_text"] for item in translations_buffer],
             "translated_text": [item["translated_text"] for item in translations_buffer],
             "model_used": [item["model_used"] for item in translations_buffer],
             "timestamp": [item["timestamp"] for item in translations_buffer]
         })
         
+        # Try to load existing dataset
+        try:
+            existing_dataset = load_dataset(DATASET_REPO, split="live_translations", token=TOKEN)
+            print(f"[INFO] Loaded existing dataset with {len(existing_dataset)} entries")
+            
+            # Concatenate existing data with new data
+            combined_dataset = existing_dataset.concatenate(new_data)
+            print(f"[INFO] Combined dataset now has {len(combined_dataset)} entries")
+        except Exception as e:
+            print(f"[INFO] No existing dataset found or error loading: {str(e)}")
+            print(f"[INFO] Creating new dataset")
+            combined_dataset = new_data
+        
         # Push to hub
-        ds.push_to_hub(
+        combined_dataset.push_to_hub(
             DATASET_REPO, 
             token=TOKEN,
-            split=f"live_translations",
+            split="live_translations",
             private=True,
         )
         
@@ -169,18 +182,15 @@ def gradio_app():
         
         translate_button = gr.Button("Translate")
         
-        # Status message -> hidden
-        # status = gr.Markdown(f"Translations in buffer: 0")
-        
         # Link input and output
         def translate_and_update_status(text, model):
             translation = translate_text(text, model)
-            return translation #, f"Translations in buffer: {len(translations_buffer)} (Will push when reaching {BATCH_SIZE} or after {UPDATE_INTERVAL/3600} hours)"
+            return translation
         
         translate_button.click(
             fn=translate_and_update_status,
             inputs=[input_text, model_choice],
-            outputs=[output_text] #, status] Status message -> hidden
+            outputs=[output_text]
         )
     
     return app