private dataset structure

app.py

@@ -5,6 +5,7 @@ from datetime import datetime
 from huggingface_hub import HfApi
 from typing import Optional
 from PIL import Image  # Needed for working with PIL images
+import datasets
 
 # The list of sentences from our previous conversation.
 sentences = [
@@ -129,23 +130,16 @@ def create_gradio_interface():
             return f"Logged in as: {profile.username}", {"username": profile.username}
     
         def handle_submit(profile, dataset_choice, image, text):
-            """
-            Depending on the dataset toggle selection, this function either:
-              - For 'Private': strips metadata from the image, checks/creates a private HF dataset,
-                pushes the image to it, and logs the submission.
-              - For 'Public': uses the existing `submit_image` logic.
-            """
             if not profile or "username" not in profile:
                 raise gr.Error("Please log in to use this application")
             username = profile["username"]
-            # Use the username from the OAuth profile directly
             repo_id = f"{username}/handwriting-ocr-private"
             
             if dataset_choice == "Private":
                 # Remove all metadata for privacy
                 stripped_image = strip_metadata(image)
                 
-                # Check if the dataset exists; if not, create it as private.
+                # Check if the dataset exists; if not, create it as private
                 try:
                     collector.hf_api.dataset_info(repo_id)
                 except Exception as e:
@@ -159,30 +153,39 @@ def create_gradio_interface():
                 temp_path = os.path.join(temp_dir, filename)
                 stripped_image.save(temp_path)
                 
-                # Upload the file to the Hugging Face dataset repository
-                collector.hf_api.upload_file(
-                    path_or_fileobj=temp_path,
-                    path_in_repo=filename,
-                    repo_id=repo_id,
-                    repo_type="dataset",
-                )
+                # Create a dataset dictionary with the image-text pair
+                features = datasets.Features({
+                    'text': datasets.Value('string'),
+                    'image': datasets.Image(),
+                    'timestamp': datasets.Value('string')
+                })
+                
+                dataset_dict = {
+                    'text': [text],
+                    'image': [temp_path],
+                    'timestamp': [timestamp]
+                }
+                
+                # Create the dataset and push to hub
+                dataset = datasets.Dataset.from_dict(dataset_dict, features=features)
+                dataset.push_to_hub(repo_id)
                 
                 # Remove the temporary file
                 os.remove(temp_path)
                 
-                # Log the submission locally with an indicator for the private dataset.
+                # Log the submission locally
                 collector.collected_pairs.append({
                     "text": text,
-                    "image": image,  # The original image is recorded locally.
+                    "image": image,
                     "timestamp": timestamp,
                     "username": username,
                     "dataset": "private"
                 })
+                
                 new_text = collector.get_random_text_block()
-                # Return a tuple to clear the image input (set to None) and update the text.
                 return None, new_text
             else:
-                # Fallback to public submission using the existing logic.
+                # Fallback to public submission
                 new_text = collector.submit_image(image, text, username)
                 return None, new_text
 

requirements.txt

@@ -4,4 +4,5 @@ Pillow>=10.0.0
 pytest>=7.0.0
 pytest-playwright>=0.4.0
 pytest-asyncio>=0.23.0
-playwright>=1.40.0 
+playwright>=1.40.0
+datasets>=2.16.0