Merge remote-tracking branch 'origin/main'

.gitmodules

@@ -0,0 +1,2 @@
+[submodule "vllm"]
+	update = checkout

Dockerfile

@@ -0,0 +1,55 @@
+FROM docker.io/library/python:3.10@sha256:76f22e4ce53774c1f5eb0ba145edb57b908e7aa329fee75eca69b511c1d0cd8a
+
+WORKDIR /home/user/app
+
+# Install uv and create virtual environment
+RUN python -m pip install uv && \
+    uv venv --python=3.12 .venv && \
+    . .venv/bin/activate
+
+# Base pip updates and initial packages
+RUN pip install --no-cache-dir pip -U && \
+    pip install --no-cache-dir datasets "huggingface-hub>=0.19" "hf-transfer>=0.1.4" "protobuf<4" "click<8.1" "pydantic~=1.0"
+
+# System packages installation
+RUN --mount=target=/tmp/packages.txt,source=packages.txt \
+    apt-get update && \
+    xargs -r -a /tmp/packages.txt apt-get install -y && \
+    apt-get install -y curl && \
+    curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
+    apt-get install -y nodejs && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get clean
+
+# Fakeroot setup
+RUN apt-get update && \
+    apt-get install -y fakeroot && \
+    mv /usr/bin/apt-get /usr/bin/.apt-get && \
+    echo '#!/usr/bin/env sh\nfakeroot /usr/bin/.apt-get $@' > /usr/bin/apt-get && \
+    chmod +x /usr/bin/apt-get && \
+    rm -rf /var/lib/apt/lists/* && \
+    useradd -m -u 1000 user
+
+# Copy files and install requirements
+COPY --chown=1000:1000 --from=root / /
+
+# Install requirements in specific order
+RUN . .venv/bin/activate && \
+    pip install -r .venv/requirements-cpu.txt && \
+    pip install -e .venv/. && \
+    pip install -e .
+
+RUN pip freeze > /tmp/freeze.txt
+
+# Additional system packages
+RUN apt-get update && \
+    apt-get install -y git git-lfs ffmpeg libsm6 libxext6 cmake rsync libgl1-mesa-glx && \
+    rm -rf /var/lib/apt/lists/* && \
+    git lfs install
+
+# Install Gradio and related packages
+RUN . .venv/bin/activate && \
+    pip install --no-cache-dir gradio[oauth]==5.8.0 "uvicorn>=0.14.0" spaces
+
+COPY --link --chown=1000 ./ /home/user/app
+COPY --from=pipfreeze --link --chown=1000 /tmp/freeze.txt /tmp/freeze.txt

a.py

@@ -0,0 +1,51 @@
+import os
+import re
+
+def modify_synthetic_imports(file_path):
+    """Modify imports of synthetic_dataset_generator to add src."""
+    try:
+        with open(file_path, 'r') as file:
+            content = file.read()
+        
+        # Replace both import patterns to add src.
+        modified_content = re.sub(
+            r'from src.synthetic_dataset_generator',
+            'from src.synthetic_dataset_generator',
+            content
+        )
+        modified_content = re.sub(
+            r'import src.synthetic_dataset_generator',
+            'import src.synthetic_dataset_generator',
+            modified_content
+        )
+
+        # Only write if changes were made
+        if modified_content != content:
+            with open(file_path, 'w') as file:
+                file.write(modified_content)
+            print(f"Modified imports in: {file_path}")
+
+    except Exception as e:
+        print(f"Error processing {file_path}: {str(e)}")
+
+def process_directory(start_path):
+    """Recursively process all Python files in directory"""
+    for root, _, files in os.walk(start_path):
+        for file in files:
+            if file.endswith('.py'):
+                file_path = os.path.join(root, file)
+                modify_synthetic_imports(file_path)
+
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) != 2:
+        print("Usage: python script.py <directory_path>")
+        sys.exit(1)
+
+    directory_path = sys.argv[1]
+    if not os.path.isdir(directory_path):
+        print(f"Error: {directory_path} is not a valid directory")
+        sys.exit(1)
+
+    process_directory(directory_path)
+    print("Processing complete!")

app.py

@@ -1,3 +1,3 @@
-from synthetic_dataset_generator import launch
+from src.synthetic_dataset_generator import launch
 
 launch()

examples/argilla-deployment.py

@@ -6,7 +6,7 @@
 # ///
 import os
 
-from synthetic_dataset_generator import launch
+from src.synthetic_dataset_generator import launch
 
 # Follow https://docs.argilla.io/latest/getting_started/quickstart/ to get your Argilla API key and URL
 os.environ["HF_TOKEN"] = "hf_..."

examples/hf-dedicated-or-tgi-deployment.py

@@ -6,7 +6,7 @@
 # ///
 import os
 
-from synthetic_dataset_generator import launch
+from src.synthetic_dataset_generator import launch
 
 os.environ["HF_TOKEN"] = "hf_..."  # push the data to huggingface
 os.environ["HUGGINGFACE_BASE_URL"] = "http://127.0.0.1:3000/"  # dedicated endpoint/TGI

examples/hf-serverless-deployment.py

@@ -6,7 +6,7 @@
 # ///
 import os
 
-from synthetic_dataset_generator import launch
+from src.synthetic_dataset_generator import launch
 
 os.environ["HF_TOKEN"] = "hf_..."  # push the data to huggingface
 os.environ["MODEL"] = "meta-llama/Llama-3.1-8B-Instruct"  # use model for generation

examples/ollama-deployment.py

@@ -8,7 +8,7 @@
 # ollama run qwen2.5:32b-instruct-q5_K_S
 import os
 
-from synthetic_dataset_generator import launch
+from src.synthetic_dataset_generator import launch
 
 os.environ["HF_TOKEN"] = "hf_..."  # push the data to huggingface
 os.environ["OLLAMA_BASE_URL"] = "http://127.0.0.1:11434/"  # ollama base url

examples/openai-deployment.py

@@ -7,7 +7,7 @@
 
 import os
 
-from synthetic_dataset_generator import launch
+from src.synthetic_dataset_generator import launch
 
 os.environ["HF_TOKEN"] = "hf_..."  # push the data to huggingface
 os.environ["OPENAI_BASE_URL"] = "https://api.openai.com/v1/"  # openai base url

examples/vllm-deployment.py

@@ -7,7 +7,7 @@
 # vllm serve Qwen/Qwen2.5-1.5B-Instruct
 import os
 
-from synthetic_dataset_generator import launch
+from src.synthetic_dataset_generator import launch
 
 os.environ["HF_TOKEN"] = "hf_..."  # push the data to huggingface
 os.environ["VLLM_BASE_URL"] = "http://127.0.0.1:8000/"  # vllm base url

src/synthetic_dataset_generator/__init__.py

@@ -2,7 +2,7 @@ import inspect
 
 from gradio import TabbedInterface
 
-from synthetic_dataset_generator import (  # noqa
+from src.synthetic_dataset_generator import (  # noqa
     _distiset,
     _inference_endpoints,
 )
@@ -13,7 +13,7 @@ def launch(*args, **kwargs):
     Based on the `TabbedInterface` from Gradio.
     Parameters: https://www.gradio.app/docs/gradio/tabbedinterface
     """
-    from synthetic_dataset_generator.app import demo
+    from src.synthetic_dataset_generator.app import demo
 
     return demo.launch(*args, **kwargs)
 

src/synthetic_dataset_generator/__main__.py

@@ -1,4 +1,4 @@
 if __name__ == "__main__":
-    from synthetic_dataset_generator import launch
+    from src.synthetic_dataset_generator import launch
 
     launch()

src/synthetic_dataset_generator/app.py

@@ -1,10 +1,10 @@
-from synthetic_dataset_generator._tabbedinterface import TabbedInterface
+from src.synthetic_dataset_generator._tabbedinterface import TabbedInterface
 
-# from synthetic_dataset_generator.apps.eval import app as eval_app
-from synthetic_dataset_generator.apps.rag import app as rag_app
-from synthetic_dataset_generator.apps.about import app as about_app
-from synthetic_dataset_generator.apps.chat import app as chat_app
-from synthetic_dataset_generator.apps.textcat import app as textcat_app
+# from src.synthetic_dataset_generator.apps.eval import app as eval_app
+from src.synthetic_dataset_generator.apps.rag import app as rag_app
+from src.synthetic_dataset_generator.apps.about import app as about_app
+from src.synthetic_dataset_generator.apps.chat import app as chat_app
+from src.synthetic_dataset_generator.apps.textcat import app as textcat_app
 
 theme = "argilla/argilla-theme"
 

src/synthetic_dataset_generator/apps/base.py

@@ -12,8 +12,8 @@ from huggingface_hub import HfApi, upload_file, repo_exists
 from unstructured.chunking.title import chunk_by_title
 from unstructured.partition.auto import partition
 
-from synthetic_dataset_generator.constants import MAX_NUM_ROWS
-from synthetic_dataset_generator.utils import get_argilla_client
+from src.synthetic_dataset_generator.constants import MAX_NUM_ROWS
+from src.synthetic_dataset_generator.utils import get_argilla_client
 
 
 def validate_argilla_user_workspace_dataset(

src/synthetic_dataset_generator/apps/chat.py

@@ -13,7 +13,7 @@ from gradio.oauth import OAuthToken
 from gradio_huggingfacehub_search import HuggingfaceHubSearch
 from huggingface_hub import HfApi
 
-from synthetic_dataset_generator.apps.base import (
+from src.synthetic_dataset_generator.apps.base import (
     combine_datasets,
     hide_success_message,
     load_dataset_from_hub,
@@ -24,15 +24,15 @@ from synthetic_dataset_generator.apps.base import (
     validate_argilla_user_workspace_dataset,
     validate_push_to_hub,
 )
-from synthetic_dataset_generator.constants import (
+from src.synthetic_dataset_generator.constants import (
     BASE_URL,
     DEFAULT_BATCH_SIZE,
     MODEL,
     MODEL_COMPLETION,
     SFT_AVAILABLE,
 )
-from synthetic_dataset_generator.pipelines.base import get_rewritten_prompts
-from synthetic_dataset_generator.pipelines.chat import (
+from src.synthetic_dataset_generator.pipelines.base import get_rewritten_prompts
+from src.synthetic_dataset_generator.pipelines.chat import (
     DEFAULT_DATASET_DESCRIPTIONS,
     generate_pipeline_code,
     get_follow_up_generator,
@@ -41,12 +41,11 @@ from synthetic_dataset_generator.pipelines.chat import (
     get_response_generator,
     get_sentence_pair_generator,
 )
-from synthetic_dataset_generator.pipelines.embeddings import (
+from src.synthetic_dataset_generator.pipelines.embeddings import (
     get_embeddings,
     get_sentence_embedding_dimensions,
 )
-from synthetic_dataset_generator.utils import (
-    column_to_list,
+from src.synthetic_dataset_generator.utils import (
     get_argilla_client,
     get_org_dropdown,
     get_random_repo_name,

src/synthetic_dataset_generator/apps/eval.py

@@ -17,7 +17,7 @@ from gradio.oauth import OAuthToken  #
 from gradio_huggingfacehub_search import HuggingfaceHubSearch
 from huggingface_hub import HfApi
 
-from synthetic_dataset_generator.apps.base import (
+from src.synthetic_dataset_generator.apps.base import (
     combine_datasets,
     get_iframe,
     hide_success_message,
@@ -27,17 +27,17 @@ from synthetic_dataset_generator.apps.base import (
     validate_argilla_user_workspace_dataset,
     validate_push_to_hub,
 )
-from synthetic_dataset_generator.constants import DEFAULT_BATCH_SIZE
-from synthetic_dataset_generator.pipelines.embeddings import (
+from src.synthetic_dataset_generator.constants import DEFAULT_BATCH_SIZE
+from src.synthetic_dataset_generator.pipelines.embeddings import (
     get_embeddings,
     get_sentence_embedding_dimensions,
 )
-from synthetic_dataset_generator.pipelines.eval import (
+from src.synthetic_dataset_generator.pipelines.eval import (
     generate_pipeline_code,
     get_custom_evaluator,
     get_ultrafeedback_evaluator,
 )
-from synthetic_dataset_generator.utils import (
+from src.synthetic_dataset_generator.utils import (
     column_to_list,
     extract_column_names,
     get_argilla_client,

src/synthetic_dataset_generator/apps/rag.py

@@ -13,7 +13,7 @@ from gradio.oauth import OAuthToken
 from gradio_huggingfacehub_search import HuggingfaceHubSearch
 from huggingface_hub import HfApi
 
-from synthetic_dataset_generator.apps.base import (
+from src.synthetic_dataset_generator.apps.base import (
     combine_datasets,
     hide_success_message,
     load_dataset_from_hub,
@@ -24,13 +24,13 @@ from synthetic_dataset_generator.apps.base import (
     validate_argilla_user_workspace_dataset,
     validate_push_to_hub,
 )
-from synthetic_dataset_generator.constants import DEFAULT_BATCH_SIZE, MODEL, MODEL_COMPLETION
-from synthetic_dataset_generator.pipelines.base import get_rewritten_prompts
-from synthetic_dataset_generator.pipelines.embeddings import (
+from src.synthetic_dataset_generator.constants import DEFAULT_BATCH_SIZE
+from src.synthetic_dataset_generator.pipelines.base import get_rewritten_prompts
+from src.synthetic_dataset_generator.pipelines.embeddings import (
     get_embeddings,
     get_sentence_embedding_dimensions,
 )
-from synthetic_dataset_generator.pipelines.rag import (
+from src.synthetic_dataset_generator.pipelines.rag import (
     DEFAULT_DATASET_DESCRIPTIONS,
     generate_pipeline_code,
     get_chunks_generator,
@@ -38,7 +38,7 @@ from synthetic_dataset_generator.pipelines.rag import (
     get_response_generator,
     get_sentence_pair_generator,
 )
-from synthetic_dataset_generator.utils import (
+from src.synthetic_dataset_generator.utils import (
     column_to_list,
     get_argilla_client,
     get_org_dropdown,

src/synthetic_dataset_generator/apps/textcat.py

@@ -10,7 +10,7 @@ from datasets import ClassLabel, Dataset, Features, Sequence, Value
 from distilabel.distiset import Distiset
 from huggingface_hub import HfApi
 
-from synthetic_dataset_generator.apps.base import (
+from src.synthetic_dataset_generator.apps.base import (
     combine_datasets,
     hide_success_message,
     push_pipeline_code_to_hub,
@@ -19,20 +19,20 @@ from synthetic_dataset_generator.apps.base import (
     validate_argilla_user_workspace_dataset,
     validate_push_to_hub,
 )
-from synthetic_dataset_generator.constants import DEFAULT_BATCH_SIZE
-from synthetic_dataset_generator.pipelines.base import get_rewritten_prompts
-from synthetic_dataset_generator.pipelines.embeddings import (
+from src.synthetic_dataset_generator.constants import DEFAULT_BATCH_SIZE
+from src.synthetic_dataset_generator.pipelines.base import get_rewritten_prompts
+from src.synthetic_dataset_generator.pipelines.embeddings import (
     get_embeddings,
     get_sentence_embedding_dimensions,
 )
-from synthetic_dataset_generator.pipelines.textcat import (
+from src.synthetic_dataset_generator.pipelines.textcat import (
     DEFAULT_DATASET_DESCRIPTIONS,
     generate_pipeline_code,
     get_labeller_generator,
     get_prompt_generator,
     get_textcat_generator,
 )
-from synthetic_dataset_generator.utils import (
+from src.synthetic_dataset_generator.utils import (
     get_argilla_client,
     get_org_dropdown,
     get_preprocess_labels,

src/synthetic_dataset_generator/pipelines/base.py

@@ -4,7 +4,7 @@ import random
 from distilabel.models import ClientvLLM, InferenceEndpointsLLM, OllamaLLM, OpenAILLM
 from distilabel.steps.tasks import TextGeneration
 
-from synthetic_dataset_generator.constants import (
+from src.synthetic_dataset_generator.constants import (
     API_KEYS,
     DEFAULT_BATCH_SIZE,
     HUGGINGFACE_BASE_URL,

src/synthetic_dataset_generator/pipelines/chat.py

@@ -6,11 +6,11 @@ from distilabel.steps.tasks import (
     TextGeneration,
 )
 
-from synthetic_dataset_generator.constants import (
+from src.synthetic_dataset_generator.constants import (
     MAGPIE_PRE_QUERY_TEMPLATE,
     MAX_NUM_TOKENS,
 )
-from synthetic_dataset_generator.pipelines.base import _get_llm, _get_llm_class
+from src.synthetic_dataset_generator.pipelines.base import _get_llm, _get_llm_class
 
 INFORMATION_SEEKING_PROMPT = (
     "You are an AI assistant designed to provide accurate and concise information on a wide"

src/synthetic_dataset_generator/pipelines/embeddings.py

@@ -3,7 +3,7 @@ from typing import List
 from sentence_transformers import SentenceTransformer
 from sentence_transformers.models import StaticEmbedding
 
-from synthetic_dataset_generator.constants import STATIC_EMBEDDING_MODEL
+from src.synthetic_dataset_generator.constants import STATIC_EMBEDDING_MODEL
 
 static_embedding = StaticEmbedding.from_model2vec(STATIC_EMBEDDING_MODEL)
 model = SentenceTransformer(modules=[static_embedding])

src/synthetic_dataset_generator/pipelines/eval.py

@@ -7,9 +7,9 @@ from distilabel.steps.tasks import (
     UltraFeedback,
 )
 
-from synthetic_dataset_generator.constants import BASE_URL, MAX_NUM_TOKENS, MODEL
-from synthetic_dataset_generator.pipelines.base import _get_next_api_key
-from synthetic_dataset_generator.utils import extract_column_names
+from src.synthetic_dataset_generator.constants import BASE_URL, MAX_NUM_TOKENS, MODEL
+from src.synthetic_dataset_generator.pipelines.base import _get_next_api_key
+from src.synthetic_dataset_generator.utils import extract_column_names
 
 
 def get_ultrafeedback_evaluator(aspect: str, is_sample: bool):

src/synthetic_dataset_generator/pipelines/rag.py

@@ -4,8 +4,8 @@ from distilabel.steps.tasks import (
     TextGeneration,
 )
 
-from synthetic_dataset_generator.constants import MAX_NUM_TOKENS
-from synthetic_dataset_generator.pipelines.base import _get_llm, _get_llm_class
+from src.synthetic_dataset_generator.constants import MAX_NUM_TOKENS
+from src.synthetic_dataset_generator.pipelines.base import _get_llm, _get_llm_class
 
 DEFAULT_DATASET_DESCRIPTIONS = [
     "A dataset to retrieve information from legal documents.",

src/synthetic_dataset_generator/pipelines/textcat.py

@@ -8,11 +8,11 @@ from distilabel.steps.tasks import (
 )
 from pydantic import BaseModel, Field
 
-from synthetic_dataset_generator.constants import (
+from src.synthetic_dataset_generator.constants import (
     MAX_NUM_TOKENS,
 )
-from synthetic_dataset_generator.pipelines.base import _get_llm, _get_llm_class
-from synthetic_dataset_generator.utils import get_preprocess_labels
+from src.synthetic_dataset_generator.pipelines.base import _get_llm, _get_llm_class
+from src.synthetic_dataset_generator.utils import get_preprocess_labels
 
 PROMPT_CREATION_PROMPT = """You are an AI assistant specialized in generating very precise text classification tasks for dataset creation.
 

src/synthetic_dataset_generator/utils.py

@@ -14,7 +14,7 @@ from gradio.oauth import (
 from huggingface_hub import whoami
 from jinja2 import Environment, meta
 
-from synthetic_dataset_generator.constants import argilla_client
+from src.synthetic_dataset_generator.constants import argilla_client
 
 
 def get_duplicate_button():