PoC push

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +1,12 @@
 ---
 title: 3GPPIndexers
-emoji: 📉
-colorFrom: red
-colorTo: yellow
+emoji: ⚙️
+colorFrom: blue
+colorTo: red
 sdk: gradio
 sdk_version: 5.29.0
 app_file: app.py
-pinned: false
+pinned: true
 license: gpl-3.0
 short_description: A Gradio app for indexing documents and specifications
 ---

app.py

@@ -0,0 +1,152 @@
+import gradio as gr
+import threading
+import sys
+import io
+import time
+from dotenv import load_dotenv
+import os
+from contextlib import redirect_stdout
+from git import Repo
+from huggingface_hub import Repository, HfApi
+
+load_dotenv()
+
+# == PATHS and SETTINGS ==
+DOC_INDEXER = "indexer_multi.py"
+SPEC_INDEXER = "spec_indexer_multi.py"
+DOC_INDEX_FILE = "indexed_docs.json"
+SPEC_INDEX_FILE = "indexed_specifications.json"
+GIT_REPO_PATH = os.path.abspath(".")  # or absolute path to repo
+HF_REPO_ID = "OrganizedProgrammers/3GPPDocFinder"
+HF_TOKEN = os.environ.get("HF_TOKEN")  # set this as env var
+
+# == Helpers ==
+def run_python_module(module_path):
+    """
+    Dynamically run a python module, capture and yield stdout in real time.
+    """
+    def runner():
+        local_vars = {}
+        buffer = io.StringIO()
+        try:
+            with redirect_stdout(buffer):
+                # Import as module, call main()
+                import runpy
+                runpy.run_path(module_path, run_name="__main__")
+        except Exception as e:
+            print(f"\n❌ Error: {e}")
+        finally:
+            yield buffer.getvalue()
+            buffer.close()
+    yield from runner()
+
+def commit_and_push_github(files, message):
+    repo = Repo(GIT_REPO_PATH)
+    repo.git.add(files)
+    repo.index.commit(message)
+    try:
+        repo.git.push()
+    except Exception as e:
+        print(f"Git push failed: {e}")
+
+def commit_and_push_hf(files, message):
+    if not HF_TOKEN:
+        return "No HF_TOKEN provided. Skipping HuggingFace push."
+    hf_repo_dir = os.path.join(GIT_REPO_PATH, "hf_spaces")
+    repo = None
+    if not os.path.exists(hf_repo_dir):
+        repo = Repository(
+            local_dir=hf_repo_dir,
+            clone_from=HF_REPO_ID,
+            token=HF_TOKEN,
+            skip_lfs_files=True
+        )
+    else:
+        repo = Repository(
+            local_dir=hf_repo_dir,
+            token=HF_TOKEN,
+            skip_lfs_files=True
+        )
+        repo.git_pull()
+    # Copy artifact files to huggingface space
+    for f in files:
+        import shutil
+        shutil.copy2(f, os.path.join(hf_repo_dir, f))
+    repo.git_add(auto_lfs_track=True)
+    repo.git_commit(message)
+    repo.git_push()
+    return "Pushed to HuggingFace."
+
+def get_docs_stats():
+    if os.path.exists(DOC_INDEX_FILE):
+        import json
+        with open(DOC_INDEX_FILE, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        return len(data["docs"])
+    return 0
+
+def get_specs_stats():
+    if os.path.exists(SPEC_INDEX_FILE):
+        import json
+        with open(SPEC_INDEX_FILE, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        return len(data["specs"])
+    return 0
+
+def get_scopes_stats():
+    if os.path.exists(SPEC_INDEX_FILE):
+        import json
+        with open(SPEC_INDEX_FILE, 'r', encoding="utf-8") as f:
+            data = json.load(f)
+        return len(data['scopes'])
+    return 0
+
+# == Gradio Functions ==
+
+def index_documents(progress=gr.Progress()):
+    progress(0, desc="Starting document indexing…")
+    log = ""
+    for output in run_python_module(DOC_INDEXER):
+        log = output
+        progress(0.7, desc="Indexing in progress...")
+        yield log
+    commit_and_push_github([DOC_INDEX_FILE], "Update doc index via Gradio")
+    commit_and_push_hf([DOC_INDEX_FILE], "Update doc index via Gradio")
+    progress(1, desc="Done!")
+    yield log + "\n\n✅ Finished! Committed and pushed."
+
+def index_specifications(progress=gr.Progress()):
+    progress(0, desc="Starting specifications indexing…")
+    log = ""
+    for output in run_python_module(SPEC_INDEXER):
+        log = output
+        progress(0.7, desc="Indexing in progress...")
+        yield log
+    commit_and_push_github([SPEC_INDEX_FILE], "Update spec index via Gradio")
+    commit_and_push_hf([SPEC_INDEX_FILE], "Update spec index via Gradio")
+    progress(1, desc="Done!")
+    yield log + "\n\n✅ Finished! Committed and pushed."
+
+def refresh_stats():
+    return str(get_docs_stats()), str(get_specs_stats()), str(get_scopes_stats())
+
+# == UI ==
+with gr.Blocks(theme=gr.themes.Soft()) as demo:
+    gr.Markdown("## 📄 3GPP Indexers")
+    with gr.Row():
+        with gr.Column():
+            doc_count = gr.Textbox(label="Docs Indexed", value=str(get_docs_stats()), interactive=False)
+            btn_docs = gr.Button("Re-index Documents", variant="primary")
+        with gr.Column():
+            spec_count = gr.Textbox(label="Specs Indexed", value=str(get_specs_stats()), interactive=False)
+            btn_specs = gr.Button("Re-index Specifications", variant="primary")
+        with gr.Column():
+            scope_count = gr.Textbox(label="Scopes Indexed", value=str(get_scopes_stats()), interactive=False)
+    out = gr.Textbox(label="Output/Log", lines=13)
+    refresh = gr.Button("🔄 Refresh Stats")
+    btn_docs.click(index_documents, outputs=out)
+    btn_specs.click(index_specifications, outputs=out)
+    refresh.click(refresh_stats, outputs=[doc_count, spec_count, scope_count])
+
+if __name__ == "__main__":
+    demo.launch()

indexed_docs.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dac006bf20aea05292515391d649398e0466e64a7a81c1b010a21c40cebb828e
+size 59739552

indexed_scopes.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6035f66c4f02a1961da415490a96a1e52ef137acfad06b6e9d9ba247abed5ef7
+size 2870060

indexed_specifications.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8758708d306204eb8531717e9119a292d3b997d17f469191a573882329f933c
+size 41158533