Create app.py

app.py

@@ -0,0 +1,256 @@
+gooodo shiet
+
+
+
+import os
+import json
+import gradio as gr
+import zipfile
+import tempfile
+import requests
+import urllib.parse
+import io
+
+from huggingface_hub import HfApi, login
+#from PyPDF2 import PdfReader
+#from langchain_huggingface import HuggingFaceEmbeddings
+#from langchain_community.vectorstores import Chroma
+#from langchain.text_splitter import RecursiveCharacterTextSplitter
+#from langchain_groq import ChatGroq
+from dotenv import load_dotenv
+#from langchain.docstore.document import Document
+
+# Load environment variables from .env file
+load_dotenv()
+
+# Load configuration from JSON file
+with open('config.json') as config_file:
+    config = json.load(config_file)
+
+
+PERSIST_DIRECTORY = config["persist_directory"]
+CHUNK_SIZE = config["chunk_size"]
+CHUNK_OVERLAP = config["chunk_overlap"]
+EMBEDDING_MODEL_NAME = config["embedding_model"]
+LLM_MODEL_NAME = config["llm_model"]
+LLM_TEMPERATURE = config["llm_temperature"]
+GITLAB_API_URL = config["gitlab_api_url"]
+HF_SPACE_NAME = config["hf_space_name"]
+REPOSITORY_DIRECTORY = config["repository_directory"]
+
+GROQ_API_KEY = os.environ["GROQ_API_KEY"]
+HF_TOKEN = os.environ["HF_Token"]
+
+
+
+login(HF_TOKEN)
+api = HfApi()
+
+def load_project_ids(json_file):
+    with open(json_file, 'r') as f:
+        data = json.load(f)
+    return data['project_ids']
+
+def upload_gitRepository():
+    project_ids = load_project_ids('repository_ids.json')
+    
+    for project_id in project_ids:
+        encoded_project_id = urllib.parse.quote_plus(project_id)
+        # Define the URL to download the repository archive
+        archive_url = f"{GITLAB_API_URL}/projects/{encoded_project_id}/repository/archive.zip"
+        
+        # Download the repository archive
+        response = requests.get(archive_url)
+        archive_bytes = io.BytesIO(response.content)
+        
+        # Retrieve the original file name from the response headers
+        content_disposition = response.headers.get('content-disposition')
+        if content_disposition:
+            filename = content_disposition.split('filename=')[-1].strip('\"')
+        else:
+            filename = 'archive.zip'  # Fallback to a default name if not found
+
+        # Check if the file already exists in the repository
+        existing_files = api.list_repo_files(repo_id=HF_SPACE_NAME, repo_type='space')
+        target_path = f"{REPOSITORY_DIRECTORY}/{filename}"
+        
+        if target_path in existing_files:
+            print(f"File '{target_path}' already exists in the repository. Skipping upload...")
+            continue
+
+        # Upload the ZIP file to the new folder in the Hugging Face space repository
+        api.upload_file(
+            path_or_fileobj=archive_bytes,
+            path_in_repo=target_path,
+            repo_id=HF_SPACE_NAME,
+            repo_type='space'
+        )
+        
+        print(f"Repository '{project_id}' downloaded and uploaded to Hugging Face space '{HF_SPACE_NAME}' in folder '{DIRECTORY}' with original file name '{filename}'")
+
+def process_directory(directory):
+    all_texts = []
+    file_references = []
+
+    if not os.path.exists(directory):
+        raise ValueError(f"Directory {directory} does not exist.")
+
+    # Find all zip files in the directory
+    zip_files = [file for file in os.listdir(directory) if file.endswith('.zip')]
+    
+    if not zip_files:
+        print("No zip files found in the directory.")
+    else:
+        for zip_filename in zip_files:
+            zip_file_path = os.path.join(directory, zip_filename)
+            
+            # Create a temporary directory for each zip file
+            with tempfile.TemporaryDirectory() as tmpdirname:
+                # Unzip the file into the temporary directory
+                with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:
+                    zip_ref.extractall(tmpdirname)
+                
+                # Process the temporary directory
+                temp_texts, temp_references = process_directory(tmpdirname)
+                all_texts.extend(temp_texts)
+                file_references.extend(temp_references)
+    
+    for root, _, files in os.walk(directory):
+        for file in files:
+            file_path = os.path.join(root, file)
+            file_ext = os.path.splitext(file_path)[1]
+
+            if os.path.getsize(file_path) == 0:
+                print(f"Skipping an empty file: {file_path}")
+                continue
+
+            with open(file_path, 'rb') as f:
+                if file_ext in ['.rst', '.md', '.txt', '.html', '.json', '.yaml']:
+                    text = f.read().decode('utf-8')
+                elif file_ext == '.pdf':
+                    reader = PdfReader(f)
+                    text = ""
+                    for page in reader.pages:
+                        text += page.extract_text()
+                elif file_ext in ['.svg']:
+                    text = f"SVG file content from {file_path}"
+                elif file_ext in ['.png', '.ico']:
+                    text = f"Image metadata from {file_path}"
+                else:
+                    continue
+
+                all_texts.append(text)
+                file_references.append(file_path)
+
+    return all_texts, file_references
+
+
+
+
+# Split text into chunks
+def split_into_chunks(texts, references, chunk_size, chunk_overlap):
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+    chunks = []
+
+    for text, reference in zip(texts, references):
+        chunks.extend([Document(page_content=chunk, metadata={"source": reference}) for chunk in text_splitter.split_text(text)])
+
+    return chunks
+
+# Setup Chroma
+def setup_chroma(chunks, model_name="sentence-transformers/all-mpnet-base-v2", persist_directory="chroma_data"):
+    embedding_model = HuggingFaceEmbeddings(model_name=model_name)
+    vectorstore = Chroma.from_documents(chunks, embedding=embedding_model, persist_directory=persist_directory)
+    return vectorstore
+
+# Setup LLM
+def setup_llm(model_name, temperature, api_key):
+    llm = ChatGroq(model=model_name, temperature=temperature, api_key=api_key)
+    return llm
+
+def query_chroma(vectorstore, query, k):
+    results = vectorstore.similarity_search(query, k=k)
+    chunks_with_references = [(result.page_content, result.metadata["source"]) for result in results]
+    # Print the chosen chunks and their sources to the console
+    print("\nChosen chunks and their sources for the query:")
+    for chunk, source in chunks_with_references:
+        print(f"Source: {source}\nChunk: {chunk}\n")
+        print("-" * 50)
+    return chunks_with_references
+
+def rag_workflow(query):
+    docs = query_chroma(vectorstore, query, k=5)
+    context = "\n\n".join([doc for doc, _ in docs])
+    references = "\n".join([f"[{i+1}] {ref}" for i, (_, ref) in enumerate(docs)])
+    print(f"Context for the query:\n{context}\n")
+    print(f"References for the query:\n{references}\n")
+    prompt = f"You are an intelligent AI assistant who is very good in giving answers for anything asked or instructed by the user. Provide a clear and concise answer based only on the pieces of retrieved context. You must follow this very strictly, do not use anything else other than the retrieved context. If no related Information is found from the context, reply that you do not know. \n\nContext:\n{context}\n\nQuery: {query}"
+
+    
+    response = llm.invoke(prompt)
+    return response.content, references
+
+
+def initialize():
+    global vectorstore, chunks, llm
+
+    upload_gitRepository()
+    all_texts, file_references = process_directory(REPOSITORY_DIRECTORY)
+
+    for i, text in enumerate(all_texts): 
+        print(f"Content of file {i+1}:\n") 
+        print(text) 
+        print("\n" + "-"*40 + "\n")
+    
+    #chunks = split_into_chunks(all_texts, file_references, CHUNK_SIZE, CHUNK_OVERLAP)
+    #vectorstore = setup_chroma(chunks, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY)
+    #llm = setup_llm(LLM_MODEL_NAME, LLM_TEMPERATURE, GROQ_API_KEY)
+
+# Gradio utils
+def check_input_text(text):
+    if not text:
+        gr.Warning("Please input a question.")
+        raise TypeError
+    return True
+
+def add_text(history, text):
+    history = history + [(text, None)]
+    yield history, ""
+
+def bot_kadi(history):
+    user_query = history[-1][0]
+    response, references = rag_workflow(user_query)
+    history[-1] = (user_query, response)
+
+    # Format references for display with text passages
+    formatted_references = ""
+    docs = query_chroma(vectorstore, user_query, k=5)
+    for i, (doc, ref) in enumerate(docs):
+        formatted_references += f"""
+        <div style="border: 1px solid #ddd; padding: 10px; margin-bottom: 10px; border-radius: 5px;">
+            <h3 style="margin-top: 0;">Reference {i+1}</h3>
+            <p><strong>Source:</strong> {ref}</p>
+            <button onclick="var elem = document.getElementById('text-{i}'); var button = this; if (elem.style.display === 'block') {{ elem.style.display = 'none'; button.innerHTML = '&#9654; show source text'; }} else {{ elem.style.display = 'block'; button.innerHTML = '&#9660; hide source text'; }}">{{'&#9654; show source text'}}</button>
+            <div id="text-{i}" style="display: none;">
+                <p><strong>Text:</strong> {doc}</p>
+            </div>
+        </div>
+        """
+
+    yield history, formatted_references    
+
+def main():
+    
+    def greet(name, intensity):
+        return "Hello, " + name + "!" * int(intensity)
+
+    demo = gr.Interface(
+        fn=greet,
+        inputs=["text", "slider"],
+        outputs=["text"],
+    )
+
+    demo.launch   
+
+initialize()
+main()