removed uneeded scripts for ease of use

dl_bin.py

@@ -1,32 +0,0 @@
-import requests
-from typing import Optional
-
-def download_binary_file(url: str, file_path: Optional[str] = None) -> None:
-    """
-    Download a binary file from a given URL and save it to the specified path.
-
-    :param url: URL of the binary file to be downloaded.
-    :param file_path: Local path to save the file. If None, the file will be saved with its original name.
-    """
-    try:
-        response = requests.get(url, stream=True)
-        response.raise_for_status()
-
-        # If no specific file path is provided, extract the file name from the URL
-        if file_path is None:
-            file_path = url.split('/')[-1]
-
-        with open(file_path, 'wb') as file:
-            for chunk in response.iter_content(chunk_size=8192):
-                file.write(chunk)
-
-        print(f"File downloaded successfully: {file_path}")
-    except requests.exceptions.RequestException as e:
-        print(f"Error downloading file: {e}")
-
-# Example usage
-url = "https://llamahack.slack.com/files/U069A8NRB9T/F068ZTLK9KR/anthem_hsa_medical_insurance_benefit_booklet.pdf"
-# download_binary_file(url)
-
-import urllib.request
-urllib.request.urlretrieve(url, "filename.pdf")

ingest.py

@@ -1,43 +0,0 @@
-from llama_index import SimpleDirectoryReader, VectorStoreIndex, ServiceContext
-from llama_index.text_splitter import SentenceSplitter
-import dotenv
-import os
-
-dotenv.load_dotenv()
-
-
-
-documents = SimpleDirectoryReader("./data").load_data()
-
-text_splitter = SentenceSplitter(chunk_size=512, chunk_overlap=10)
-service_context = ServiceContext.from_defaults(text_splitter=text_splitter)
-
-index = VectorStoreIndex.from_documents(
-    documents, service_context=service_context
-)
-
-query_engine = index.as_query_engine()
-
-
-# from llama_index.query import QueryBuilder
-
-# Define the query text
-query_text = "How does the weather affect crop growth?"
-
-data = query_engine.query(query_text)
-
-# Preprocess the query text
-# query_builder = QueryBuilder(service_context)
-# query = query_builder.build_query(query_text)
-
-# # Search for similar documents or retrieve relevant information
-# results = index.search(query)
-
-# Process the search results
-for result in results:
-    document_id = result.document_id
-    score = result.score
-    document = documents[document_id]
-    # Process the retrieved document or display the relevant information
-    print(f"Document ID: {document_id}, Score: {score}")
-    print(f"Document Text: {document.text}")

utils.py

@@ -1,54 +0,0 @@
-import requests
-import os
-from typing import Optional
-from urllib.parse import urlparse, unquote
-import gdrive
-
-def get_filename_from_url(url: str, cd: Optional[str]) -> str:
-    """
-    Extracts and returns the filename from the URL or content-disposition header.
-    """
-    if cd:
-        fname = [x.strip() for x in cd.split(';') if x.strip().startswith('filename=')]
-        if fname:
-            return unquote(fname[0].split('=')[1].strip('"'))
-
-    # Fallback to extracting filename from URL
-    parsed_url = urlparse(url)
-    return os.path.basename(parsed_url.path)
-
-def download_file(url: str, save_dir: Optional[str] = None, save_name: Optional[str] = None) -> None:
-    """
-    Downloads a file from the given URL and saves it in the specified directory.
-    If the directory does not exist, it will be created.
-    """
-    try:
-        response = requests.get(url, stream=True)
-        response.raise_for_status()
-
-        filename = save_name if save_name else get_filename_from_url(url, response.headers.get('content-disposition'))
-
-        if save_dir:
-            os.makedirs(save_dir, exist_ok=True)
-            file_path = os.path.join(save_dir, filename)
-        else:
-            file_path = filename
-
-        with open(file_path, 'wb') as file:
-            for chunk in response.iter_content(chunk_size=8192): 
-                if chunk:
-                    file.write(chunk)
-
-        print(f"File downloaded and saved as: {file_path}")
-
-    except requests.exceptions.HTTPError as err:
-        print(f"HTTP Error: {err}")
-    except Exception as e:
-        print(f"Error: {e}")
-
-
-
-if __name__ == "__main__":
-    # Example Usage
-    url = "https://llamahack.slack.com/files/U069A8NRB9T/F068ZTLK9KR/anthem_hsa_medical_insurance_benefit_booklet.pdf"
-    download_file(url,"data")