reduced to 3b

app.py

@@ -1,20 +1,18 @@
 import streamlit as st
-
+import torch
 from langchain_text_splitters import Language, RecursiveCharacterTextSplitter
 from langchain_community.embeddings import HuggingFaceEmbeddings
 from langchain_community.vectorstores import FAISS
 from langchain_community.llms.huggingface_pipeline import HuggingFacePipeline
 from langchain.memory import ConversationBufferMemory
 from langchain.chains import ConversationalRetrievalChain
-from transformers import pipeline
 from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
-import torch
 
 
 
-gpt_model = 'gpt-4-1106-preview'
-embedding_model = 'text-embedding-3-small'
-default_model_id = "bigcode/starcoder2-7b"
+# gpt_model = 'gpt-4-1106-preview'
+# embedding_model = 'text-embedding-3-small'
+default_model_id = "bigcode/starcoder2-3b"
 
 def init():
     if "conversation" not in st.session_state:
@@ -23,12 +21,13 @@ def init():
         st.session_state.chat_history = None        
 
 def init_llm_pipeline(model_id):
-    if "llm" not in st.session_state:        
-        tokenizer = AutoTokenizer.from_pretrained(model_id)
+    if "llm" not in st.session_state:     
+
         model = AutoModelForCausalLM.from_pretrained(
             model_id,
-            device_map="auto",
-        )
+            device_map="auto"
+        )      
+        tokenizer = AutoTokenizer.from_pretrained(model_id)
         tokenizer.add_eos_token = True
         tokenizer.pad_token_id = 0
         tokenizer.padding_side = "left"
@@ -37,21 +36,17 @@ def init_llm_pipeline(model_id):
         model=model,
         tokenizer=tokenizer,
         task="text-generation",
-        temperature=0.2,
-        repetition_penalty=1.1,
-        return_full_text=True,
-        max_new_tokens=300,
+        max_new_tokens=1024
         )
         st.session_state.llm = HuggingFacePipeline(pipeline=text_generation_pipeline)          
 
-def get_text(docs):
-    return docs.getvalue().decode("utf-8")
-
-def get_vectorstore(documents):
+def get_retriever(files):
+    documents = [doc.getvalue().decode("utf-8") for doc in files]
     python_splitter = RecursiveCharacterTextSplitter.from_language(
         language=Language.PYTHON, chunk_size=2000, chunk_overlap=200
     )
-    texts = python_splitter.split_documents(documents)
+
+    texts = python_splitter.create_documents(documents)
 
     embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
 
@@ -83,7 +78,6 @@ def handle_user_input(question):
                 st.write(message.content)
 
 def main():
-    #load_dotenv()
     init()
 
     st.set_page_config(page_title="Coding-Assistent", page_icon=":books:")
@@ -103,9 +97,8 @@ def main():
         if st.button("Hochladen"):
             with st.spinner("Analysiere Dokumente ..."):
                 init_llm_pipeline(model_id)
-                raw_text = get_text(upload_docs)
-                vectorstore = get_vectorstore(raw_text)
-                st.session_state.conversation = get_conversation(vectorstore) 
+                retriever = get_retriever(upload_docs)
+                st.session_state.conversation = get_conversation(retriever) 
 
 
 if __name__ == "__main__":

requirements.txt

@@ -7,5 +7,5 @@ huggingface-hub
 torch
 accelerate
 langchain-text-splitters
-sentence_transformers 
+sentence_transformers
 git+https://github.com/huggingface/transformers.git