daniel

.gitattributes

@@ -1,5 +1,36 @@
-*.sqlite3 filter=lfs diff=lfs merge=lfs -text
-*.gitignore filter=lfs diff=lfs merge=lfs -text
-*.yaml filter=lfs diff=lfs merge=lfs -text
-/Dockerfile filter=lfs diff=lfs merge=lfs -text
-*.sqlite3 filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.sqlite3 filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d91b20accd452971bc5df3a5644bfdc2f19a7c6f9e0916c19ac4fc129004ddd0
-size 54

.pre-commit-config.yaml

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:94d98d9ee1e7e5fd27a1d608ddd0ffe1d2ec349df83178d5b12287376637010a
-size 603

Dockerfile

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5f937f23f227ebf14b66fe97c7a715467211833c03a8da8fb2c1f708f0e4421d
-size 358

README.md

@@ -1,12 +1,12 @@
----
-title: Emsland Bot
-emoji: 📈
-colorFrom: red
-colorTo: blue
-sdk: streamlit
-sdk_version: 1.34.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: Emsland Bot
+emoji: 📈
+colorFrom: red
+colorTo: blue
+sdk: streamlit
+sdk_version: 1.34.0
+app_file: app.py
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,121 +0,0 @@
-from haystack.components.embedders import SentenceTransformersTextEmbedder
-from haystack import Pipeline
-from haystack_integrations.components.retrievers.chroma import ChromaEmbeddingRetriever
-from haystack_integrations.document_stores.chroma import ChromaDocumentStore
-from haystack.components.generators import OpenAIGenerator
-from haystack.components.builders import PromptBuilder
-import haystack.logging
-
-
-import streamlit as st
-
-from dotenv import load_dotenv
-from haystack import component
-import logging
-
-haystack.logging.configure_logging(use_json=True)
-
-logging.basicConfig(
-    format="%(levelname)s - %(name)s -  %(message)s", level=logging.WARNING
-)
-logging.getLogger("haystack").setLevel(logging.INFO)
-
-load_dotenv()
-
-
-@component
-class ListToString:
-    @component.output_types(text=str)
-    def run(self, input_list: list[str]):
-        print(input_list[0])
-        return {"text": input_list[0]}
-
-
-@st.cache_resource
-def retrieval_pipeline(path):
-    document_store = ChromaDocumentStore(persist_path=path)
-    retriever = ChromaEmbeddingRetriever(document_store, top_k=5)
-
-    template = """Transform this query into a imaginary response that the
-    user could expect based on your knowledge. Use 1-3 sentences. Replace
-    entities or names that you invent with <axz>. The result should be in
-    German.
-    Query: {{
-    query}}"""
-
-    prompt_builder = PromptBuilder(template=template)
-
-    generator = OpenAIGenerator()
-
-    # Create a pipeline
-    basic_rag_pipeline = Pipeline()
-
-    # Add components to your pipeline
-    basic_rag_pipeline.add_component("prompt_builder", prompt_builder)
-    basic_rag_pipeline.add_component("generator", generator)
-    basic_rag_pipeline.add_component("list_to_string", ListToString())
-    basic_rag_pipeline.add_component("retriever", retriever)
-    basic_rag_pipeline.add_component(
-        "text_embedder",
-        SentenceTransformersTextEmbedder(model="intfloat/multilingual-e5-small"),
-    )
-
-    basic_rag_pipeline.connect("prompt_builder", "generator")
-    basic_rag_pipeline.connect("generator.replies", "list_to_string.input_list")
-    basic_rag_pipeline.connect("list_to_string.text", "text_embedder.text")
-    basic_rag_pipeline.connect("text_embedder.embedding", "retriever.query_embedding")
-
-    return basic_rag_pipeline
-
-
-def generation_pipeline():
-    template = """
-        Given the following information, answer the question.
-
-        Context:
-        {% for document in documents %}
-            {{ document.content }}
-        {% endfor %}
-
-        Bleibe chronologisch. Erkläre Konzepte und Begriffe wenn nötig.
-
-        Question: {{question}}
-        Answer:
-        """
-
-    prompt_builder = PromptBuilder(template=template)
-
-    generator = OpenAIGenerator(model="gpt-4")
-
-    # Create a pipeline
-    basic_rag_pipeline = Pipeline()
-
-    basic_rag_pipeline.add_component("prompt_builder", prompt_builder)
-    basic_rag_pipeline.add_component("llm", generator)
-
-    basic_rag_pipeline.connect("prompt_builder", "llm")
-
-    return basic_rag_pipeline
-
-
-retrieval_pipe = retrieval_pipeline("chatbot/chromadb")
-generation_pipe = generation_pipeline()
-
-prompt = st.chat_input("Say something")
-
-if prompt:
-    response = retrieval_pipe.run({"prompt_builder": {"query": prompt}})
-
-    st.markdown("### Sources")
-    st.write(response["retriever"]["documents"])
-
-    answer = generation_pipe.run(
-        {
-            "prompt_builder": {
-                "question": prompt,
-                "documents": response["retriever"]["documents"],
-            }
-        }
-    )
-    st.markdown("### Answer")
-    st.write(answer["llm"]["replies"][0])

chatbot/build_store.py

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9d6e26a1c8724e58f7db4ca64b08415a406690bfe0703024f86ae2a0686d7661
-size 2571

chatbot/chromadb/6085ec03-95a1-4c3b-99fc-51882acccba3/data_level0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6342cb9fe72281766832989823eb5f623785f82c18981e4501d27d21707e338c
-size 3352000

chatbot/chromadb/6085ec03-95a1-4c3b-99fc-51882acccba3/header.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:165bc99a5045bf6fddf5b0248ae1f7b2a7095b4d2ea27b8a6009770df21cf01c
-size 100

chatbot/chromadb/6085ec03-95a1-4c3b-99fc-51882acccba3/length.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38f4d23fecc5bdec9fd8f19e0df75a7823844054ef28ce0bd949314be707bd5e
-size 8000

chatbot/chromadb/6085ec03-95a1-4c3b-99fc-51882acccba3/link_lists.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2ad572bf16992d01cd6da523f5febd2c458964c611155a1bf5ee2e98246e6f2b
-size 16976

chatbot/chromadb/chroma.sqlite3

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:426e55f129c2ed11a8f30c4a07a4948158e20764cc64c955b911bbb31bc03b39
-size 43278336

processing/wiki/links.txt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1cc172879b60fc68c1b2c910a0075665e2f396cc3234964cee493ca441833655
-size 16707

processing/wiki/scrape_wiki.py

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0044a5c0e51f394175a24cc24ad206c792882b0136221a31760e928984a6a769
-size 1700

processing/wiki/wiki.py

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:22c7316cd5ba296fae6a1cf9f14e0d92737ad85808f8826b4945109d3511b26a
-size 2399

requirements.txt

@@ -1,9 +0,0 @@
-streamlit
-haystack-ai
-bitsandbytes
-accelerate
-pypdf
-cryptography
-pre-commit
-chroma-haystack
-sentence_transformers