Upload 9 files

.gitattributes

@@ -1,35 +1,35 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,160 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

.streamlit/config.toml

@@ -0,0 +1,5 @@
+[theme]
+primaryColor="#afbac2"
+backgroundColor="#3d4850"
+secondaryBackgroundColor="#081310"
+textColor="#f5eff8"

README.md

@@ -1,12 +1,9 @@
----
-title: Test Space
-emoji: 📈
-colorFrom: gray
-colorTo: gray
-sdk: streamlit
-sdk_version: 1.26.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+This repository presents a collection of files to create a question answering website, using a sports dataset called "QASports". To be able to run it on your machine, you need to follow the following steps:
+
+1st open git bash and git clone the repository
+
+2nd, still in the terminal, download the necessary libraries using the command "pip install -r requirements.txt"
+
+3rd, enter the folder where the repository is and execute the command "streamlit run app.py"
+
+By following these steps you will be able to run this website on your machine.

app.py

@@ -1,57 +1,133 @@
-import streamlit as st
-from PIL import Image
-from haystack.document_stores import InMemoryDocumentStore
-from haystack.utils import fetch_archive_from_http
-import os
-from haystack.pipelines.standard_pipelines import TextIndexingPipeline
-from haystack.nodes import BM25Retriever
-from transformers import pipeline
-
-# Criando o objeto documentStore
-document_store = InMemoryDocumentStore(use_bm25=True)
-
-# Exportando os dados necessários sobre o Game Of Thrones
-doc_dir = "data/build_your_first_question_answering_system"
-fetch_archive_from_http(
-    url="https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt1.zip",
-    output_dir=doc_dir,
-)
-
-files_to_index = [doc_dir + "/" + f for f in os.listdir(doc_dir)]
-indexing_pipeline = TextIndexingPipeline(document_store)
-indexing_pipeline.run_batch(file_paths=files_to_index)
-
-retriever = BM25Retriever(document_store=document_store)
-
-# Utilizando um pipeline da biblioteca Transformers
-pipe = pipeline("question-answering", model="deepset/roberta-base-squad2", tokenizer="deepset/roberta-base-squad2")
-
-image = Image.open('comida.jpg')
-st.image(image)
-st.text("""QASports é um sistema de pergunta e respostas, o primeiro grande conjunto de dados
-e respotas a perguntas de vários domínios sobre esportes para perguntas abertas""")
-
-st.subheader('QASports',divider='rainbow')
-
-user_input = None
-if not user_input:
-    user_input = st.text_input("Por favor, digite uma pergunta.")
-
-if user_input:
-    res = retriever.retrieve(user_input, top_k=5)  # Recupera os top 5 documentos relevantes
-    if res:
-        st.write(f"Foram encontrados {len(res)} documentos relevantes.")
-        for document in res:
-            prediction = pipe(question=user_input, context=document.content)
-            context = document.content
-            confidence = prediction["score"]
-            answer = prediction["answer"]
-            st.write("Pergunta:", user_input)
-            st.write("Resposta:", answer)
-            st.write("Confiança:", confidence)
-            st.write("Contexto:", context)
-            st.write("-" * 50)
-
-if st.button('Buscar Resposta'):
-    st.write(prediction["answer"])
-
+import streamlit as st
+from datasets import load_dataset
+from haystack import Pipeline
+from haystack.components.readers import ExtractiveReader
+from haystack.components.retrievers.in_memory import InMemoryBM25Retriever
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+
+from utils import get_unique_docs
+
+
+# Load the dataset
+@st.cache_data(show_spinner=False)
+def load_documents():
+    """
+    Load the documents from the dataset considering only unique documents.
+    Returns:
+    - documents: list of dictionaries with the documents.
+    """
+    unique_docs = set()
+    dataset_name = "PedroCJardim/QASports"
+    dataset_split = "basketball"
+    st.caption(f'Fetching "{dataset_name}" dataset')
+    # build the dataset
+    dataset = load_dataset(dataset_name, dataset_split)
+    docs_validation = get_unique_docs(dataset["validation"], unique_docs)
+    docs_train = get_unique_docs(dataset["train"], unique_docs)
+    docs_test = get_unique_docs(dataset["test"], unique_docs)
+    documents = docs_validation + docs_train + docs_test
+    return documents
+
+
+@st.cache_resource(show_spinner=False)
+def get_document_store(documents):
+    """
+    Index the files in the document store.
+    Args:
+    - files: list of dictionaries with the documents.
+    """
+    # Create in memory database
+    st.caption(f"Building the Document Store")
+    document_store = InMemoryDocumentStore()
+    document_store.write_documents(documents=documents)
+    return document_store
+
+
+@st.cache_resource(show_spinner=False)
+def get_question_pipeline(_doc_store):
+    """
+    Create the pipeline with the retriever and reader components.
+    Args:
+    - doc_store: instance of the document store.
+    Returns:
+    - pipe: instance of the pipeline.
+    """
+    st.caption(f"Building the Question Answering pipeline")
+    # Create the retriever and reader
+    retriever = InMemoryBM25Retriever(document_store=_doc_store)
+    reader = ExtractiveReader(model="deepset/roberta-base-squad2")
+    reader.warm_up()
+    # Create the pipeline
+    pipe = Pipeline()
+    pipe.add_component(instance=retriever, name="retriever")
+    pipe.add_component(instance=reader, name="reader")
+    pipe.connect("retriever.documents", "reader.documents")
+    return pipe
+
+
+def search(pipeline, question: str):
+    """
+    Search for the answer to a question in the documents.
+    Args:
+    - pipeline: instance of the pipeline.
+    - question: string with the question.
+    Returns:
+    - answer: dictionary with the answer.
+    """
+    # Get the answers
+    top_k = 3
+    answer = pipeline.run(
+        data={
+            "retriever": {"query": question, "top_k": 10},
+            "reader": {"query": question, "top_k": top_k},
+        }
+    )
+    max_k = min(top_k, len(answer["reader"]["answers"]))
+    return answer["reader"]["answers"][0:max_k]
+
+
+# Streamlit interface
+_, centering_column, _ = st.columns(3)
+with centering_column:
+    st.image("assets/qasports-logo.png", use_column_width=True)
+
+# Loading status
+with st.status(
+    "Downloading dataset...", expanded=st.session_state.get("expanded", True)
+) as status:
+    documents = load_documents()
+    status.update(label="Indexing documents...")
+    doc_store = get_document_store(documents)
+    status.update(label="Creating pipeline...")
+    pipe = get_question_pipeline(doc_store)
+    status.update(
+        label="Download and indexing complete!", state="complete", expanded=False
+    )
+    st.session_state["expanded"] = False
+
+st.subheader("🔎 Basketball", divider="rainbow")
+st.caption(
+    """This website presents a collection of documents from the dataset named "QASports", the first large sports question answering dataset for open questions. QASports contains real data of players, teams and matches from the sports soccer, basketball and American football. It counts over 1.5 million questions and answers about 54k preprocessed, cleaned and organized documents from Wikipedia-like sources."""
+)
+
+if user_query := st.text_input(
+    label="Ask a question about Basketball! 🏀",
+    placeholder="How many field goals did Kobe Bryant score?",
+):
+    # Get the answers
+    with st.spinner("Waiting"):
+        try:
+            answer = search(pipe, user_query)
+            for idx, ans in enumerate(answer):
+                st.info(
+                    f"""
+                    Answer {idx+1}: "{ans.data}" | Score: {ans.score:0.4f}  
+                    Document: "{ans.document.meta["title"]}"  
+                    URL: {ans.document.meta["url"]}
+                """
+                )
+                with st.expander("See details", expanded=False):
+                    st.write(ans)
+                st.divider()
+        except Exception as e:
+            st.error("We do not have an answer for your question")

requirements.txt

@@ -1,3 +1,11 @@
-torch
-transformers
-farm-haystack==1.6.0
+# main
+# streamlit # commented to avoid miss instalation
+
+# HuggingFace
+datasets==2.18.0
+# Question Answering
+haystack-ai==2.0.1
+accelerate==0.29.2
+sentence-transformers==2.7.0
+# Extra
+mmh3==4.1.0

utils.py

@@ -0,0 +1,28 @@
+"""This module contains utility functions for the project"""
+
+import mmh3
+from haystack import Document
+
+
+def get_unique_docs(dataset, unique_docs: set):
+    """Get unique documents from dataset
+    Args:
+    dataset: list of dictionaries
+    Returns:
+    docs: list of haystack.Document
+    """
+    docs = list()
+    for doc in dataset:
+        if doc["context"] is not None and doc["context_id"] not in unique_docs:
+            unique_docs.add(doc["context_id"])
+            document = Document(
+                content=doc["context"],
+                meta={
+                    "title": doc["context_title"],
+                    "context_id": doc["context_id"],
+                    "url": doc["url"],
+                    "source": "QASports",
+                },
+            )
+            docs.append(document)
+    return docs