Initial commit

.github/workflows/format.yaml

@@ -0,0 +1,24 @@
+name: Check code formatting with Black
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up uv
+        # Install a specific uv version using the installer
+        run: curl -LsSf https://astral.sh/uv/0.3.3/install.sh | sh
+      - name: "Set up Python"
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: "pyproject.toml"
+      - name: Install the project
+        run: |
+          uv sync --all-extras --dev
+      - name: Check formatting with Black
+        run: |
+          uv run black --check .
+        continue-on-error: true

.github/workflows/lint.yaml

@@ -0,0 +1,24 @@
+name: Lint code with Ruff
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up uv
+        # Install a specific uv version using the installer
+        run: curl -LsSf https://astral.sh/uv/0.3.3/install.sh | sh
+      - name: "Set up Python"
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: "pyproject.toml"
+      - name: Install the project
+        run: |
+          uv sync --all-extras --dev
+      - name: Lint with Ruff
+        run: |
+          uv run ruff check --output-format=github .
+        continue-on-error: true

.gitignore

@@ -0,0 +1,171 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Manually added
+draft_notebooks/
+.env
+outputs
+module_html
+module_catalogue_html
+module_md
+module_catalogue_md

.pre-commit-config.yaml

@@ -0,0 +1,12 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 24.8.0
+    hooks:
+      - id: black
+        language_version: python3
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.6.2
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]

LICENSE

@@ -0,0 +1,193 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1.  Definitions.
+
+    "License" shall mean the terms and conditions for use, reproduction,
+    and distribution as defined by Sections 1 through 9 of this document.
+
+    "Licensor" shall mean the copyright owner or entity authorized by
+    the copyright owner that is granting the License.
+
+    "Legal Entity" shall mean the union of the acting entity and all
+    other entities that control, are controlled by, or are under common
+    control with that entity. For the purposes of this definition,
+    "control" means (i) the power, direct or indirect, to cause the
+    direction or management of such entity, whether by contract or
+    otherwise, or (ii) ownership of fifty percent (50%) or more of the
+    outstanding shares, or (iii) beneficial ownership of such entity.
+
+    "You" (or "Your") shall mean an individual or Legal Entity
+    exercising permissions granted by this License.
+
+    "Source" form shall mean the preferred form for making modifications,
+    including but not limited to software source code, documentation
+    source, and configuration files.
+
+    "Object" form shall mean any form resulting from mechanical
+    transformation or translation of a Source form, including but
+    not limited to compiled object code, generated documentation,
+    and conversions to other media types.
+
+    "Work" shall mean the work of authorship, whether in Source or
+    Object form, made available under the License, as indicated by a
+    copyright notice that is included in or attached to the work
+    (an example is provided in the Appendix below).
+
+    "Derivative Works" shall mean any work, whether in Source or Object
+    form, that is based on (or derived from) the Work and for which the
+    editorial revisions, annotations, elaborations, or other modifications
+    represent, as a whole, an original work of authorship. For the purposes
+    of this License, Derivative Works shall not include works that remain
+    separable from, or merely link (or bind by name) to the interfaces of,
+    the Work and Derivative Works thereof.
+
+    "Contribution" shall mean any work of authorship, including
+    the original version of the Work and any modifications or additions
+    to that Work or Derivative Works thereof, that is intentionally
+    submitted to Licensor for inclusion in the Work by the copyright owner
+    or by an individual or Legal Entity authorized to submit on behalf of
+    the copyright owner. For the purposes of this definition, "submitted"
+    means any form of electronic, verbal, or written communication sent
+    to the Licensor or its representatives, including but not limited to
+    communication on electronic mailing lists, source code control systems,
+    and issue tracking systems that are managed by, or on behalf of, the
+    Licensor for the purpose of discussing and improving the Work, but
+    excluding communication that is conspicuously marked or otherwise
+    designated in writing by the copyright owner as "Not a Contribution."
+
+    "Contributor" shall mean Licensor and any individual or Legal Entity
+    on behalf of whom a Contribution has been received by Licensor and
+    subsequently incorporated within the Work.
+
+2.  Grant of Copyright License. Subject to the terms and conditions of
+    this License, each Contributor hereby grants to You a perpetual,
+    worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+    copyright license to reproduce, prepare Derivative Works of,
+    publicly display, publicly perform, sublicense, and distribute the
+    Work and such Derivative Works in Source or Object form.
+
+3.  Grant of Patent License. Subject to the terms and conditions of
+    this License, each Contributor hereby grants to You a perpetual,
+    worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+    (except as stated in this section) patent license to make, have made,
+    use, offer to sell, sell, import, and otherwise transfer the Work,
+    where such license applies only to those patent claims licensable
+    by such Contributor that are necessarily infringed by their
+    Contribution(s) alone or by combination of their Contribution(s)
+    with the Work to which such Contribution(s) was submitted. If You
+    institute patent litigation against any entity (including a
+    cross-claim or counterclaim in a lawsuit) alleging that the Work
+    or a Contribution incorporated within the Work constitutes direct
+    or contributory patent infringement, then any patent licenses
+    granted to You under this License for that Work shall terminate
+    as of the date such litigation is filed.
+
+4.  Redistribution. You may reproduce and distribute copies of the
+    Work or Derivative Works thereof in any medium, with or without
+    modifications, and in Source or Object form, provided that You
+    meet the following conditions:
+
+    (a) You must give any other recipients of the Work or
+    Derivative Works a copy of this License; and
+
+    (b) You must cause any modified files to carry prominent notices
+    stating that You changed the files; and
+
+    (c) You must retain, in the Source form of any Derivative Works
+    that You distribute, all copyright, patent, trademark, and
+    attribution notices from the Source form of the Work,
+    excluding those notices that do not pertain to any part of
+    the Derivative Works; and
+
+    (d) If the Work includes a "NOTICE" text file as part of its
+    distribution, then any Derivative Works that You distribute must
+    include a readable copy of the attribution notices contained
+    within such NOTICE file, excluding those notices that do not
+    pertain to any part of the Derivative Works, in at least one
+    of the following places: within a NOTICE text file distributed
+    as part of the Derivative Works; within the Source form or
+    documentation, if provided along with the Derivative Works; or,
+    within a display generated by the Derivative Works, if and
+    wherever such third-party notices normally appear. The contents
+    of the NOTICE file are for informational purposes only and
+    do not modify the License. You may add Your own attribution
+    notices within Derivative Works that You distribute, alongside
+    or as an addendum to the NOTICE text from the Work, provided
+    that such additional attribution notices cannot be construed
+    as modifying the License.
+
+    You may add Your own copyright statement to Your modifications and
+    may provide additional or different license terms and conditions
+    for use, reproduction, or distribution of Your modifications, or
+    for any such Derivative Works as a whole, provided Your use,
+    reproduction, and distribution of the Work otherwise complies with
+    the conditions stated in this License.
+
+5.  Submission of Contributions. Unless You explicitly state otherwise,
+    any Contribution intentionally submitted for inclusion in the Work
+    by You to the Licensor shall be under the terms and conditions of
+    this License, without any additional terms or conditions.
+    Notwithstanding the above, nothing herein shall supersede or modify
+    the terms of any separate license agreement you may have executed
+    with Licensor regarding such Contributions.
+
+6.  Trademarks. This License does not grant permission to use the trade
+    names, trademarks, service marks, or product names of the Licensor,
+    except as required for reasonable and customary use in describing the
+    origin of the Work and reproducing the content of the NOTICE file.
+
+7.  Disclaimer of Warranty. Unless required by applicable law or
+    agreed to in writing, Licensor provides the Work (and each
+    Contributor provides its Contributions) on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+    implied, including, without limitation, any warranties or conditions
+    of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+    PARTICULAR PURPOSE. You are solely responsible for determining the
+    appropriateness of using or redistributing the Work and assume any
+    risks associated with Your exercise of permissions under this License.
+
+8.  Limitation of Liability. In no event and under no legal theory,
+    whether in tort (including negligence), contract, or otherwise,
+    unless required by applicable law (such as deliberate and grossly
+    negligent acts) or agreed to in writing, shall any Contributor be
+    liable to You for damages, including any direct, indirect, special,
+    incidental, or consequential damages of any character arising as a
+    result of this License or out of the use or inability to use the
+    Work (including but not limited to damages for loss of goodwill,
+    work stoppage, computer failure or malfunction, or any and all
+    other commercial damages or losses), even if such Contributor
+    has been advised of the possibility of such damages.
+
+9.  Accepting Warranty or Additional Liability. While redistributing
+    the Work or Derivative Works thereof, You may choose to offer,
+    and charge a fee for, acceptance of support, warranty, indemnity,
+    or other liability obligations and/or rights consistent with this
+    License. However, in accepting such obligations, You may act only
+    on Your own behalf and on Your sole responsibility, not on behalf
+    of any other Contributor, and only if You agree to indemnify,
+    defend, and hold each Contributor harmless for any liability
+    incurred by, or claims asserted against, such Contributor by reason
+    of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+---
+
+Copyright 2024 Joseph Farrington
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+

README.md

@@ -0,0 +1,94 @@
+# Chat with the 2024/2025 UCL module catalogue
+
+## NOTE
+This is a demonstration developed for educational purposes only and is not affiliated with or endorsed by University College London (UCL). The model may provide incorrect or outdated information. Interactions should therefore not be used to inform decisions such as programme choices or module selection.
+
+Please refer to the official [UCL module catalogue](https://www.ucl.ac.uk/module-catalogue) for accurate and up-to-date information.
+
+The code is licensed under Apache License 2.0 but the module catalogue content is copyright UCL. 
+
+## Get started
+
+### Hugging Face Space
+
+The easiest way to chat with the model is using the Hugging Face space.
+
+### Local use
+
+You can use the code snippet below to run the app locally. This project uses [uv](https://docs.astral.sh/uv/) to manage dependencies and the snippet assumes that you have [uv installed](https://docs.astral.sh/uv/getting-started/installation/). 
+
+The app requires an [OpenAI API key](https://help.openai.com/en/articles/4936850-where-do-i-find-my-openai-api-key) to run locally.
+
+```bash
+# Clone repo and install dependencies in venv
+git clone https://github.com/joefarrington/ucl_module_chat.git
+cd ucl_module_chat
+
+uv venv
+source .venv/bin/activate
+uv pip install .
+
+# Add API keys as environment variables
+# Alternatively, create a .env file in the main directory
+export OPENAI_API_KEY=<Your API key>
+
+# Run the app
+python app.py
+```
+One advantage of LangChain is that you could easily substitute the embedding model and/or LLM for alternatives including locally hosted models using [Hugging Face](https://python.langchain.com/docs/integrations/providers/huggingface/), [llama.cpp](https://python.langchain.com/docs/integrations/providers/llamacpp/) or [Ollama](https://python.langchain.com/docs/integrations/providers/ollama/). 
+
+### Rerun scraping and embedding
+
+The repository includes the vectorstore with pages from the module catalogue embedded using OpenAI's [text-embedding-3-small](https://platform.openai.com/docs/guides/embeddings).
+
+The process for downloading the pages from the module catalogue, converting the pages to markdown documents, and embedding the documents can be re-run using the script `setup.py`. There is no need to run this script unless you want to change the way data is extracted from the HTML pages to markdown, or embed the documents using an alternative model. 
+
+## Implementation details
+
+### Document scraping
+
+The documents in the vectorstore used to provide context to the LLM are based on the publicly available webpages describing each module offered by UCL. 
+
+The URLs for the individual module catalogue pages are identified from the module catalogue search page. The modules pages are then visited in sequence and the HTML is downloaded for each page. 
+
+There are more efficient ways to scrape the content from the module catalogue (e.g. [scrapy](https://scrapy.org/)). The current method is designed to minimise the effect on the server. There is long wait time between requests and the raw HTML is saved so that alternative methods of extracting the content can be considered without needing to request additional data from the server.
+
+### Document conversion
+
+The raw HTML for each module page is converted to a markdown document using [Beautiful Soup](https://www.crummy.com/software/BeautifulSoup/bs4/doc/) to parse the HTML and a [Jinja](https://jinja.palletsprojects.com/en/stable/intro/) template to format the extracted information.  
+
+### Document embedding
+
+The module pages are relatively short documents and therefore each is treated as a single chunk and embedded as a whole. 
+
+Each page is embedded using [text-embedding-3-small](https://platform.openai.com/docs/guides/embeddings). [FAISS](https://faiss.ai/) is used to store and search the embedded documents. 
+
+### Q&A based using RAG
+
+The chat interface is a simple [Gradio](https://www.gradio.app/) app, and uses OpenAI's [gpt-4o-mini](https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence/) as the underlying LLM. 
+
+At each turn of the conversation the following steps are performed, managed using [Langchain](https://python.langchain.com/docs/introduction/):
+
+* Call the LLM to rephrase the user's query, given the conversation history, so that it includes relevant context from the conversation.
+
+* Embed the rephrased query and retrieve relevant documents from the vectorstore.
+
+* Call the LLM with the current user input, retrieved documents for context and conversation history.  Output the result as the LLM's response in the chat interface.
+
+## Potential extensions
+
+* Add [course descriptions](https://www.ucl.ac.uk/prospective-students/undergraduate/undergraduate-courses) to the vectorstore so that the app is more useful to potential applicants and can explain, for example, which modules are mandatory on certain courses.
+
+* Provide links to the module catalogue for modules suggested by the application, either within the conversation or as a separate interface element.
+
+* Use a agent-based approach to avoid unnecessary retrieval steps and/or support more complex queries that require multiple retrieval steps. 
+
+* Use a LangGraph app to manage the conversation history and state. 
+
+## Useful resources
+
+* [UCL module catalogue](https://www.ucl.ac.uk/module-catalogue?collection=drupal-module-catalogue&facetsort=alpha&num_ranks=20&daat=10000&sort=title)
+
+* [Langchain official tutorials](https://python.langchain.com/docs/tutorials/)
+
+* Hands-On Large Language Models [book](https://learning.oreilly.com/library/view/hands-on-large-language/9781098150952/) and [GitHub repository](https://github.com/HandsOnLLM/Hands-On-Large-Language-Models)

app.py

@@ -0,0 +1,74 @@
+import gradio as gr
+import hydra
+import omegaconf
+from dotenv import load_dotenv
+from langchain_core.messages import AIMessage, BaseMessage, HumanMessage
+
+from ucl_module_chat.chains.rag_chain import build_rag_chain
+from ucl_module_chat.utils.resolve_paths import get_abs_path_using_repo_root
+
+load_dotenv()
+
+# Text paragraph to be added below the title
+description = """
+<b>NOTE</b>: This is a demonstration developed for educational purposes only 
+and is not affiliated with or endorsed by University College London (UCL).
+The model may provide incorrect or outdated information.  Interactions should 
+therefore not be used to inform decisions such as programme choices or module selection.
+
+Please refer to the official [UCL module catalogue](https://www.ucl.ac.uk/module-catalogue)
+for accurate and up-to-date information.
+"""
+
+examples = [
+    "When can I take a module on medical statistics?",
+    "What are the prerequisites for taking Supervised Learning?",
+    "What is the difference between the two modules on Trauma for \
+        paediatric dentistry?",
+]
+
+
+def convert_history(history: list[dict]) -> list[BaseMessage]:
+    """Convert conversation history into Langchain messages"""
+    lc_history = []
+    for msg in history:
+        if msg["role"] == "user":
+            lc_history.append(HumanMessage(msg["content"]))
+        elif msg["role"] == "assistant":
+            lc_history.append(AIMessage(msg["content"]))
+    return lc_history
+
+
+@hydra.main(
+    version_base=None, config_path="src/ucl_module_chat/conf", config_name="config"
+)
+def main(cfg: omegaconf.DictConfig) -> None:
+    """Run the UCL module chatbot in a Gradio interface."""
+
+    vectorstore_dir = get_abs_path_using_repo_root(cfg.vectorstore.dir)
+    llm = hydra.utils.instantiate(cfg.models.llm)
+    embedding_model = hydra.utils.instantiate(cfg.models.embedding)
+    rag_chain = build_rag_chain(
+        llm=llm, embedding_model=embedding_model, vectorstore_dir=vectorstore_dir
+    )
+
+    def chat(input: str, history: list[dict] = None) -> str:
+        result = rag_chain.invoke(
+            {"input": input, "chat_history": convert_history(history)},
+        )
+        return result["answer"]
+
+    with gr.Blocks(fill_height=True) as module_chat:
+        gr.Markdown("# Chat with the module catalogue")
+        gr.Markdown(description)
+        gr.ChatInterface(
+            fn=chat,
+            type="messages",
+            examples=examples,
+        )
+
+    module_chat.launch()
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -0,0 +1,51 @@
+[project]
+name = "ucl-module-chat"
+version = "0.1.0"
+description = "Unofficial chat app for the UCL module catalogue"
+readme = "README.md"
+requires-python = ">=3.9"
+dependencies = [
+    "beautifulsoup4>=4.12.3",
+    "langchain>=0.3.3",
+    "tqdm>=4.66.5",
+    "langchain-community>=0.3.2",
+    "faiss-cpu>=1.9.0",
+    "langchain-openai>=0.2.3",
+    "gradio>=4.44.1",
+    "jinja2>=3.1.4",
+    "loguru>=0.7.2",
+    "hydra-core>=1.3.2",
+    "gitpython>=3.1.43",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/ucl_module_chat"]
+
+[tool.uv]
+dev-dependencies = [
+    "jupyter>=1.1.1",
+    "black>=24.8.0",
+    "ruff>=0.6.9",
+    "pre-commit>=4.0.1",
+]
+
+[tool.black]
+line-length = 88
+
+[tool.ruff]
+line-length = 88
+fix = true
+exclude = ["src/ucl_module_chat/data_processing/document_templates.py"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I"]
+fixable = ["ALL"]
+unfixable = []
+
+[tool.ruff.lint.isort]
+force-single-line = false
+combine-as-imports = true

requirements.txt

@@ -0,0 +1,285 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile pyproject.toml -o requirements.txt
+aiofiles==23.2.1
+    # via gradio
+aiohappyeyeballs==2.4.3
+    # via aiohttp
+aiohttp==3.10.10
+    # via
+    #   langchain
+    #   langchain-community
+aiosignal==1.3.1
+    # via aiohttp
+annotated-types==0.7.0
+    # via pydantic
+antlr4-python3-runtime==4.9.3
+    # via
+    #   hydra-core
+    #   omegaconf
+anyio==4.6.2.post1
+    # via
+    #   gradio
+    #   httpx
+    #   openai
+    #   starlette
+attrs==24.2.0
+    # via aiohttp
+beautifulsoup4==4.12.3
+    # via ucl-module-chat (pyproject.toml)
+certifi==2024.8.30
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
+charset-normalizer==3.4.0
+    # via requests
+click==8.1.7
+    # via
+    #   typer
+    #   uvicorn
+dataclasses-json==0.6.7
+    # via langchain-community
+distro==1.9.0
+    # via openai
+faiss-cpu==1.9.0
+    # via ucl-module-chat (pyproject.toml)
+fastapi==0.115.3
+    # via gradio
+ffmpy==0.4.0
+    # via gradio
+filelock==3.16.1
+    # via huggingface-hub
+frozenlist==1.5.0
+    # via
+    #   aiohttp
+    #   aiosignal
+fsspec==2024.10.0
+    # via
+    #   gradio-client
+    #   huggingface-hub
+gitdb==4.0.11
+    # via gitpython
+gitpython==3.1.43
+    # via ucl-module-chat (pyproject.toml)
+gradio==5.3.0
+    # via ucl-module-chat (pyproject.toml)
+gradio-client==1.4.2
+    # via gradio
+greenlet==3.1.1
+    # via sqlalchemy
+h11==0.14.0
+    # via
+    #   httpcore
+    #   uvicorn
+httpcore==1.0.6
+    # via httpx
+httpx==0.27.2
+    # via
+    #   gradio
+    #   gradio-client
+    #   langsmith
+    #   openai
+huggingface-hub==0.26.1
+    # via
+    #   gradio
+    #   gradio-client
+hydra-core==1.3.2
+    # via ucl-module-chat (pyproject.toml)
+idna==3.10
+    # via
+    #   anyio
+    #   httpx
+    #   requests
+    #   yarl
+jinja2==3.1.4
+    # via
+    #   ucl-module-chat (pyproject.toml)
+    #   gradio
+jiter==0.6.1
+    # via openai
+jsonpatch==1.33
+    # via langchain-core
+jsonpointer==3.0.0
+    # via jsonpatch
+langchain==0.3.4
+    # via
+    #   ucl-module-chat (pyproject.toml)
+    #   langchain-community
+langchain-community==0.3.3
+    # via ucl-module-chat (pyproject.toml)
+langchain-core==0.3.12
+    # via
+    #   langchain
+    #   langchain-community
+    #   langchain-openai
+    #   langchain-text-splitters
+langchain-openai==0.2.3
+    # via ucl-module-chat (pyproject.toml)
+langchain-text-splitters==0.3.0
+    # via langchain
+langsmith==0.1.137
+    # via
+    #   langchain
+    #   langchain-community
+    #   langchain-core
+loguru==0.7.2
+    # via ucl-module-chat (pyproject.toml)
+markdown-it-py==3.0.0
+    # via rich
+markupsafe==2.1.5
+    # via
+    #   gradio
+    #   jinja2
+marshmallow==3.23.0
+    # via dataclasses-json
+mdurl==0.1.2
+    # via markdown-it-py
+multidict==6.1.0
+    # via
+    #   aiohttp
+    #   yarl
+mypy-extensions==1.0.0
+    # via typing-inspect
+numpy==1.26.4
+    # via
+    #   faiss-cpu
+    #   gradio
+    #   langchain
+    #   langchain-community
+    #   pandas
+omegaconf==2.3.0
+    # via hydra-core
+openai==1.52.2
+    # via langchain-openai
+orjson==3.10.10
+    # via
+    #   gradio
+    #   langsmith
+packaging==24.1
+    # via
+    #   faiss-cpu
+    #   gradio
+    #   gradio-client
+    #   huggingface-hub
+    #   hydra-core
+    #   langchain-core
+    #   marshmallow
+pandas==2.2.3
+    # via gradio
+pillow==10.4.0
+    # via gradio
+propcache==0.2.0
+    # via yarl
+pydantic==2.9.2
+    # via
+    #   fastapi
+    #   gradio
+    #   langchain
+    #   langchain-core
+    #   langsmith
+    #   openai
+    #   pydantic-settings
+pydantic-core==2.23.4
+    # via pydantic
+pydantic-settings==2.6.0
+    # via langchain-community
+pydub==0.25.1
+    # via gradio
+pygments==2.18.0
+    # via rich
+python-dateutil==2.9.0.post0
+    # via pandas
+python-dotenv==1.0.1
+    # via pydantic-settings
+python-multipart==0.0.12
+    # via gradio
+pytz==2024.2
+    # via pandas
+pyyaml==6.0.2
+    # via
+    #   gradio
+    #   huggingface-hub
+    #   langchain
+    #   langchain-community
+    #   langchain-core
+    #   omegaconf
+regex==2024.9.11
+    # via tiktoken
+requests==2.32.3
+    # via
+    #   huggingface-hub
+    #   langchain
+    #   langchain-community
+    #   langsmith
+    #   requests-toolbelt
+    #   tiktoken
+requests-toolbelt==1.0.0
+    # via langsmith
+rich==13.9.3
+    # via typer
+ruff==0.7.1
+    # via gradio
+semantic-version==2.10.0
+    # via gradio
+shellingham==1.5.4
+    # via typer
+six==1.16.0
+    # via python-dateutil
+smmap==5.0.1
+    # via gitdb
+sniffio==1.3.1
+    # via
+    #   anyio
+    #   httpx
+    #   openai
+soupsieve==2.6
+    # via beautifulsoup4
+sqlalchemy==2.0.36
+    # via
+    #   langchain
+    #   langchain-community
+starlette==0.41.0
+    # via
+    #   fastapi
+    #   gradio
+tenacity==9.0.0
+    # via
+    #   langchain
+    #   langchain-community
+    #   langchain-core
+tiktoken==0.8.0
+    # via langchain-openai
+tomlkit==0.12.0
+    # via gradio
+tqdm==4.66.5
+    # via
+    #   ucl-module-chat (pyproject.toml)
+    #   huggingface-hub
+    #   openai
+typer==0.12.5
+    # via gradio
+typing-extensions==4.12.2
+    # via
+    #   fastapi
+    #   gradio
+    #   gradio-client
+    #   huggingface-hub
+    #   langchain-core
+    #   openai
+    #   pydantic
+    #   pydantic-core
+    #   sqlalchemy
+    #   typer
+    #   typing-inspect
+typing-inspect==0.9.0
+    # via dataclasses-json
+tzdata==2024.2
+    # via pandas
+urllib3==2.2.3
+    # via requests
+uvicorn==0.32.0
+    # via gradio
+websockets==12.0
+    # via gradio-client
+yarl==1.16.0
+    # via aiohttp

src/ucl_module_chat/chains/rag_chain.py

@@ -0,0 +1,84 @@
+from pathlib import Path
+
+from dotenv import load_dotenv
+from langchain.chains import create_history_aware_retriever, create_retrieval_chain
+from langchain.chains.combine_documents import create_stuff_documents_chain
+from langchain_community.vectorstores import FAISS
+from langchain_core.embeddings.embeddings import Embeddings
+from langchain_core.language_models import BaseChatModel
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+
+load_dotenv()
+
+context_prompt = """Given a chat history and the latest user question
+        which might reference context in the chat history,
+        formulate a standalone question which can be understood
+        without the chat history. Do NOT answer the question,
+        just reformulate it if needed and otherwise return it as is."""
+
+rag_prompt = """You are an assistant for question-answering tasks related
+        to university courses (modules) at Univeristy College London (UCL).
+        Use the following pieces of retrieved context to answer the question.
+        The context is from entires in the UCL module catalogue,
+        which is available publicly on the internet. 
+        The first time you refer to a module in the conversation, refer to
+        it by it's full name followed by the module code in brackets, e.g.
+        Supervised Learning (COMP0078).
+        If you don't know the answer, say that you don't know.
+        Use five sentences maximum and keep the answer concise. 
+        Use professional British English and avoid using slang.
+        Do not refer to the context directly in your answer, but you
+        should use it to answer the question. 
+        You can ask the user if they would like to know more about a 
+        specific area if you think it may be helpful.
+        \n\n
+        {context}"""
+
+
+def build_rag_chain(
+    llm: BaseChatModel,
+    embedding_model: Embeddings,
+    vectorstore_dir: str | Path,
+    context_system_prompt: str = context_prompt,
+    rag_system_prompt: str = rag_prompt,
+):
+    """Build a RAG chain for the UCL module chatbot."""
+
+    contextualize_q_prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", context_system_prompt),
+            MessagesPlaceholder("chat_history"),
+            ("human", "{input}"),
+        ]
+    )
+    vectorstore = FAISS.load_local(
+        vectorstore_dir,
+        embeddings=embedding_model,
+        allow_dangerous_deserialization=True,
+    )
+    retriever = vectorstore.as_retriever()
+
+    history_aware_retriever = create_history_aware_retriever(
+        llm, retriever, contextualize_q_prompt
+    )
+
+    qa_prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", rag_system_prompt),
+            MessagesPlaceholder("chat_history"),
+            ("human", "{input}"),
+        ]
+    )
+
+    # Stuff documents chain combines documents from retreiver | qa_prompt
+    # | llm | StrOutputParser
+    question_answer_chain = create_stuff_documents_chain(llm, qa_prompt).with_config(
+        tags=["qa"]
+    )
+
+    # Full rag chain is history aware retriever | question answer chain
+    rag_chain = create_retrieval_chain(
+        history_aware_retriever, question_answer_chain
+    ).with_config(tags=["rag"])
+
+    return rag_chain

src/ucl_module_chat/conf/config.yaml

@@ -0,0 +1,29 @@
+defaults:
+  - override hydra/job_logging: disabled
+
+# All paths are specified relative to the root of the project
+
+setup:
+  scrape_documents:
+    index_page_url: "https://search.ucl.ac.uk/s/search.html?collection=drupal-module-catalogue&facetsort=alpha&num_ranks=10000&daat=10000&form=ucl&start_rank=0"
+    output_dir: 'data/module_html'
+    regex_url_pattern: 'https://www.ucl.ac.uk/module-catalogue/modules/[a-zA-Z0-9-]+[A-Z]{4}\d{4}'
+    wait_time_seconds: 2
+  convert_documents:
+    input_dir: ${setup.scrape_documents.output_dir}
+    output_dir: 'data/module_md'
+  embed_documents:
+    input_dir: ${setup.convert_documents.output_dir}
+    output_dir: ${vectorstore.dir}
+
+vectorstore:
+  dir: 'data/module_catalogue_vectorstore'
+  
+models:
+  embedding:
+    _target_: langchain_openai.OpenAIEmbeddings
+    model: text-embedding-3-small
+  
+  llm:
+    _target_: langchain_openai.ChatOpenAI
+    model: gpt-4o-mini

src/ucl_module_chat/data_processing/document_conversion.py

@@ -0,0 +1,227 @@
+import re
+from pathlib import Path
+
+import hydra
+import jinja2
+import omegaconf
+from bs4 import BeautifulSoup
+from loguru import logger
+from tqdm import tqdm
+
+from ucl_module_chat.data_processing.document_templates import module_template
+from ucl_module_chat.utils.resolve_paths import get_abs_path_using_repo_root
+
+
+def _extract_module_info_from_html(module_html: str) -> dict:
+    """Parse HTML content for a UCL module page and extract key information."""
+
+    soup = BeautifulSoup(module_html, "html.parser")
+
+    # Extract the module title and code from the og:title meta tag
+    pattern = r"""
+    (?P<module_title>.*?)                # Capture the module title
+    \s*                                  # Optional whitespace
+    \((?P<module_code>[A-Z]{4}\d{4})\)   # Capture the alphanumeric code
+    """
+    og_title = soup.find("meta", attrs={"name": "og:title"})["content"]
+    match = re.search(pattern, og_title, re.VERBOSE)
+    module_title = match.group("module_title").strip()
+    module_code = match.group("module_code").strip()
+
+    url = soup.find("meta", attrs={"property": "og:url"})["content"]
+
+    faculty = soup.find("meta", attrs={"name": "ucl:sanitized_faculty"})["content"]
+
+    teaching_department = soup.find(
+        "meta", attrs={"name": "ucl:sanitized_teaching_department"}
+    )["content"]
+
+    level = soup.find("meta", attrs={"name": "ucl:sanitized_level"})["content"]
+
+    teaching_term = soup.find(
+        "meta", attrs={"name": "ucl:sanitized_intended_teaching_term"}
+    )["content"]
+
+    credit_value = soup.find("meta", attrs={"name": "ucl:sanitized_credit_value"})[
+        "content"
+    ]
+
+    sanitized_subject = soup.find("meta", attrs={"name": "ucl:sanitized_subject"})[
+        "content"
+    ]
+
+    sanitized_keywords = soup.find("meta", attrs={"name": "ucl:sanitized_keywords"})[
+        "content"
+    ]
+
+    restrictions = (
+        soup.find("dt", string="Restrictions")
+        .find_next_sibling("dd")
+        .get_text()
+        .strip()
+        .replace("\n", " ")
+    )
+
+    alternative_credit_options = (
+        soup.find("h2", string="Alternative credit options")
+        .find_next("p")
+        .get_text()
+        .strip()
+    )
+
+    description = soup.find("div", class_="module-description").get_text()
+
+    # Deliveries - there may be multiple deliveries for a module
+    potential_deliveries = soup.find_all("div", class_="box tagged box--bar-thick")
+    deliveries = [
+        d
+        for d in potential_deliveries
+        if d.find("h3", string="Teaching and assessment") is not None
+    ]
+    collated_d = []
+    for d in deliveries:
+        delivery_info = {}
+
+        # Info from the header
+        header = d.find("h2").get_text()
+        # Might need to modify this regex pattern if some modules are different
+        pattern = r"""
+            Intended\steaching\sterm:               # Matches 'Intended teaching term:'
+            \s*                                     # Optional whitespace
+            (?P<term>[\w\s,\(\)]+)                  # Capture the term
+            \s*                                     # Optional whitespace
+            (?P<type>Undergraduate|Postgraduate)    # Matches UG or PG
+            \s*                                     # Optional whitespace
+            \(FHEQ\sLevel\s(?P<fheq_level>\d+)\)    # Matches 'FHEQ Level X' 
+            """  # and captures level number
+
+        # Search for matches in the header string
+        match = re.search(pattern, header, re.VERBOSE)
+
+        if match:
+            # Extracted values from the regex groups
+            delivery_info["teaching_term"] = match.group("term").strip()
+            delivery_info["type"] = match.group("type").strip()
+            delivery_info["fheq_level"] = match.group("fheq_level").strip()
+
+        # Info from the table for this delivery
+        col_1 = d.find("section", class_="middle-split__column1")
+
+        delivery_info["mode_of_study"] = (
+            col_1.find("dt", string="Mode of study").find_next("dd").text.strip()
+        )
+
+        assessment_methods = (
+            col_1.find("dt", string="Methods of assessment")
+            .find_next("dd")
+            .find_all("div")
+        )
+        delivery_info["methods_of_assessment"] = ", ".join(
+            [" ".join(method.text.strip().split()) for method in assessment_methods]
+        )
+
+        delivery_info["mark_scheme"] = (
+            col_1.find("dt", string="Mark scheme").find_next("dd").text.strip()
+        )
+
+        col_2 = d.find("section", class_="middle-split__column2")
+
+        email = col_2.find("a", href=re.compile(r"^mailto:"))
+        delivery_info["contact_email"] = email.text.strip() if email else None
+
+        delivery_info["number_of_students_prior_year"] = (
+            col_2.find("dt", string="Number of students on module in previous year")
+            .find_next("dd")
+            .text.strip()
+        )
+
+        collated_d.append(delivery_info)
+
+    info = {
+        "module_title": module_title,
+        "module_code": module_code,
+        "url": url,
+        "faculty": faculty,
+        "teaching_department": teaching_department,
+        "level": level,
+        "teaching_term": teaching_term,
+        "credit_value": credit_value,
+        "subject": sanitized_subject,
+        "keywords": sanitized_keywords,
+        "alternative_credit_options": alternative_credit_options,
+        "description": description,
+        "restrictions": restrictions,
+        "deliveries": collated_d,
+    }
+
+    return info
+
+
+def _module_info_to_markdown(module_info: dict, template: jinja2.Template) -> str:
+    """Process module information dictionary into markdown document using template."""
+    return template.render(module_info)
+
+
+def _convert_module_html_to_markdown(
+    module_html: str, extract_function: callable, markdown_template: jinja2.Template
+) -> None:
+    """Convert a single UCL module HTML page to a markdown document."""
+    module_info = extract_function(module_html)
+    module_markdown = _module_info_to_markdown(module_info, markdown_template)
+    return module_markdown
+
+
+def convert_all_documents_html_to_markdown(
+    input_dir: str | Path,
+    output_dir: str | Path,
+    extract_function: callable = _extract_module_info_from_html,
+    markdown_template: jinja2.Template = module_template,
+):
+    """Convert all UCL module HTML pages in a directory to markdown documents."""
+
+    input_dir = Path(input_dir)
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    logger.info("""Converting HTML module files to markdown documents""")
+
+    all_module_html_files = list(input_dir.glob("*.html"))
+
+    n_modules = len(all_module_html_files)
+
+    logger.info(
+        f"Identified {n_modules} HTML module files to convert to markdown documents"
+    )
+
+    errors = 0
+    for module_html_path in tqdm(all_module_html_files):
+        try:
+            with open(module_html_path, "r") as f:
+                module_html = f.read()
+
+            module_markdown = _convert_module_html_to_markdown(
+                module_html, extract_function, markdown_template
+            )
+
+            output_path = output_dir / f"{module_html_path.stem}.md"
+            with open(output_path, "w") as f:
+                f.write(module_markdown)
+        except Exception as e:
+            errors += 1
+            logger.error(f"Error converting {module_html_path.stem}: {e}")
+    logger.info(f"{n_modules - errors} HTML files successfully converted to markdown")
+    logger.info(f"{errors} HTML files could not be converted.")
+
+
+@hydra.main(version_base=None, config_path="../conf", config_name="config")
+def main(cfg: omegaconf.DictConfig) -> None:
+    """Run the document conversion process."""
+    cfg = cfg.setup.convert_documents
+    cfg.input_dir = get_abs_path_using_repo_root(cfg.input_dir)
+    cfg.output_dir = get_abs_path_using_repo_root(cfg.output_dir)
+
+    convert_all_documents_html_to_markdown(**cfg)
+
+
+if __name__ == "__main__":
+    main()

src/ucl_module_chat/data_processing/document_embedding.py

@@ -0,0 +1,48 @@
+from pathlib import Path
+
+import hydra
+import omegaconf
+from dotenv import load_dotenv
+from langchain_community.vectorstores import FAISS
+from langchain_core.embeddings.embeddings import Embeddings
+from loguru import logger
+
+from ucl_module_chat.utils.resolve_paths import get_abs_path_using_repo_root
+
+load_dotenv()
+
+
+def embed_documents(input_dir: str | Path, embedding_model: Embeddings) -> FAISS:
+    """Create a FAISS vectorstore from a directory of markdown documents."""
+    input_dir = Path(input_dir)
+
+    all_module_document_paths = list(input_dir.glob("*.md"))
+
+    module_docs = []
+
+    for module_md_path in all_module_document_paths:
+        with open(module_md_path, "r") as f:
+            module_md = f.read()
+        module_docs.append(module_md)
+
+    logger.info(f"Embedding {len(module_docs)} documents")
+    vectorstore = FAISS.from_texts(module_docs, embedding=embedding_model)
+    logger.info(f"Vectorstore created with {vectorstore.index.ntotal} vectors")
+    return vectorstore
+
+
+@hydra.main(version_base=None, config_path="../conf", config_name="config")
+def main(cfg: omegaconf.DictConfig) -> None:
+    """Run the document embedding process."""
+    embedding_model = hydra.utils.instantiate(cfg.models.embedding)
+    cfg.setup.embed_documents.input_dir = get_abs_path_using_repo_root(
+        cfg.setup.embed_documents.input_dir
+    )
+    cfg.vectorstore.dir = get_abs_path_using_repo_root(cfg.vectorstore.dir)
+    vectorstore = embed_documents(cfg.setup.embed_documents.input_dir, embedding_model)
+    vectorstore.save_local(cfg.vectorstore.dir)
+    logger.info(f"Vectorstore saved to {cfg.vectorstore.dir}")
+
+
+if __name__ == "__main__":
+    main()

src/ucl_module_chat/data_processing/document_scraping.py

@@ -0,0 +1,94 @@
+import re
+import time
+from pathlib import Path
+
+import hydra
+import omegaconf
+import requests
+from bs4 import BeautifulSoup
+from loguru import logger
+from tqdm import tqdm
+
+from ucl_module_chat.utils.resolve_paths import get_abs_path_using_repo_root
+
+
+def _get_index_page_html(index_page_url: str):
+    """Get the HTML content of the index page."""
+    response = requests.get(index_page_url)
+    index_page_html = response.text
+    return index_page_html
+
+
+def _get_module_urls_from_index_page(index_page_html: str, regex_url_pattern: str):
+    """Extract module URLs from the index page HTML using regex."""
+    regex_url_pattern = re.compile(regex_url_pattern)
+    soup = BeautifulSoup(index_page_html, "html.parser")
+    pattern = re.compile(regex_url_pattern)
+
+    module_urls = []
+    for cite_tag in soup.find_all("cite"):
+        url = cite_tag["data-url"]
+        if pattern.match(url):
+            module_urls.append(url)
+
+    return module_urls
+
+
+def _save_module_page_html(module_url: str, output_dir: str | Path):
+    """Save the HTML content of a module page to a text file."""
+    output_dir = Path(output_dir)
+
+    # Send a GET request to fetch the HTML content
+    response = requests.get(module_url)
+    response.raise_for_status()  # Raise an exception for HTTP errors
+
+    # Extract the part of the URL after "/modules/" for the filename
+    module_id = module_url.split("/modules/")[1]
+
+    # Save the HTML content to a text file
+    file_path = output_dir / f"{module_id}.html"
+    with open(file_path, "w", encoding="utf-8") as file:
+        file.write(response.text)
+
+
+def scrape_documents(
+    index_page_url: str | Path,
+    output_dir: str | Path,
+    regex_url_pattern: str,
+    wait_time_seconds: int = 2,
+):
+    """Scrape module pages and save HTML content to text files."""
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    logger.info(f"Identifying module pages from {index_page_url}")
+    index_page_html = _get_index_page_html(index_page_url)
+
+    module_urls = _get_module_urls_from_index_page(index_page_html, regex_url_pattern)
+    n_modules = len(module_urls)
+    logger.info(f"Identified {len(module_urls)} module pages to save to {output_dir}.")
+
+    errors = 0
+
+    for url in tqdm(module_urls):
+        try:
+            _save_module_page_html(url, output_dir)
+            time.sleep(wait_time_seconds)  # Pause to avoid abusing the server
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Error saving HTML for {url}: {e}")
+            errors += 1
+
+    logger.info(f"{n_modules - errors} module pages successfully saved")
+    logger.info(f"{errors} module pages could not be saved.")
+
+
+@hydra.main(version_base=None, config_path="../conf", config_name="config")
+def main(cfg: omegaconf.DictConfig) -> None:
+    """Run the document scraping process."""
+    cfg = cfg.setup.scrape_documents
+    cfg.output_dir = get_abs_path_using_repo_root(cfg.output_dir)
+    scrape_documents(**cfg)
+
+
+if __name__ == "__main__":
+    main()

src/ucl_module_chat/data_processing/document_templates.py

@@ -0,0 +1,40 @@
+from jinja2 import Template
+
+module_template = Template(
+    """
+# {{ module_title }} ({{module_code}})
+
+## Key information
+
+**Course Code:** {{ module_code }} \\
+**Subject Area:** {{ subject }} \\
+**Keywords:** {{ keywords }} \\
+**Module catalogue URL:** {{ url }}  
+
+**Faculty:** {{ faculty }} \\
+**Teaching Department:** {{ teaching_department }} \\
+**Credit Value:** {{ credit_value }} \\
+**Restrictions:** {{ restrictions }}
+
+## Alternative credit options
+{{ alternative_credit_options }}
+
+## Description
+{{ description }}
+
+## Module deliveries for 2024/25 academic year
+{% for delivery in deliveries %}
+### {{delivery.type}} (FHEQ Level {{delivery.fheq_level}})
+
+#### Teaching and assessment
+**Intended teaching term:** {{ delivery.teaching_term }} \\
+**Mode of study:** {{ delivery.mode_of_study }} \\
+**Methods of assessment:** {{ delivery.methods_of_assessment }} \\
+**Mark scheme:** {{ delivery.mark_scheme }}
+
+#### Other information
+**Number of students on module in previous year:** {{ delivery.number_of_students_prior_year }} \\
+**Who to contact for more information:** {{ delivery.contact_email }}
+{% endfor %}
+"""
+)

src/ucl_module_chat/setup.py

@@ -0,0 +1,44 @@
+import hydra
+import omegaconf
+from dotenv import load_dotenv
+from loguru import logger
+
+from ucl_module_chat.data_processing.document_conversion import (
+    convert_all_documents_html_to_markdown,
+)
+from ucl_module_chat.data_processing.document_embedding import embed_documents
+from ucl_module_chat.data_processing.document_scraping import scrape_documents
+from ucl_module_chat.utils.resolve_paths import get_abs_path_using_repo_root
+
+load_dotenv()
+
+
+@hydra.main(version_base=None, config_path="conf", config_name="config")
+def main(cfg: omegaconf.DictConfig) -> None:
+    """Scrape module catalogue, convert HTML to markdown, and embed in vectorstore."""
+
+    cfg.setup.scrape_documents.output_dir = get_abs_path_using_repo_root(
+        cfg.setup.scrape_documents.output_dir
+    )
+    cfg.setup.convert_documents.input_dir = get_abs_path_using_repo_root(
+        cfg.setup.convert_documents.input_dir
+    )
+    cfg.setup.convert_documents.output_dir = get_abs_path_using_repo_root(
+        cfg.setup.convert_documents.output_dir
+    )
+    cfg.setup.embed_documents.input_dir = get_abs_path_using_repo_root(
+        cfg.setup.embed_documents.input_dir
+    )
+    cfg.vectorstore.dir = get_abs_path_using_repo_root(cfg.vectorstore.dir)
+
+    scrape_documents(**cfg.setup.scrape_documents)
+    convert_all_documents_html_to_markdown(**cfg.setup.convert_documents)
+
+    embedding_model = hydra.utils.instantiate(cfg.models.embedding)
+    vectorstore = embed_documents(cfg.setup.embed_documents.input_dir, embedding_model)
+    vectorstore.save_local(cfg.vectorstore.dir)
+    logger.info(f"Vectorstore saved to {cfg.vectorstore.dir}")
+
+
+if __name__ == "__main__":
+    main()

src/ucl_module_chat/utils/resolve_paths.py

@@ -0,0 +1,17 @@
+import os
+from pathlib import Path
+
+from git import Repo
+
+
+def get_abs_path_using_repo_root(path: str | Path) -> Path:
+    """Takes path relative to the repo root and returns the absolute path."""
+
+    # Initialize the repo (this will automatically find the root if
+    # you're in a subdirectory)
+    repo = Repo(os.getcwd(), search_parent_directories=True)
+
+    # Get the root directory
+    repo_root = repo.git.rev_parse("--show-toplevel")
+    abs_path = Path(repo_root) / path
+    return abs_path