Merge pull request #1 from tools4ds/retargeting

.bandit

@@ -0,0 +1,2 @@
+[bandit]
+skips = B104

.github/workflows/deploy_to_hf.yml

@@ -0,0 +1,21 @@
+name: Push Production to HuggingFace
+
+on:
+  push:
+    branches: [main, retargeting]
+
+  # run this workflow manualy from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Deploy Production (main) to HuggingFace
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: git push --force https://faridkarimli:[email protected]/spaces/tools4ds/ai_tutor

.gitignore

@@ -1,12 +1,15 @@
 **/vectorstores/*
 **/private/students.json
 .ragatouille/*
-*/__pycache__/*
+**/__pycache__/*
 **/.chainlit/translations/*
-storage/logs/*
-vectorstores/*
+**/storage/logs/*
+**/vectorstores/*
 **/apps/*/storage/logs/*
 **/apps/*/private/*
 *.log
 **/.files/*
 .env
+*.pyc
+venv/
+apps

Dockerfile

@@ -0,0 +1,31 @@
+FROM python:3.11
+
+# Change permissions to allow writing to the directory
+RUN chmod -R 777 /apps/ai_tutor
+
+WORKDIR /apps/ai_tutor
+
+RUN pip install --upgrade pip
+RUN pip install edubotics-core
+
+# Create a logs directory and set permissions
+RUN mkdir logs && chmod 777 logs
+
+# Create a cache directory within the application's working directory
+RUN mkdir /.cache && chmod -R 777 /.cache
+
+# Expose the port the app runs on
+EXPOSE 7860
+
+RUN --mount=type=secret,id=HUGGINGFACEHUB_API_TOKEN,mode=0444,required=true 
+RUN --mount=type=secret,id=OPENAI_API_KEY,mode=0444,required=true 
+RUN --mount=type=secret,id=CHAINLIT_URL,mode=0444,required=true 
+RUN --mount=type=secret,id=LITERAL_API_URL,mode=0444,required=true 
+RUN --mount=type=secret,id=LLAMA_CLOUD_API_KEY,mode=0444,required=true 
+RUN --mount=type=secret,id=OAUTH_GOOGLE_CLIENT_ID,mode=0444,required=true 
+RUN --mount=type=secret,id=OAUTH_GOOGLE_CLIENT_SECRET,mode=0444,required=true 
+RUN --mount=type=secret,id=LITERAL_API_KEY_LOGGING,mode=0444,required=true 
+RUN --mount=type=secret,id=CHAINLIT_AUTH_SECRET,mode=0444,required=true 
+
+# Default command to run the application
+CMD vectorstore_creator --config_file config/config.yml --project_config_file config/project_config.yml && python app.py --host 0.0.0.0 --port 7860

README.md

@@ -1,7 +1,90 @@
-## App Templates
+# DS701: Tools for Data Science - Education Chatbot
 
-```bash
-pip install edubotics-core
-```
+## Overview
 
-P.S. - Do remmber to setup your .env file
+This repository contains an LLM-based chatbot designed for the [DS701: Tools for Data Science](https://tools4ds.github.io/fa2024/) course at Boston University. The chatbot serves as an interactive learning assistant, helping students with course-related queries, explanations of concepts, and guidance on data science tools. It is powered by [edubotics-core](https://github.com/edubotics-ai/edubotics-core) a package of modules for data loading, vector store creation and management.
+
+The chatbot will be available on HuggingFace Spaces soon. Lecture slides are available [here](https://tools4ds.github.io/DS701-Course-Notes/).
+
+## Features
+
+- 24/7 availability for student queries
+- Explanations of key data science concepts
+- Guidance on using data science tools and libraries
+- Practice problem suggestions and explanations
+- Course syllabus and schedule information
+
+## Getting Started Locally
+
+To run the chatbot locally, follow these steps:
+
+1. Clone this repository
+
+   `git clone https://github.com/tools4ds/ds701-tutor.git`
+
+2. Set up your Python environment
+
+   a. Using venv (Python's built-in virtual environment):
+
+   ```python
+   python -m venv ds701_env
+   source ds701_env/bin/activate  # On Windows, use: ds701_env\Scripts\activate
+   pip install -r requirements.txt
+   ```
+
+   b. Using Conda:
+
+   ```python
+   conda create --name ds701_env python=3.9
+   conda activate ds701_env
+   conda install --file requirements.txt
+   ```
+
+Note: Make sure to use Python 3.9 or later.
+
+3. Install dependencies
+
+   `pip install edubotics-core`
+
+4. Set up environment variables. See `apps/ai_tutor/.env.example` for reference.
+
+5. Run the chatbot
+
+   ```python
+    cd apps/ai-tutor
+    python app.py
+   ```
+
+   and navigate to localhost:8000 in your browser.
+
+## Usage
+
+Students can interact with the chatbot through the web interface.
+
+Example queries:
+
+- "Explain the concept of data normalization"
+- "How do I use pandas for data manipulation?"
+- "What's the difference between supervised and unsupervised learning?"
+
+## Contributing
+
+We welcome contributions from the community to enhance the DS701 AI Tutor. Whether you're a student, instructor, or developer, your input is valuable. Here's how you can contribute:
+
+1. Fork the repository and create your feature branch.
+2. Make your changes, ensuring they align with the project's goals.
+3. Submit a pull request with a clear description of your improvements.
+
+For more detailed information on the contribution process, please refer to our `CONTRIBUTING.md` file (unavailable yet).
+
+## Privacy and Data Handling
+
+This chatbot is designed with student privacy in mind. No personal information is stored, and all interactions are anonymized. For more details, please refer to our `PRIVACY_POLICY.md`.
+
+## Support
+
+If you encounter any issues or have suggestions for improvement, please open an issue in this repository or contact the course instructor.
+
+## License
+
+This project is licensed under the MIT License - see the `LICENSE.md` file for details.

apps/ai_tutor/.chainlit/config.toml

@@ -53,10 +53,10 @@ edit_message = true
 
 [UI]
 # Name of the assistant.
-name = "AI Tutor"
+name = "DS701 AI Tutor"
 
 # Description of the assistant. This is used for HTML tags.
-# description = ""
+description = "Your AI tutor for DS701: Tools for Data Science. Here to assist you with course content, answer questions, and provide guidance on data science concepts."
 
 # Large size content are by default collapsed for a cleaner ui
 default_collapse_content = true
@@ -65,7 +65,7 @@ default_collapse_content = true
 cot = "hidden"
 
 # Link to your github repo. This will add a github button in the UI's header.
-github = "https://github.com/edubotics-ai/edubot-core"
+github = "https://github.com/tools4ds/ds701-tutor"
 
 # Specify a CSS file that can be used to customize the user interface.
 # The CSS file can be served from the public directory or via an external link.

apps/ai_tutor/.env.example

@@ -0,0 +1,11 @@
+LLAMA_CLOUD_API_KEY=
+OPENAI_API_KEY=
+HF_TOKEN=
+HUGGING_FACE_TOKEN=
+LITERAL_API_KEY_LOGGING=
+LITERAL_API_URL=https://cloud.getliteral.ai
+CHAINLIT_AUTH_SECRET=
+CHAINLIT_URL=http://localhost:8000
+OAUTH_GOOGLE_CLIENT_ID=
+OAUTH_GOOGLE_CLIENT_SECRET=
+EMAIL_ENCRYPTION_KEY=

apps/ai_tutor/app.py

@@ -25,6 +25,9 @@ from edubotics_core.chat_processor.helpers import get_user_details, update_user_
 from config.config_manager import config_manager
 import hashlib
 
+import uvicorn
+import argparse
+
 # set config
 config = config_manager.get_config().dict()
 
@@ -385,6 +388,10 @@ async def get_tokens_left(request: Request):
 mount_chainlit(app=app, target="chainlit_app.py", path=CHAINLIT_PATH)
 
 if __name__ == "__main__":
-    import uvicorn
+    parser = argparse.ArgumentParser(description="Run the AI Tutor application")
+    parser.add_argument("--host", default="0.0.0.0", help="Host to run the server on")
+    parser.add_argument("--port", type=int, default=7860,
+                        help="Port to run the server on")
+    args = parser.parse_args()
 
-    uvicorn.run(app, host="127.0.0.1", port=7860)
+    uvicorn.run(app, host=args.host, port=args.port)

apps/ai_tutor/chainlit_app.py

@@ -239,23 +239,23 @@ class Chatbot:
             print(e)
             return [
                 cl.Starter(
-                    label="recording on Transformers?",
-                    message="Where can I find the recording for the lecture on Transformers?",
+                    label="What is this course about?",
+                    message="What is this course about? I'm not sure I'm in the right class.",
                     icon="/public/assets/images/starter_icons/adv-screen-recorder-svgrepo-com.svg",
                 ),
                 cl.Starter(
-                    label="where's the schedule?",
+                    label="Where is the schedule?",
                     message="When are the lectures? I can't find the schedule.",
                     icon="/public/assets/images/starter_icons/alarmy-svgrepo-com.svg",
                 ),
                 cl.Starter(
-                    label="Due Date?",
-                    message="When is the final project due?",
+                    label="When are office hours?",
+                    message="When are the office hours for this course?",
                     icon="/public/assets/images/starter_icons/calendar-samsung-17-svgrepo-com.svg",
                 ),
                 cl.Starter(
-                    label="Explain backprop.",
-                    message="I didn't understand the math behind backprop, could you explain it?",
+                    label="Explain KMeans",
+                    message="I didn't understand the math behind KMeans, could you explain it?",
                     icon="/public/assets/images/starter_icons/acastusphoton-svgrepo-com.svg",
                 ),
             ]

apps/ai_tutor/config/config.yml

@@ -3,12 +3,12 @@ log_chunk_dir: 'storage/logs/chunks' # str
 device: 'cpu' # str [cuda, cpu]
 
 vectorstore:
-  load_from_HF: True # bool
+  load_from_HF: False # bool
   reparse_files: True # bool
   data_path: 'storage/data' # str
   url_file_path: 'storage/data/urls.txt' # str
   expand_urls: True # bool
-  db_option : 'RAGatouille' # str [FAISS, Chroma, RAGatouille, RAPTOR]
+  db_option : 'FAISS' # str [FAISS, Chroma, RAGatouille, RAPTOR]
   db_path : 'vectorstores' # str
   model : 'sentence-transformers/all-MiniLM-L6-v2' # str [sentence-transformers/all-MiniLM-L6-v2, text-embedding-ada-002']
   search_top_k : 3 # int
@@ -39,7 +39,7 @@ llm_params:
     filename: 'tinyllama-1.1b-chat-v1.0.Q5_0.gguf' # Specific name of gguf file in the repo
     model_path: 'storage/models/tinyllama-1.1b-chat-v1.0.Q5_0.gguf' # Path to the model file
   stream: False # bool
-  pdf_reader: 'gpt' # str [llama, pymupdf, gpt]
+  pdf_reader: 'pymupdf' # str [llama, pymupdf, gpt]
 
 chat_logging:
   log_chat: True # bool

apps/ai_tutor/config/prompts.py

@@ -5,15 +5,15 @@ prompts = {
             "Incorporate relevant details from the chat history to make the question clearer and more specific. "
             "Do not change the meaning of the original statement, and maintain the student's tone and perspective. "
             "If the question is conversational and doesn't require context, do not rephrase it. "
-            "Example: If the student previously asked about backpropagation in the context of deep learning and now asks 'what is it', rephrase to 'What is backpropagation.'. "
-            "Example: Do not rephrase if the user is asking something specific like 'cool, suggest a project with transformers to use as my final project' "
+            "Example: If the student previously asked about normalization in the context of data science and now asks 'what is it', rephrase to 'What is normalization?' "
+            "Example: Do not rephrase if the user is asking something specific like 'cool, suggest a project with SVMs to use as my final project' "
             "Chat history: \n{chat_history}\n"
             "Rephrase the following question only if necessary: '{input}'"
             "Rephrased Question:'"
         ),
         "prompt_with_history": {
             "normal": (
-                "You are an AI Tutor for the course DS598, taught by Prof. Thomas Gardos. Answer the user's question using the provided context. Only use the context if it is relevant. The context is ordered by relevance. "
+                "You are an AI Tutor for the course DS701, taught by Prof. Thomas Gardos and Prof. Scott Ladenheim. Answer the user's question using the provided context. Only use the context if it is relevant. The context is ordered by relevance. "
                 "If you don't know the answer, do your best without making things up. Keep the conversation flowing naturally. "
                 "Use chat history and context as guides but avoid repeating past responses. Provide links from the source_file metadata. Use the source context that is most relevant. "
                 "Render math equations in LaTeX format between $ or $$ signs, stick to the parameter and variable icons found in your context. Be sure to explain the parameters and variables in the equations."
@@ -26,7 +26,7 @@ prompts = {
                 "AI Tutor:"
             ),
             "eli5": (
-                "You are an AI Tutor for the course DS598, taught by Prof. Thomas Gardos. Your job is to explain things in the simplest and most engaging way possible, just like the 'Explain Like I'm 5' (ELI5) concept."
+                "You are an AI Tutor for the course DS701, taught by Prof. Thomas Gardos and Prof. Scott Ladenheim. Your job is to explain things in the simplest and most engaging way possible, just like the 'Explain Like I'm 5' (ELI5) concept."
                 "If you don't know the answer, do your best without making things up. Keep your explanations straightforward and very easy to understand."
                 "Use the chat history and context to help you, but avoid repeating past responses. Provide links from the source_file metadata when they're helpful."
                 "Use very simple language and examples to explain any math equations, and put the equations in LaTeX format between $ or $$ signs."
@@ -41,7 +41,7 @@ prompts = {
                 "AI Tutor:"
             ),
             "socratic": (
-                "You are an AI Tutor for the course DS598, taught by Prof. Thomas Gardos. Engage the student in a Socratic dialogue to help them discover answers on their own. Use the provided context to guide your questioning."
+                "You are an AI Tutor for the course DS701, taught by Prof. Thomas Gardos and Prof. Scott Ladenheim. Engage the student in a Socratic dialogue to help them discover answers on their own. Use the provided context to guide your questioning."
                 "If you don't know the answer, do your best without making things up. Keep the conversation engaging and inquisitive."
                 "Use chat history and context as guides but avoid repeating past responses. Provide links from the source_file metadata when relevant. Use the source context that is most relevant."
                 "Speak in a friendly and engaging manner, encouraging critical thinking and self-discovery."
@@ -55,7 +55,7 @@ prompts = {
             ),
         },
         "prompt_no_history": (
-            "You are an AI Tutor for the course DS598, taught by Prof. Thomas Gardos. Answer the user's question using the provided context. Only use the context if it is relevant. The context is ordered by relevance. "
+            "You are an AI Tutor for the course DS701, taught by Prof. Thomas Gardos and Prof. Scott Ladenheim. Answer the user's question using the provided context. Only use the context if it is relevant. The context is ordered by relevance. "
             "If you don't know the answer, do your best without making things up. Keep the conversation flowing naturally. "
             "Provide links from the source_file metadata. Use the source context that is most relevant. "
             "Speak in a friendly and engaging manner, like talking to a friend. Avoid sounding repetitive or robotic.\n\n"
@@ -68,7 +68,7 @@ prompts = {
     "tiny_llama": {
         "prompt_no_history": (
             "system\n"
-            "Assistant is an intelligent chatbot designed to help students with questions regarding the course DS598, taught by Prof. Thomas Gardos. Answer the user's question using the provided context. Only use the context if it is relevant. The context is ordered by relevance.\n"
+            "Assistant is an intelligent chatbot designed to help students with questions regarding the course DS701, taught by Prof. Thomas Gardos and Prof. Scott Ladenheim. Answer the user's question using the provided context. Only use the context if it is relevant. The context is ordered by relevance.\n"
             "If you don't know the answer, do your best without making things up. Keep the conversation flowing naturally.\n"
             "Provide links from the source_file metadata. Use the source context that is most relevant.\n"
             "Speak in a friendly and engaging manner, like talking to a friend. Avoid sounding repetitive or robotic.\n"
@@ -81,7 +81,7 @@ prompts = {
         ),
         "prompt_with_history": (
             "system\n"
-            "You are an AI Tutor for the course DS598, taught by Prof. Thomas Gardos. Answer the user's question using the provided context. Only use the context if it is relevant. The context is ordered by relevance. "
+            "You are an AI Tutor for the course DS701, taught by Prof. Thomas Gardos and Prof. Scott Ladenheim. Answer the user's question using the provided context. Only use the context if it is relevant. The context is ordered by relevance. "
             "If you don't know the answer, do your best without making things up. Keep the conversation flowing naturally. "
             "Use chat history and context as guides but avoid repeating past responses. Provide links from the source_file metadata. Use the source context that is most relevant. "
             "Speak in a friendly and engaging manner, like talking to a friend. Avoid sounding repetitive or robotic.\n"

apps/ai_tutor/public/files/test.css

@@ -1,32 +1,33 @@
-a[href*='https://github.com/Chainlit/chainlit'] {
-    visibility: hidden;
+a[href*='https://github.com/Chainlit/chainlit']
+{
+  visibility: hidden;
 }
 
-/* Hide the default avatar image */ 
+/* Hide the default avatar image */
 .MuiAvatar-root img.MuiAvatar-img {
-    display: none;
-  }
-  
+  display: none;
+}
+
 /* Target the container of the image and set a custom background image */
 .MuiAvatar-root.MuiAvatar-circular.css-m2icte {
-    background-image: url('/public/assets/images/avatars/ai-tutor.png'); /* Replace with your custom image URL */
-    background-size: cover; /* Ensure the image covers the entire container */
-    background-position: center; /* Center the image */
-    width: 100px; /* Ensure the dimensions match the original */
-    height: 100px; /* Ensure the dimensions match the original */
-    border-radius: 50%; /* Maintain circular shape */
+  background-image: url("/public/assets/images/avatars/logo.png"); /* Replace with your custom image URL */
+  background-size: cover; /* Ensure the image covers the entire container */
+  background-position: center; /* Center the image */
+  width: 100px; /* Ensure the dimensions match the original */
+  height: 100px; /* Ensure the dimensions match the original */
+  border-radius: 50%; /* Maintain circular shape */
 }
 .MuiAvatar-root.MuiAvatar-circular.css-v72an7 {
-    background-image: url('/public/assets/images/avatars/ai-tutor.png'); /* Replace with your custom image URL */
-    background-size: cover; /* Ensure the image covers the entire container */
-    background-position: center; /* Center the image */
-    width: 40px; /* Ensure the dimensions match the original */
-    height: 40px; /* Ensure the dimensions match the original */
-    border-radius: 50%; /* Maintain circular shape */
+  background-image: url("/public/assets/images/avatars/logo.png"); /* Replace with your custom image URL */
+  background-size: cover; /* Ensure the image covers the entire container */
+  background-position: center; /* Center the image */
+  width: 40px; /* Ensure the dimensions match the original */
+  height: 40px; /* Ensure the dimensions match the original */
+  border-radius: 50%; /* Maintain circular shape */
 }
 
 .MuiStack-root.css-14k6mw7 img {
-    content: url('/public/assets/images/avatars/ai-tutor.png'); /* Replace with the path to your custom image */
-    max-height: 45px; /* Ensure the height remains consistent */
-    max-width: 45px; /* Ensure the width remains consistent */
-}
+  content: url("/public/assets/images/avatars/logo.png"); /* Replace with the path to your custom image */
+  max-height: 45px; /* Ensure the height remains consistent */
+  max-width: 45px; /* Ensure the width remains consistent */
+}

apps/ai_tutor/storage/data/urls.txt

@@ -1 +1,2 @@
-https://dl4ds.github.io/sp2024/
+https://tools4ds.github.io/fa2024/
+https://tools4ds.github.io/DS701-Course-Notes/