change to ollama

Dockerfile

@@ -4,8 +4,17 @@
 # Use the official Python 3.10.9 image
 FROM python:3.12.1
 
+# Install curl and Ollama
+RUN apt-get update && apt-get install -y curl && \
+    curl -fsSL https://ollama.ai/install.sh | sh && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
 # RUN apt-get update -qq && apt-get install ffmpeg -y
-WORKDIR /app
 
 # Copy the current directory contents into the container at .
 COPY . .
@@ -14,10 +23,7 @@ COPY . .
 # Install requirements.txt
 RUN pip install -r requirements.txt
 
-RUN useradd -m -u 1000 user
-USER user
-ENV HOME=/home/user \
-	PATH=/home/user/.local/bin:$PATH
+
 
 WORKDIR $HOME/app
 
@@ -29,5 +35,9 @@ COPY --chown=user . $HOME/app
 
 EXPOSE 7860
 
+# Make the start script executable
+RUN chmod +x start.sh
+
 # Start the FastAPI app on port 7860, the default port expected by Spaces
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--reload"]
+CMD ["./start.sh"]
+# CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--reload"]

Dockerfile.copy

@@ -0,0 +1,33 @@
+
+
+
+# Use the official Python 3.10.9 image
+FROM python:3.12.1
+
+# RUN apt-get update -qq && apt-get install ffmpeg -y
+WORKDIR /app
+
+# Copy the current directory contents into the container at .
+COPY . .
+
+
+# Install requirements.txt
+RUN pip install -r requirements.txt
+
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
+WORKDIR $HOME/app
+
+# RUN mkdir -p $HOME/app/cached/hub/models--Systran--faster-whisper-small
+
+# COPY --from=model /tmp/model $HOME/app/cached/hub/models--Systran--faster-whisper-small
+
+COPY --chown=user . $HOME/app
+
+EXPOSE 7860
+
+# Start the FastAPI app on port 7860, the default port expected by Spaces
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--reload"]

app.py

@@ -3,7 +3,7 @@ import os
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
-from routers import get_transcript
+from routers import get_transcript, get_chatrespone
 
 os.environ['HF_HOME'] = "./cached/"
 
@@ -12,6 +12,7 @@ app = FastAPI()
 app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["Content-Type", "Authorization", "x-api-key"])
 
 app.include_router(get_transcript.router)
+app.include_router(get_chatrespone.router)
 
 @app.get("/")
 def read_root():

routers/get_chatrespone.py

@@ -0,0 +1,83 @@
+import os
+from typing import Iterator
+from dotenv import load_dotenv
+from fastapi import APIRouter, Depends, Request
+
+from langchain_huggingface import HuggingFaceEndpoint
+from langchain_core.prompts import PromptTemplate
+
+from libs.header_api_auth import get_api_key
+from pydantic import BaseModel
+from fastapi.responses import StreamingResponse
+from langchain_ollama import ChatOllama, OllamaLLM
+
+load_dotenv()
+HUGGINGFACEHUB_API_TOKEN = os.getenv("HUGGINGFACEHUB_API_TOKEN", )
+
+os.environ["HUGGINGFACEHUB_API_TOKEN"] = HUGGINGFACEHUB_API_TOKEN
+
+router = APIRouter(prefix="/get-chat-response", tags=["chat"])
+
+class ChatInputForm(BaseModel):
+    textInput: str
+    repo_id: str
+    prompt: str
+
+@router.post("/")
+async def get_chat_respone(body: ChatInputForm):
+
+    prompt = get_prompt(body.prompt)
+
+    promptTemplate = PromptTemplate.from_template(prompt)
+    try:
+        llm = OllamaLLM(
+            model="llama3.2",
+            temperature=0.2,
+            # huggingfacehub_api_token=HUGGINGFACEHUB_API_TOKEN,
+        )
+
+        messages = [
+            ("system", prompt),
+            ("human", body.textInput)
+        ]
+
+
+        llm_chain = promptTemplate | llm
+        response = llm.stream(messages)
+
+        return StreamingResponse(get_response(response), media_type='text/event-stream')
+    except Exception:
+        return {"success": False, "status": Exception}
+
+
+# get_chat_respone()
+
+def get_response(response: Iterator[str]):
+    for chunk in response:
+        yield chunk
+
+
+checkWritting = """You'll be provided with a text: {prompt}
+---------------
+IMPORTANT:
+- If the text is empty, do nothing.
+- If the given text maintains grammatical accuracy, no suggestions are needed.
+- If the given text is empty, do nothing.
+- If the given text contains any errors in grammatical accuracy, provide the corrected text.
+
+"""
+
+template = """You are a helpful English teacher. Chat and do user requirement.
+    Answer: Let's think step by step."""
+baiGiang = """Provide the given phrase in English. Provide the correct and popularly used English phrase along with its American IPA pronunciation and a brief explanation for it. Use the correct English phrase to create 4 example sentences along with the example IPA and brief meanings. Finally, suggest 4 similar English phrases with the correct English version, along with American IPA and their brief meanings.
+Provie your response in markdown format"""
+
+def get_prompt(prompt: str):
+    prompts = {
+        'template' : template,
+        'checkWritting': checkWritting,
+        'baiGiang': baiGiang
+    }
+
+    return prompts.get('template', template)
+

start.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Set environment variables for optimization
+export OMP_NUM_THREADS=4
+export MKL_NUM_THREADS=4
+export CUDA_VISIBLE_DEVICES=0
+
+# Start Ollama in the background
+ollama serve &
+
+# Pull the model if not already present
+if ! ollama list | grep -q "llama3.2:latest"; then
+    ollama pull llama3.2:latest
+fi
+
+# Wait for Ollama to start up
+max_attempts=30
+attempt=0
+while ! curl -s http://localhost:11434/api/tags >/dev/null; do
+    sleep 1
+    attempt=$((attempt + 1))
+    if [ $attempt -eq $max_attempts ]; then
+        echo "Ollama failed to start within 30 seconds. Exiting."
+        exit 1
+    fi
+done
+
+echo "Ollama is ready."
+
+# Print the API URL
+echo "API is running on: http://0.0.0.0:7860"
+
+# Start the FastAPI server
+uvicorn app:app --host 0.0.0.0 --port 7860 --workers 4 --limit-concurrency 20