Product Recommendation RestAPI

.gitignore

@@ -0,0 +1,29 @@
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+.env
+.venv
+venv/
+ENV/
+.idea/
+.vscode/
+*.pkl
+models/*
+!models/.gitkeep

README.md

@@ -11,4 +11,55 @@ license: apache-2.0
 short_description: Testing for e-commerce
 ---
 
-An example chatbot using [Gradio](https://gradio.app), [`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/v0.25.2/en/index), and the [Hugging Face Inference API](https://huggingface.co/docs/api-inference/index).
+An example chatbot using [Gradio](https://gradio.app), [`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/v0.25.2/en/index), and the [Hugging Face Inference API](https://huggingface.co/docs/api-inference/index).
+
+# README.md
+# Product Recommender
+
+A machine learning-powered product recommendation system that uses semantic search to find similar products based on user queries.
+
+## Features
+
+- Semantic search using sentence transformers
+- FastAPI backend for quick recommendations
+- BigQuery integration for training data
+- HuggingFace model hosting
+- Docker support for easy deployment
+
+## Setup
+
+1. Install dependencies:
+```bash
+pip install -e .
+```
+
+2. Copy and configure environment variables:
+```bash
+cp backend/.env.example backend/.env
+# Edit .env with your credentials
+```
+
+3. Train the model:
+```bash
+python -m backend.train
+```
+
+4. Start the API:
+```bash
+python -m backend.server
+```
+
+## API Documentation
+
+Once running, visit http://localhost:8000/docs for the OpenAPI documentation.
+
+## Docker Usage
+
+```bash
+cd backend
+docker-compose up --build
+```
+
+## License
+
+MIT License - See LICENSE file for details

backend/Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.9
+
+WORKDIR /app
+COPY . /app
+
+RUN pip install --no-cache-dir fastapi uvicorn transformers torch sentence-transformers scikit-learn python-dotenv google-cloud-bigquery
+
+CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000"]

backend/__init__.py

@@ -0,0 +1 @@
+# Empty file to make the directory a Python package

app.py → backend/app.py

@@ -1,74 +1,62 @@
+import os
 import gradio as gr
 from huggingface_hub import InferenceClient
 from huggingface_hub.utils import HfHubHTTPError
-import os
 from dotenv import load_dotenv
 
-# Load environment variables from .env file
-load_dotenv()
+# Load environment variables from .env file in parent directory
+env_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.env')
+load_dotenv(dotenv_path=env_path)
 
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
 api_token = os.getenv("HUGGINGFACE_API_TOKEN")
 if not api_token:
-    raise ValueError("HUGGINGFACE_API_TOKEN environment variable not set")
+    raise ValueError("HUGGINGFACE_API_TOKEN environment variable not set. Please check your .env file.")
 
 client = InferenceClient("HuggingFaceH4/zephyr-7b-beta", token=api_token)
 
-
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
+def respond(message, history, system_message, max_tokens, temperature, top_p):
     messages = [{"role": "system", "content": system_message}]
-
+    
     for val in history:
         if val[0]:
             messages.append({"role": "user", "content": val[0]})
         if val[1]:
             messages.append({"role": "assistant", "content": val[1]})
-
+    
     messages.append({"role": "user", "content": message})
-
     response = ""
-
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
-
-        response += token
-        yield response
-
-
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
+    
+    try:
+        for message in client.chat_completion(
+            messages,
+            max_tokens=max_tokens,
+            stream=True,
+            temperature=temperature,
+            top_p=top_p,
+        ):
+            if hasattr(message.choices[0], 'delta') and hasattr(message.choices[0].delta, 'content'):
+                token = message.choices[0].delta.content
+                response += token
+                yield response
+    except Exception as e:
+        yield f"Error: {str(e)}"
+
+# Create the Gradio interface
 demo = gr.ChatInterface(
-    respond,
+    fn=respond,
+    title="E-commerce Chatbot",
+    description="Ask me anything about products!",
+    examples=["Tell me about gaming laptops", "What are the best smartphones?"],
     additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
+        gr.Textbox(value="You are a friendly E-commerce assistant.", label="System message"),
         gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
         gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
+        gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p")
+    ]
 )
 
-
 if __name__ == "__main__":
-    demo.launch()
+    try:
+        demo.launch(server_name="0.0.0.0", server_port=8000, share=True)
+    except Exception as e:
+        print(f"Error launching the demo: {e}")

backend/config/settings.py

@@ -0,0 +1,26 @@
+from pydantic_settings import BaseSettings
+from typing import Optional
+
+class Settings(BaseSettings):
+    # API Settings
+    API_HOST: str = "0.0.0.0"
+    API_PORT: int = 8000
+    
+    # Model Settings
+    MODEL_NAME: str = "all-MiniLM-L6-v2"
+    MODEL_PATH: str = "../models/product_embeddings.pkl"
+    
+    # HuggingFace Settings
+    HUGGINGFACE_API_TOKEN: str
+    HUGGINGFACE_USERNAME: Optional[str] = None
+    
+    # BigQuery Settings
+    BIGQUERY_PROJECT_ID: str
+    BIGQUERY_DATASET: str
+    BIGQUERY_TABLE: str
+    GOOGLE_APPLICATION_CREDENTIALS: str
+    
+    class Config:
+        env_file = ".env"
+
+settings = Settings()

backend/docker-compose.yml

@@ -0,0 +1,16 @@
+version: '3.8'
+
+services:
+  api:
+    build: .
+    ports:
+      - "8000:8000"
+    volumes:
+      - ../models:/app/models
+    env_file:
+      - .env
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3

backend/models/__init__.py

@@ -0,0 +1 @@
+# Empty file to make the directory a Python package

backend/models/schemas.py

@@ -0,0 +1,18 @@
+from pydantic import BaseModel
+from typing import List, Optional
+
+class RecommendationRequest(BaseModel):
+    query: str
+    top_n: Optional[int] = 5
+
+class ProductRecommendation(BaseModel):
+    product_id: str
+    product_name: str
+    description: str
+    brand: str
+    price: float
+    categories: List[str]
+    score: float
+
+class RecommendationResponse(BaseModel):
+    recommendations: List[ProductRecommendation]

backend/recommend.py

@@ -0,0 +1,36 @@
+import pickle
+import numpy as np
+from sentence_transformers import SentenceTransformer
+from sklearn.metrics.pairwise import cosine_similarity
+
+# Load model embeddings
+with open("../models/product_embeddings.pkl", "rb") as f:
+    data = pickle.load(f)
+
+# Load transformer model
+model = SentenceTransformer('all-MiniLM-L6-v2')
+
+def recommend_products(user_query, top_n=5):
+    """Find similar products based on user search"""
+    query_embedding = model.encode(user_query).reshape(1, -1)
+    
+    # Compute similarity scores
+    similarities = cosine_similarity(query_embedding, data["embeddings"])
+    top_indices = np.argsort(similarities[0])[-top_n:][::-1]  # Get top matches
+
+    recommendations = []
+    for i in top_indices:
+        recommendations.append({
+            "search_query": data["search_queries"][i],
+            "product": data["product_names"][i],
+            "score": float(similarities[0][i])
+        })
+
+    return recommendations
+
+# Example test
+if __name__ == "__main__":
+    query = "gaming laptop"
+    results = recommend_products(query)
+    for r in results:
+        print(f"🔹 {r['product']} (Score: {r['score']:.2f})")

backend/requirements.txt

@@ -0,0 +1,12 @@
+huggingface_hub==0.25.2
+requests==2.31.0
+python-dotenv==1.0.0
+fastapi==0.103.1
+uvicorn==0.23.2
+torch>=2.2.0
+transformers>=4.38.1
+gradio>=4.19.2
+sentence-transformers>=2.5.0
+scikit-learn>=1.4.0
+numpy>=1.26.0
+pandas>=2.2.0

backend/server.py

@@ -0,0 +1,101 @@
+import os
+import warnings
+import logging
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+import uvicorn
+
+# Suppress TensorFlow warnings
+warnings.filterwarnings('ignore', category=UserWarning)
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'  # Suppress TensorFlow logging
+logging.getLogger('tensorflow').setLevel(logging.ERROR)
+
+from pydantic import BaseModel
+import pickle
+import numpy as np
+from sentence_transformers import SentenceTransformer
+from sklearn.metrics.pairwise import cosine_similarity
+from backend.models.schemas import RecommendationRequest, ProductRecommendation, RecommendationResponse
+
+# Initialize FastAPI app with metadata
+app = FastAPI(
+    title="Product Recommendation API",
+    description="API for getting product recommendations based on user queries",
+    version="1.0.0"
+)
+
+# Add CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Load embeddings from the correct path
+models_dir = os.path.join(os.path.dirname(__file__), "models")
+embeddings_path = os.path.join(models_dir, "product_embeddings.pkl")
+
+# Ensure models directory exists
+os.makedirs(models_dir, exist_ok=True)
+
+try:
+    # Load embeddings
+    if os.path.exists(embeddings_path):
+        with open(embeddings_path, "rb") as f:
+            data = pickle.load(f)
+    else:
+        raise FileNotFoundError("Product embeddings file not found")
+
+    # Load model
+    model = SentenceTransformer('all-MiniLM-L6-v2')
+except Exception as e:
+    print(f"Error initializing server: {e}")
+    raise
+
+@app.get("/")
+def read_root():
+    return {
+        "message": "Product Recommendation API",
+        "status": "active",
+        "total_products": len(data["product_ids"]) if "product_ids" in data else 0
+    }
+
+@app.post("/recommend", response_model=RecommendationResponse)
+def recommend_products(request: RecommendationRequest):
+    """Get product recommendations based on user search"""
+    try:
+        query_embedding = model.encode(request.query).reshape(1, -1)
+        
+        # Compute similarity scores
+        similarities = cosine_similarity(query_embedding, data["embeddings"])
+        top_indices = np.argsort(similarities[0])[-request.top_n:][::-1]
+
+        recommendations = []
+        for i in top_indices:
+            recommendations.append(ProductRecommendation(
+                product_id=data["product_ids"][i],
+                product_name=data["product_names"][i],
+                description=data["descriptions"][i],
+                brand=data["brands"][i],
+                price=float(data["prices"][i]),
+                categories=eval(data["categories"][i]) if isinstance(data["categories"][i], str) else data["categories"][i],
+                score=float(similarities[0][i])
+            ))
+
+        return RecommendationResponse(recommendations=recommendations)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+if __name__ == "__main__":
+    try:
+        uvicorn.run(
+            "backend.server:app",
+            host="0.0.0.0",
+            port=8000,
+            reload=True,
+            access_log=True
+        )
+    except Exception as e:
+        print(f"Error starting server: {e}")

backend/tests/conftest.py

@@ -0,0 +1,19 @@
+import pytest
+from pathlib import Path
+import pickle
+import numpy as np
+
+@pytest.fixture
+def mock_embeddings():
+    return {
+        "embeddings": np.random.rand(10, 384),
+        "search_queries": ["query1", "query2"],
+        "product_names": ["product1", "product2"]
+    }
+
+@pytest.fixture
+def mock_model_path(tmp_path, mock_embeddings):
+    model_path = tmp_path / "product_embeddings.pkl"
+    with open(model_path, "wb") as f:
+        pickle.dump(mock_embeddings, f)
+    return model_path

backend/train.py

@@ -0,0 +1,90 @@
+import os
+import pickle
+import numpy as np
+import pandas as pd
+from google.cloud import bigquery
+from sentence_transformers import SentenceTransformer
+from dotenv import load_dotenv
+from pathlib import Path
+
+# Load environment variables
+load_dotenv()
+
+def setup_credentials():
+    """Setup Google Cloud credentials"""
+    creds_path = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
+    if not creds_path:
+        raise ValueError("GOOGLE_APPLICATION_CREDENTIALS not set in .env file")
+    
+    # Convert to absolute path if relative
+    if not os.path.isabs(creds_path):
+        creds_path = os.path.join(os.path.dirname(__file__), creds_path)
+    
+    if not os.path.exists(creds_path):
+        raise FileNotFoundError(f"Credentials file not found at: {creds_path}")
+    
+    os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = creds_path
+    return creds_path
+
+def main():
+    # Setup credentials
+    creds_path = setup_credentials()
+    print(f"Using credentials from: {creds_path}")
+
+    # BigQuery setup
+    client = bigquery.Client()
+    query = f"""
+    SELECT 
+        product_id,
+        product_name,
+        description,
+        category,
+        brand,
+        price
+    FROM `{os.getenv("BIGQUERY_PROJECT_ID")}.{os.getenv("BIGQUERY_DATASET")}.{os.getenv("BIGQUERY_TABLE")}`
+    WHERE status = 'ACTIVE'
+    """
+    
+    try:
+        df = client.query(query).to_dataframe()
+    except Exception as e:
+        print(f"Error querying BigQuery: {e}")
+        return
+
+    # Prepare product descriptions for embedding
+    df['combined_text'] = df.apply(
+        lambda x: f"{x['product_name']} {x['description']} {x['brand']} {' '.join(eval(x['category']))}",
+        axis=1
+    )
+
+    # Load pre-trained NLP model
+    model = SentenceTransformer('all-MiniLM-L6-v2')
+
+    # Generate embeddings for product descriptions
+    print("Generating embeddings...")
+    df["embedding"] = df["combined_text"].apply(lambda x: model.encode(x))
+
+    # Save embeddings
+    embeddings_dict = {
+        "product_ids": df["product_id"].tolist(),
+        "product_names": df["product_name"].tolist(),
+        "descriptions": df["description"].tolist(),
+        "brands": df["brand"].tolist(),
+        "prices": df["price"].tolist(),
+        "categories": df["category"].tolist(),
+        "embeddings": np.vstack(df["embedding"].values)
+    }
+
+    # Save model embeddings
+    output_dir = os.path.join(os.path.dirname(__file__), "models")
+    os.makedirs(output_dir, exist_ok=True)
+    
+    output_path = os.path.join(output_dir, "product_embeddings.pkl")
+    with open(output_path, "wb") as f:
+        pickle.dump(embeddings_dict, f)
+
+    print(f"✅ Model trained and embeddings saved to: {output_path}")
+    print(f"Total products processed: {len(df)}")
+
+if __name__ == "__main__":
+    main()

backend/upload_model.py

@@ -0,0 +1,22 @@
+import shutil
+from huggingface_hub import HfApi
+
+# Hugging Face setup
+HF_USERNAME = "your-hf-username"
+REPO_NAME = f"{HF_USERNAME}/product-recommendation"
+api = HfApi()
+
+# Create Hugging Face repo (if not exists)
+api.create_repo(REPO_NAME, exist_ok=True)
+
+# Copy embeddings to upload
+shutil.copy("../models/product_embeddings.pkl", "product_embeddings.pkl")
+
+# Upload to Hugging Face
+api.upload_file(
+    path_or_fileobj="product_embeddings.pkl",
+    path_in_repo="product_embeddings.pkl",
+    repo_id=REPO_NAME
+)
+
+print(f"✅ Model uploaded to Hugging Face: https://huggingface.co/{REPO_NAME}")

backend/utils/helpers.py

@@ -0,0 +1,25 @@
+import logging
+from pathlib import Path
+from typing import Optional
+
+def setup_logging(name: str, level: str = "INFO") -> logging.Logger:
+    logger = logging.getLogger(name)
+    logger.setLevel(level)
+    
+    if not logger.handlers:
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        )
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+    
+    return logger
+
+def ensure_model_path(path: Optional[str] = None) -> Path:
+    if path is None:
+        path = "../models"
+    
+    model_path = Path(path)
+    model_path.mkdir(parents=True, exist_ok=True)
+    return model_path

key/carteya-432304-f95ff2ea1148.json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "carteya-432304",
+  "private_key_id": "f95ff2ea1148613b8c8c5957d7abda01b59a6ba0",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDGFYbpV4yF2Li3\ncijUkelpIe8Tr6/akeqyBwLD/zaypyHjHK12j53ZC3I7mFNgMC5Iz+HLD/QYtIET\nxp33fH5GFIC2d7USeco67Ob9T9SAIQZFVTf0yhmixYO+uCvOobwUwhZqXJVEAdxz\nvRxSWvhaWEb+SQHfIaGlDZskwQgP6UJ0/dYa/I0kOrjDIaoID3p54Wz5+un0AmU3\nD/hWCSiLQNENt2y1Zl+X4GZMQ6N8Cx3OFAzgmFjcxNM8y+lA7vlY/YNhVWpt1Rnx\nIksxs9tGm0VuMOYR1poX5fUCAi+xjBzPAFlCd4YKNPpiR7Z33eiYsYFNwBFn7akI\nS1Uy2TE7AgMBAAECggEAD+ZVdHBzFqFNPmha76kgp7AyHQ3qmyxAPvcCGVJohRGW\nTkEUiWJAxNW+3xq8gomLhYuuL5sBuGPwwnIcZuumxSBLLppQPgHpoi424llgkMJV\n2wXannsEcMTS7OGinozr6Gyl2PbCgnl1xRKGbfPgYHwagrnAfTUZYQLFE7+s0n3+\nC0gzYMc0KxZlBZRRLoDXP+QpaGg8x+C0LKG2VFi2n7LMgPJKxGLJGDBZzCWcxIdh\nWEus1FyYCZfBAtT7Z8sUqYAEZDfuE8huMyXZM+ChMsi1C6T5H30CtpWgfHwZhoRA\nyAopPxAMR1VYKIZQ8aNE0qeE5D8SzboPfKtfeIw0jQKBgQDpjFbb38e00aAvPULP\n8FKvUwIKvIMufpfExNXLpgt3GUZwr3nUJpibn3J2Wr+E2vESBqd56F7/47Uog8Ba\n0OjoCUVofQJsKWeBXz+FSmKuYwXgFxr5JTEd7Q7m6CTJrxODbOcOw6Mgb9vUGtWA\nkdeedBdTfon7PjG6UaXg7AU89QKBgQDZIGjyWUC4xhsgxhEiUdeFvq+hFkkgdNVS\nQA9BGXHuD63qICeQl6637Vxk9R8lEguGmY6JL65Xk62/qzNNxfdM0wHo/83VIqeO\na2PKRbwKAC25alsdDjfuZx5xTF/eIetPtxHnhm5tkLsi1IzhWJhPQamm7DFW307p\n1fgBv5rXbwKBgCKxFQnuMNUcOmYp/G207iNa+jesDvRuG49ACd4JWTSkzYDZoAgy\nf4LGaZNJIZ/TgfHCiUgji1EJexR8lasn0yD8l5HQ1/ZHJDfkdSQQi54J2YqTBA+0\nRZqDZy5Hl4dZ+VURbTUVPHsKAqZ8IfDKOTQxXHGrn0Vx1KMvk6tYg+KtAoGBAL8Q\nHAmWrLNr8JdIc+9lYWgZwmbYO4VKCiWUpVe90lmQhpPNs8MzFtZXEsTsRnnShT3u\nhlUGFj6Of4h4WG8J03JRBA9KepAhLJzQt0FZV/zc51+PzZml12X8a/d7I6lO48iD\n6kd4LlhP0bz2mPn7ghvMRkPS+B/f5YszdEflmZYVAoGANuiMZ2GbCiImxltDYDfz\nDKdi7FZOUXUKpL8gQVXS6bi2B3hq+EyVIfObYauabmeGVUJq8tpUnOEiN1+qvxQv\nkMwjYQz/0lXXQVPczHVZQvBNccP80S+drQP4JlMfRE2riP2EXTOKgWcViR28W1a8\nBCbXn78RmEYNohssQER3/ss=\n-----END PRIVATE KEY-----\n",
+  "client_email": "[email protected]",
+  "client_id": "104264257508211128690",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/bronze-layer%40carteya-432304.iam.gserviceaccount.com",
+  "universe_domain": "googleapis.com"
+}

requirements.txt

@@ -1,3 +0,0 @@
-huggingface_hub==0.25.2
-requests==2.31.0
-python-dotenv==1.0.0

run.py

@@ -0,0 +1,10 @@
+import uvicorn
+
+if __name__ == "__main__":
+    uvicorn.run(
+        "backend.server:app",
+        host="0.0.0.0",
+        port=8000,
+        reload=True,
+        access_log=True
+    )

setup.py

@@ -0,0 +1,19 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="product_recommender",
+    version="0.1.0",
+    packages=find_packages(),
+    install_requires=[
+        "fastapi>=0.103.1",
+        "uvicorn>=0.23.2",
+        "sentence-transformers>=2.5.0",
+        "scikit-learn>=1.4.0",
+        "python-dotenv>=1.0.0",
+        "google-cloud-bigquery>=3.13.0",
+        "huggingface-hub>=0.25.2",
+        "pydantic>=2.5.0",
+        "pydantic-settings>=2.1.0",
+    ],
+    python_requires=">=3.9",
+)