performance

app.py

@@ -9,21 +9,23 @@ import pickle
 import numpy as np
 import requests  # For llama.cpp server calls
 from datetime import datetime
-from fastapi import FastAPI, WebSocket, WebSocketDisconnect
-from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect, BackgroundTasks, Request
+from langchain_core.messages import AIMessage
 from langgraph.graph import StateGraph, START, END
 import faiss
 from sentence_transformers import SentenceTransformer
 from tools import extract_json_from_response, apply_filters_partial, rule_based_extract, structured_property_data, estateKeywords, sendTokenViaSocket
-from langchain_core.prompts import ChatPromptTemplate
 from langchain_core.tools import tool
-from langchain_core.callbacks import StreamingStdOutCallbackHandler, CallbackManager
+from langchain_core.callbacks import StreamingStdOutCallbackHandler
 from langchain_core.callbacks.base import BaseCallbackHandler
 
 import os
 from fastapi.responses import PlainTextResponse
-from fastapi import FastAPI, Request
 from fastapi.staticfiles import StaticFiles
+from functools import lru_cache
+from contextlib import asynccontextmanager
+
+
 # ------------------------ Model Inference Wrapper ------------------------
 
 class ChatQwen:
@@ -48,6 +50,7 @@ class ChatQwen:
         self.max_new_tokens = max_new_tokens
         self.callbacks = callbacks
         self.use_server = use_server
+        self.is_hf_space = os.environ.get('SPACE_ID') is not None
 
         if self.use_server:
             # Use remote llama.cpp server – provide its URL.
@@ -57,90 +60,164 @@ class ChatQwen:
             if not model_path:
                 raise ValueError("Local mode requires a valid model_path to the gguf file.")
             from llama_cpp import Llama  # assumes llama-cpp-python is installed
-            self.model = Llama(
-                model_path=model_path,
-                temperature=self.temperature,
-                # n_ctx=512,
-                n_ctx=8192,
-                n_threads=4,  # Adjust as needed
-                batch_size=512,
-                verbose=False,
-            )
-
-    def build_prompt(self, messages: list) -> str:
-        """Build Qwen-compatible prompt with special tokens."""
-        prompt = ""
-        for msg in messages:
-            role = msg["role"]
-            content = msg["content"]
-            if role == "system":
-                prompt += f"<|im_start|>system\n{content}<|im_end|>\n"
-            elif role == "user":
-                prompt += f"<|im_start|>user\n{content}<|im_end|>\n"
-            elif role == "assistant":
-                prompt += f"<|im_start|>assistant\n{content}<|im_end|>\n"
-        prompt += "<|im_start|>assistant\n"
-        return prompt
-
-    def generate_text(self, messages: list) -> str:
-        prompt = self.build_prompt(messages)
-        stop_tokens = ["<|im_end|>", "\n"]  # Qwen's stop sequences
-        
-        if self.use_server:
-            payload = {
-                "prompt": prompt,
-                "max_tokens": self.max_new_tokens,
-                "temperature": self.temperature,
-                "stream": self.streaming,
-                "stop": stop_tokens  # Add stop tokens to server request
-            }
-            if self.streaming:
-                response = requests.post(f"{self.server_url}/generate", json=payload, stream=True)
-                generated_text = ""
-                for line in response.iter_lines():
-                    if line:
-                        token = line.decode("utf-8")
-                        # Check for stop tokens in stream
-                        if any(stop in token for stop in stop_tokens):
-                            break
-                        generated_text += token
-                        if self.callbacks:
-                            for callback in self.callbacks:
-                                callback.on_llm_new_token(token)
-                return generated_text
-            else:
-                response = requests.post(f"{self.server_url}/generate", json=payload)
-                return response.json().get("generated_text", "")
-        else:
-            # Local llama.cpp inference
-            if self.streaming:
-                stream = self.model.create_completion(
-                    prompt=prompt,
-                    max_tokens=self.max_new_tokens,
+            # self.model = Llama(
+            #     model_path=model_path,
+            #     temperature=self.temperature,
+            #     # n_ctx=512,
+            #     n_ctx=8192,
+            #     n_threads=4,  # Adjust as needed
+            #     batch_size=512,
+            #     verbose=False,
+            # )
+            # Update Llama initialization:
+            if self.is_hf_space:
+                self.model = Llama(
+                    model_path=model_path,
                     temperature=self.temperature,
-                    stream=True,
-                    stop=stop_tokens
+                    n_ctx=1024,  # Reduced from 8192
+                    n_threads=2,  # Never exceed 2 threads on free tier
+                    n_batch=128,  # Smaller batch size for low RAM
+                    use_mmap=True,  # Essential for memory mapping
+                    use_mlock=False,  # Disable memory locking
+                    low_vram=True,  # Special low-memory mode
+                    vocab_only=False,
+                    n_gqa=2,  # Grouped-query attention for 1.5B model
+                    rope_freq_base=10000,
+                    logits_all=False
                 )
-                generated_text = ""
-                for token_chunk in stream:
-                    token_text = token_chunk["choices"][0]["text"]
-                    # Stop early if we detect end token
-                    if any(stop in token_text for stop in stop_tokens):
-                        break
-                    generated_text += token_text
-                    if self.callbacks:
-                        for callback in self.callbacks:
-                            callback.on_llm_new_token(token_text)
-                return generated_text
             else:
-                result = self.model.create_completion(
-                    prompt=prompt,
-                    max_tokens=self.max_new_tokens,
+                self.model = Llama(
+                    model_path=model_path,
+                    n_gpu_layers=20,  # Offload 20 layers to GPU (adjust based on VRAM)
+                    n_threads=3,      # leave 1
+                    n_threads_batch=3,
+                    batch_size=256,
+                    main_gpu=0,       # Use first GPU
+                    use_mmap=True,
+                    use_mlock=False,
                     temperature=self.temperature,
-                    stop=stop_tokens
+                    n_ctx=2048,       # Reduced context for lower memory usage
+                    verbose=False
                 )
-                return result["choices"][0]["text"]
-
+            
+        if not self.use_server:
+            self.model.tokenize(b"Warmup")  # Pre-load model
+            self.model.create_completion("Warmup", max_tokens=1)
+
+    # def build_prompt(self, messages: list) -> str:
+    #     """Build Qwen-compatible prompt with special tokens."""
+    #     prompt = ""
+    #     for msg in messages:
+    #         role = msg["role"]
+    #         content = msg["content"]
+    #         if role == "system":
+    #             prompt += f"<|im_start|>system\n{content}<|im_end|>\n"
+    #         elif role == "user":
+    #             prompt += f"<|im_start|>user\n{content}<|im_end|>\n"
+    #         elif role == "assistant":
+    #             prompt += f"<|im_start|>assistant\n{content}<|im_end|>\n"
+    #     prompt += "<|im_start|>assistant\n"
+    #     return prompt
+
+    @lru_cache(maxsize=2)
+    def build_prompt(self, messages: list) -> str:
+        """Optimized prompt builder with string join"""
+        return "".join(
+            f"<|im_start|>{msg['role']}\n{msg['content']}<|im_end|>\n"
+            for msg in messages
+        ) + "<|im_start|>assistant\n"
+    
+    def generate_text(self, messages: list) -> str:
+        try:
+            prompt = self.build_prompt(messages)
+            stop_tokens = ["<|im_end|>", "\n"]  # Qwen's stop sequences
+            
+            if self.use_server:
+                payload = {
+                    "prompt": prompt,
+                    "max_tokens": self.max_new_tokens,
+                    "temperature": self.temperature,
+                    "stream": self.streaming,
+                    "stop": stop_tokens  # Add stop tokens to server request
+                }
+                if self.streaming:
+                    response = requests.post(f"{self.server_url}/generate", json=payload, stream=True)
+                    generated_text = ""
+                    for line in response.iter_lines():
+                        if line:
+                            token = line.decode("utf-8")
+                            # Check for stop tokens in stream
+                            if any(stop in token for stop in stop_tokens):
+                                break
+                            generated_text += token
+                            if self.callbacks:
+                                for callback in self.callbacks:
+                                    callback.on_llm_new_token(token)
+                    return generated_text
+                else:
+                    response = requests.post(f"{self.server_url}/generate", json=payload)
+                    return response.json().get("generated_text", "")
+            else:
+                # Local llama.cpp inference
+                if self.streaming:
+                    if self.is_hf_space:
+                        stream = self.model.create_completion(
+                            prompt=prompt,
+                            max_tokens=256,  # Reduced from 512
+                            temperature=0.3,
+                            stream=True,
+                            stop=stop_tokens,
+                            repeat_penalty=1.15,
+                            frequency_penalty=0.2,
+                            mirostat_mode=2,  # Better for low-resource
+                            mirostat_tau=3.0,
+                            mirostat_eta=0.1
+                        )
+                    else:
+                        stream = self.model.create_completion(
+                            prompt=prompt,
+                            max_tokens=self.max_new_tokens,
+                            temperature=self.temperature,
+                            stream=True,
+                            stop=stop_tokens,
+                            repeat_penalty=1.1,  # Reduce repetition for faster generation
+                            tfs_z=0.5            # Tail-free sampling for efficiency
+                        )
+                    
+                    
+                    generated_text = ""
+                    for token_chunk in stream:
+                        token_text = token_chunk["choices"][0]["text"]
+                        # Stop early if we detect end token
+                        if any(stop in token_text for stop in stop_tokens):
+                            break
+                        generated_text += token_text
+                        if self.callbacks:
+                            for callback in self.callbacks:
+                                callback.on_llm_new_token(token_text)
+                    return generated_text
+                else:
+                    result = self.model.create_completion(
+                        prompt=prompt,
+                        max_tokens=self.max_new_tokens,
+                        temperature=self.temperature,
+                        stop=stop_tokens
+                    )
+                    return result["choices"][0]["text"]
+        except Exception as e:
+            if "out of memory" in str(e).lower() and self.is_hf_space:
+                return self.fallback_generate(messages)
+
+    def fallback_generate(self, messages):
+        """Simpler generation for OOM situations"""
+        return self.model.create_completion(
+            prompt=self.build_prompt(messages),
+            max_tokens=128,
+            temperature=0.3,
+            stream=False,
+            stop=["<|im_end|>", "\n"]
+        )["choices"][0]["text"]
+        
     def invoke(self, messages: list, config: dict = None) -> AIMessage:
         config = config or {}
         callbacks = config.get("callbacks", self.callbacks)
@@ -184,6 +261,13 @@ llm = ChatQwen(
     # server_url="http://localhost:8000"  # Uncomment and set if using server mode.
 )
 
+llm_no_stream = ChatQwen(
+    temperature=0.3,
+    streaming=False,
+    use_server=False,
+    model_path=model_path,
+)
+
 # ------------------------ FAISS and Sentence Transformer Setup ------------------------
 
 index = faiss.read_index("./faiss.index")
@@ -212,9 +296,10 @@ general_query_prompt = make_system_prompt(
 # ------------------------ Tool Definitions ------------------------
 
 @tool
+@lru_cache(maxsize=50,typed=False)
 def extract_filters(query: str) -> dict:
     """Extract filters from the query."""
-    llm_local = ChatQwen(temperature=0.3, streaming=False, use_server=False, model_path=model_path)
+    # llm_local = ChatQwen(temperature=0.3, streaming=False, use_server=False, model_path=model_path)
     system = (
         "You are an expert in extracting filters from property-related queries. Your task is to extract and return only the keys explicitly mentioned in the query as a valid JSON object (starting with '{' and ending with '}'). Include only those keys that are directly present in the query.\n\n"
         "The possible keys are:\n"
@@ -259,7 +344,7 @@ def extract_filters(query: str) -> dict:
         {"role": "system", "content": system},
         {"role": "user", "content": human_str},
     ]
-    response = llm_local.invoke(messages=filter_prompt)
+    response = llm_no_stream.invoke(messages=filter_prompt)
     response_text = response.content if isinstance(response, AIMessage) else str(response)
     try:
         model_filters = extract_json_from_response(response_text)
@@ -274,13 +359,14 @@ def extract_filters(query: str) -> dict:
 
 
 @tool
+@lru_cache(maxsize=50,typed=False)
 def determine_route(query: str) -> dict:
     """Determine the route (search, suggest, detail, general, out_of_domain) for the query."""
     real_estate_keywords = estateKeywords
     pattern = re.compile("|".join(re.escape(keyword) for keyword in real_estate_keywords), re.IGNORECASE)
     positive_signal = bool(pattern.search(query))
 
-    llm_local = ChatQwen(temperature=0.3, streaming=False, use_server=False, model_path=model_path)
+    # llm_local = ChatQwen(temperature=0.3, streaming=False, use_server=False, model_path=model_path)
     transform_suggest_to_list = query.lower().replace("suggest ", "list ", -1)
     system = """
     Classify the user query as:
@@ -302,7 +388,7 @@ def determine_route(query: str) -> dict:
         {"role": "user", "content": human_str},
     ]
     
-    response = llm_local.invoke(messages=router_prompt)
+    response = llm_no_stream.invoke(messages=router_prompt)
     response_text = response.content if isinstance(response, AIMessage) else str(response)
     route_value = str(response_text).strip().lower()
 
@@ -410,8 +496,8 @@ def generate_response(state: dict) -> dict:
         messages.append({"role": "system", "content": "When responding, use only the provided property details."})
     
     # Add conversation history
-    # Truncate conversation history (last 4 exchanges)
-    truncated_history = state.get("messages", [])[-8:]  # Last 4 user+assistant pairs
+    # Truncate conversation history (last 6 exchanges)
+    truncated_history = state.get("messages", [])[-12:]  # Last 6 user+assistant pairs
     for msg in truncated_history:
         messages.append({"role": msg["role"], "content": msg["content"]})
 
@@ -462,23 +548,9 @@ def format_final_response(state: dict) -> dict:
         new_state["current_properties"] = state["current_properties"]
     
     
-    # print("state: ", json.dumps(new_state), "\n\n")
-    # Format the property details if available.
-    # if new_state.get("current_properties"):
     if state.get("route") in ["search", "suggest"] and new_state.get("current_properties"):
         formatted = structured_property_data(state=new_state)
-        
-        # for idx, prop in enumerate(new_state["current_properties"], 1):
-        #     cost = prop.get("totalCosts", "N/A")
-        #     cost_str = f"{cost:,}" if isinstance(cost, (int, float)) else cost
-        #     formatted.append(
-        #         f"{idx}. Type: {prop['propertyType']}, Cost: AED {cost_str}, "
-        #         f"Size: {prop.get('propertySize', 'N/A')}, Amenities: {', '.join(map(str, prop.get('amenities', []))) if prop.get('amenities') else 'N/A'}, "
-        #         f"Rental Yield: {prop.get('expectedRentalYield', 'N/A')}, "
-        #         f"Ownership: {prop.get('ownershipType', 'N/A')}\n"
-        #     )
         aggregated_response = "Here are the property details:\n" + "\n".join(formatted)
-        # print(aggregated_response)
         
         connection_id = state.get("connection_id")
         loop = state.get("loop")
@@ -727,6 +799,8 @@ async def websocket_endpoint(websocket: WebSocket):
             del conversation_managers[connection_id]
         manager_socket.disconnect(connection_id)
 
+
+
 @app.post("/query")
 async def post_query(query: str):
     conv_manager = ConversationManager()
@@ -761,3 +835,17 @@ async def check_model_middleware(request: Request, call_next):
 @app.get("/")
 async def home():
     return PlainTextResponse("Space is running. Model ready!")
+
+
+# async def clear_cache_periodically(seconds: int = 3600):
+#     while True:
+#         await asyncio.sleep(seconds)
+#         extract_filters.cache_clear()
+#         determine_route.cache_clear()
+#         ChatQwen.build_prompt.cache_clear()
+#         print("Cache cleared")
+
+# @app.on_event("startup")
+# async def startup_event():
+#     background_tasks = BackgroundTasks()
+#     background_tasks.add_task(clear_cache_periodically, 3600)  # Clear every hour

backup/app-backup.py

@@ -0,0 +1,765 @@
+import uuid
+import threading
+import asyncio
+import json
+import re
+import random
+import time
+import pickle
+import numpy as np
+import requests  # For llama.cpp server calls
+from datetime import datetime
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect
+from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
+from langgraph.graph import StateGraph, START, END
+import faiss
+from sentence_transformers import SentenceTransformer
+from tools import extract_json_from_response, apply_filters_partial, rule_based_extract, structured_property_data, estateKeywords, sendTokenViaSocket
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.tools import tool
+from langchain_core.callbacks import StreamingStdOutCallbackHandler, CallbackManager
+from langchain_core.callbacks.base import BaseCallbackHandler
+
+import os
+from fastapi.responses import PlainTextResponse
+from fastapi import FastAPI, Request
+from fastapi.staticfiles import StaticFiles
+# ------------------------ Model Inference Wrapper ------------------------
+
+class ChatQwen:
+    """
+    A chat wrapper for Qwen using llama.cpp.
+    This class can work in two modes:
+      - Local: Using a llama-cpp-python binding (gguf model file loaded locally).
+      - Server: Calling a remote llama.cpp server endpoint.
+    """
+    def __init__(
+        self,
+        temperature=0.3,
+        streaming=False,
+        max_new_tokens=512,
+        callbacks=None,
+        use_server=False,
+        model_path: str = None,
+        server_url: str = None
+    ):
+        self.temperature = temperature
+        self.streaming = streaming
+        self.max_new_tokens = max_new_tokens
+        self.callbacks = callbacks
+        self.use_server = use_server
+
+        if self.use_server:
+            # Use remote llama.cpp server – provide its URL.
+            self.server_url = server_url or "http://localhost:8000"
+        else:
+            # For local inference, a model_path must be provided.
+            if not model_path:
+                raise ValueError("Local mode requires a valid model_path to the gguf file.")
+            from llama_cpp import Llama  # assumes llama-cpp-python is installed
+            self.model = Llama(
+                model_path=model_path,
+                temperature=self.temperature,
+                # n_ctx=512,
+                n_ctx=8192,
+                n_threads=4,  # Adjust as needed
+                batch_size=512,
+                verbose=False,
+            )
+
+    def build_prompt(self, messages: list) -> str:
+        """Build Qwen-compatible prompt with special tokens."""
+        prompt = ""
+        for msg in messages:
+            role = msg["role"]
+            content = msg["content"]
+            if role == "system":
+                prompt += f"<|im_start|>system\n{content}<|im_end|>\n"
+            elif role == "user":
+                prompt += f"<|im_start|>user\n{content}<|im_end|>\n"
+            elif role == "assistant":
+                prompt += f"<|im_start|>assistant\n{content}<|im_end|>\n"
+        prompt += "<|im_start|>assistant\n"
+        return prompt
+
+    def generate_text(self, messages: list) -> str:
+        prompt = self.build_prompt(messages)
+        stop_tokens = ["<|im_end|>", "\n"]  # Qwen's stop sequences
+        
+        if self.use_server:
+            payload = {
+                "prompt": prompt,
+                "max_tokens": self.max_new_tokens,
+                "temperature": self.temperature,
+                "stream": self.streaming,
+                "stop": stop_tokens  # Add stop tokens to server request
+            }
+            if self.streaming:
+                response = requests.post(f"{self.server_url}/generate", json=payload, stream=True)
+                generated_text = ""
+                for line in response.iter_lines():
+                    if line:
+                        token = line.decode("utf-8")
+                        # Check for stop tokens in stream
+                        if any(stop in token for stop in stop_tokens):
+                            break
+                        generated_text += token
+                        if self.callbacks:
+                            for callback in self.callbacks:
+                                callback.on_llm_new_token(token)
+                return generated_text
+            else:
+                response = requests.post(f"{self.server_url}/generate", json=payload)
+                return response.json().get("generated_text", "")
+        else:
+            # Local llama.cpp inference
+            if self.streaming:
+                stream = self.model.create_completion(
+                    prompt=prompt,
+                    max_tokens=self.max_new_tokens,
+                    temperature=self.temperature,
+                    stream=True,
+                    stop=stop_tokens
+                )
+                generated_text = ""
+                for token_chunk in stream:
+                    token_text = token_chunk["choices"][0]["text"]
+                    # Stop early if we detect end token
+                    if any(stop in token_text for stop in stop_tokens):
+                        break
+                    generated_text += token_text
+                    if self.callbacks:
+                        for callback in self.callbacks:
+                            callback.on_llm_new_token(token_text)
+                return generated_text
+            else:
+                result = self.model.create_completion(
+                    prompt=prompt,
+                    max_tokens=self.max_new_tokens,
+                    temperature=self.temperature,
+                    stop=stop_tokens
+                )
+                return result["choices"][0]["text"]
+
+    def invoke(self, messages: list, config: dict = None) -> AIMessage:
+        config = config or {}
+        callbacks = config.get("callbacks", self.callbacks)
+        original_callbacks = self.callbacks
+        self.callbacks = callbacks
+
+        output_text = self.generate_text(messages)
+        self.callbacks = original_callbacks
+
+        # In streaming mode we return an empty content as tokens are being sent via callbacks.
+        if self.streaming:
+            return AIMessage(content="")
+        else:
+            return AIMessage(content=output_text)
+
+    def __call__(self, messages: list) -> AIMessage:
+        return self.invoke(messages)
+
+# ------------------------ Callback for WebSocket Streaming ------------------------
+
+class WebSocketStreamingCallbackHandler(BaseCallbackHandler):
+    def __init__(self, connection_id: str, loop):
+        self.connection_id = connection_id
+        self.loop = loop
+
+    def on_llm_new_token(self, token: str, **kwargs):
+        asyncio.run_coroutine_threadsafe(
+            manager_socket.send_message(self.connection_id, token),
+            self.loop
+        )
+
+# ------------------------ Instantiate the LLM ------------------------
+# Choose one mode: local (set use_server=False) or server (set use_server=True).
+model_path="qwen2.5-1.5b-instruct-q4_k_m.gguf"
+llm = ChatQwen(
+    temperature=0.3,
+    streaming=True,
+    max_new_tokens=512,
+    use_server=False,
+    model_path=model_path,
+    # server_url="http://localhost:8000"  # Uncomment and set if using server mode.
+)
+
+# ------------------------ FAISS and Sentence Transformer Setup ------------------------
+
+index = faiss.read_index("./faiss.index")
+with open("./metadata.pkl", "rb") as f:
+    docs = pickle.load(f)
+st_model = SentenceTransformer('all-MiniLM-L6-v2')
+
+def make_system_prompt(suffix: str) -> str:
+    return (
+        "You are EstateGuru, a real estate expert developed by Abhishek Pathak at SwavishTek. "
+        "Your role is to help customers buy properties using only the provided data—do not invent any details. "
+        "The default currency is AED; if a query mentions another currency, convert the amount to AED "
+        "(for example, convert $10k to 36726.50 AED and $1 to 3.67 AED). "
+        "If a customer is interested in a property or needs to contact an agent, instruct them to call +91 8766268285. "
+        "Keep your answers short, clear, and concise."
+        f"\n{suffix}"
+    )
+
+general_query_prompt = make_system_prompt(
+    "You are EstateGuru, a helpful real estate assistant. "
+    "Please respond only in English. "
+    "Convert any prices to USD before answering. "
+    "Provide a brief, direct answer without extra details."
+)
+
+# ------------------------ Tool Definitions ------------------------
+
+@tool
+def extract_filters(query: str) -> dict:
+    """Extract filters from the query."""
+    llm_local = ChatQwen(temperature=0.3, streaming=False, use_server=False, model_path=model_path)
+    system = (
+        "You are an expert in extracting filters from property-related queries. Your task is to extract and return only the keys explicitly mentioned in the query as a valid JSON object (starting with '{' and ending with '}'). Include only those keys that are directly present in the query.\n\n"
+        "The possible keys are:\n"
+        "  - 'projectName': The name of the project.\n"
+        "  - 'developerName': The developer's name.\n"
+        "  - 'relationshipManager': The relationship manager.\n"
+        "  - 'propertyAddress': The property address.\n"
+        "  - 'surroundingArea': The area or nearby landmarks.\n"
+        "  - 'propertyType': The type or configuration of the property.\n"
+        "  - 'amenities': Any amenities mentioned.\n"
+        "  - 'coveredParking': Parking availability.\n"
+        "  - 'petRules': Pet policies.\n"
+        "  - 'security': Security details.\n"
+        "  - 'occupancyRate': Occupancy information.\n"
+        "  - 'constructionImpact': Construction or its impact.\n"
+        "  - 'propertySize': Size of the property.\n"
+        "  - 'propertyView': View details.\n"
+        "  - 'propertyCondition': Condition of the property.\n"
+        "  - 'serviceCharges': Service or maintenance charges.\n"
+        "  - 'ownershipType': Ownership type.\n"
+        "  - 'totalCosts': A cost threshold or cost amount.\n"
+        "  - 'paymentPlans': Payment or financing plans.\n"
+        "  - 'expectedRentalYield': Expected rental yield.\n"
+        "  - 'rentalHistory': Rental history.\n"
+        "  - 'shortTermRentals': Short-term rental information.\n"
+        "  - 'resalePotential': Resale potential.\n"
+        "  - 'uniqueId': A unique identifier.\n\n"
+        "Important instructions regarding cost thresholds:\n"
+        "  - If the query contains phrases like 'under 10k', 'below 2m', or 'less than 5k', interpret these as cost thresholds.\n"
+        "  - Convert any shorthand cost values to pure numbers (for example, '10k' becomes 10000, '2m' becomes 2000000) and assign them to the key 'totalCosts'.\n"
+        "  - Do not use 'propertySize' for cost thresholds.\n\n"
+        "  - Default currency is AED, if user query have different currency symbol then convert to equivalent AED amount (eg. $10k becomes 36726.50, $1 becomes 3.67).\n\n"
+        "Example:\n"
+        "  For the query: \"properties near dubai mall under 43k\"\n"
+        "  The expected output should be:\n"
+        "    { \"surroundingArea\": \"dubai mall\", \"totalCosts\": 43000 }\n\n"
+        "Return ONLY a valid JSON object with the extracted keys and their corresponding values, with no additional text."
+    )
+
+    human_str = f"Here is the query:\n{query}"
+    filter_prompt = [
+        {"role": "system", "content": system},
+        {"role": "user", "content": human_str},
+    ]
+    response = llm_local.invoke(messages=filter_prompt)
+    response_text = response.content if isinstance(response, AIMessage) else str(response)
+    try:
+        model_filters = extract_json_from_response(response_text)
+    except Exception as e:
+        print(f"JSON parsing error: {e}")
+        model_filters = {}
+    rule_filters = rule_based_extract(query)
+    print("Rule-based extraction:", rule_filters)
+    final_filters = {**model_filters, **rule_filters}
+    print("Final extraction:", final_filters)
+    return {"filters": final_filters}
+
+
+@tool
+def determine_route(query: str) -> dict:
+    """Determine the route (search, suggest, detail, general, out_of_domain) for the query."""
+    real_estate_keywords = estateKeywords
+    pattern = re.compile("|".join(re.escape(keyword) for keyword in real_estate_keywords), re.IGNORECASE)
+    positive_signal = bool(pattern.search(query))
+
+    llm_local = ChatQwen(temperature=0.3, streaming=False, use_server=False, model_path=model_path)
+    transform_suggest_to_list = query.lower().replace("suggest ", "list ", -1)
+    system = """
+    Classify the user query as:
+    
+    - **"search"**: if it requests property listings with specific filters (e.g., location, price, property type like "2bhk", service charges, pet policies, etc.).
+    - **"suggest"**: if it asks for property suggestions without filters.
+    - **"detail"**: if it is asking for more information about a previously provided property (for example, "tell me more about property 5" or "I want more information regarding 4BHK").
+    - **"general"**: for all other real estate-related questions.
+    - **"out_of_domain"**: if the query is not related to real estate (for example, tourist attractions, restaurants, etc.).
+    
+    Keep in mind that queries mentioning terms like "service charge", "allow pets", "pet rules", etc., are considered real estate queries.
+    When user asks about you (for example, "who you are", "who made you" etc.) consider as general.
+    
+    Return only the keyword: search, suggest, detail, general, or out_of_domain.
+    """
+    human_str = f"Here is the query:\n{transform_suggest_to_list}"
+    router_prompt = [
+        {"role": "system", "content": system},
+        {"role": "user", "content": human_str},
+    ]
+    
+    response = llm_local.invoke(messages=router_prompt)
+    response_text = response.content if isinstance(response, AIMessage) else str(response)
+    route_value = str(response_text).strip().lower()
+
+    # --- NEW: Force 'detail' if query explicitly mentions a specific property (e.g., "property 2") ---
+    property_detail_pattern = re.compile(r"property\s+\d+", re.IGNORECASE)
+    if property_detail_pattern.search(query):
+        route_value = "detail"
+
+    # Fallback override if query appears detailed.
+    detail_phrases = [
+        "more information", "tell me more", "more details", "give me more details",
+        "i need more details", "can you provide more details", "additional details",
+        "further information", "expand on that", "explain further", "elaborate more",
+        "more specifics", "i want to know more", "could you elaborate", "need more info",
+        "provide more details", "detail it further", "in-depth information", "break it down further",
+        "further explanation", "property 1", "property1", "first property", "about the 2nd", "regarding number 3"
+    ]
+    if any(phrase in query.lower() for phrase in detail_phrases):
+        route_value = "detail"
+
+    if route_value not in {"search", "suggest", "detail", "general", "out_of_domain"}:
+        route_value = "general"
+    if route_value == "out_of_domain" and positive_signal:
+        route_value = "general"
+    if route_value == "out_of_domain":
+        route_value = "general" if positive_signal else "out_of_domain"
+        
+    return {"route": route_value}
+
+# ------------------------ Workflow Setup ------------------------
+
+workflow = StateGraph(state_schema=dict)
+
+def route_query(state: dict) -> dict:
+    new_state = state.copy()
+    try:
+        new_state["route"] = determine_route.invoke(new_state.get("query", "")).get("route", "general")
+        print(new_state["route"])
+    except Exception as e:
+        print(f"Routing error: {e}")
+        new_state["route"] = "general"
+    return new_state
+
+def hybrid_extract(state: dict) -> dict:
+    new_state = state.copy()
+    new_state["filters"] = extract_filters.invoke(new_state.get("query", "")).get("filters", {})
+    return new_state
+
+def search_faiss(state: dict) -> dict:
+    new_state = state.copy()
+    # Preserve previous properties until new ones are fetched:
+    new_state.setdefault("current_properties", state.get("current_properties", []))
+    query_embedding = st_model.encode([state["query"]])
+    _, indices = index.search(query_embedding.astype(np.float32), 5)
+    new_state["faiss_results"] = [docs[idx] for idx in indices[0] if idx < len(docs)]
+    return new_state
+
+def apply_filters(state: dict) -> dict:
+    new_state = state.copy()
+    new_state["final_results"] = apply_filters_partial(state["faiss_results"], state.get("filters", {}))
+    if(len(new_state["final_results"]) == 0):
+        new_state["response"] = "Sorry, There is no result found :("
+        new_state["route"] = "general"
+    return new_state
+
+def suggest_properties(state: dict) -> dict:
+    new_state = state.copy()
+    new_state["suggestions"] = random.sample(docs, 5)
+    # Explicitly update current_properties only when new listings are fetched
+    new_state["current_properties"] = new_state["suggestions"]
+    if(len(new_state["suggestions"]) == 0):
+        new_state["response"] = "Sorry, There is no result found :("
+        new_state["route"] = "general"
+    return new_state
+
+def handle_out_of_domain(state: dict) -> dict:
+    new_state = state.copy()
+    new_state["response"] = "I only handle real estate inquiries. Please ask a question related to properties."
+    return new_state
+
+
+
+def generate_response(state: dict) -> dict:
+    new_state = state.copy()
+    messages = []
+    
+    # Add the general query prompt.
+    messages.append({"role": "system", "content": general_query_prompt})
+    
+    # For detail queries (specific property queries), add extra instructions.
+    if new_state.get("route", "general") == "detail":
+        messages.append({
+            "role": "system",
+            "content": (
+                "The user is asking about a specific property from the numbered list below. "
+                "Properties are listed as 1, 2, 3, etc. Use ONLY the corresponding property details. "
+                "For example, if the user says 'property 2', respond using only the details from the second entry. Never invent data."
+            )
+        })
+        
+    if new_state.get("current_properties"):
+        # Format properties with indices starting at 1
+        property_context = format_property_data_with_indices(new_state["current_properties"])
+        messages.append({"role": "system", "content": "Available Properties:\n" + property_context})
+        messages.append({"role": "system", "content": "When responding, use only the provided property details."})
+    
+    # Add conversation history
+    # Truncate conversation history (last 6 exchanges)
+    truncated_history = state.get("messages", [])[-12:]  # Last 6 user+assistant pairs
+    for msg in truncated_history:
+        messages.append({"role": msg["role"], "content": msg["content"]})
+
+    connection_id = state.get("connection_id")
+    loop = state.get("loop")
+    if connection_id and loop:
+        print("Using WebSocket streaming")
+        callback_manager = [WebSocketStreamingCallbackHandler(connection_id, loop)]
+        _ = llm.invoke(
+            messages,
+            config={"callbacks": callback_manager}
+        )
+        new_state["response"] = ""
+    else:
+        callback_manager = [StreamingStdOutCallbackHandler()]
+        response = llm.invoke(
+            messages,
+            config={"callbacks": callback_manager}
+        )
+        new_state["response"] = response.content if isinstance(response, AIMessage) else str(response)
+    
+    return new_state
+
+
+def format_property_data_with_indices(properties: list) -> str:
+    formatted = []
+    for idx, prop in enumerate(properties, 1):
+        cost = prop.get("totalCosts", "N/A")
+        cost_str = f"{cost:,}" if isinstance(cost, (int, float)) else cost
+        formatted.append(
+            f"{idx}. Type: {prop['propertyType']}, Cost: AED {cost_str}, "
+            f"Size: {prop.get('propertySize', 'N/A')}, Amenities: {', '.join(prop.get('amenities', []))}, "
+            f"Rental Yield: {prop.get('expectedRentalYield', 'N/A')}, "
+            f"Ownership: {prop.get('ownershipType', 'N/A')}"
+        )
+    return "\n".join(formatted)
+
+
+def format_final_response(state: dict) -> dict:
+    new_state = state.copy()
+    
+    if state.get("route") in ["search", "suggest"]:
+        if "final_results" in state:
+            new_state["current_properties"] = state["final_results"]
+        elif "suggestions" in state:
+            new_state["current_properties"] = state["suggestions"]
+    elif "current_properties" in new_state:
+        new_state["current_properties"] = state["current_properties"]
+    
+    
+    # print("state: ", json.dumps(new_state), "\n\n")
+    # Format the property details if available.
+    # if new_state.get("current_properties"):
+    if state.get("route") in ["search", "suggest"] and new_state.get("current_properties"):
+        formatted = structured_property_data(state=new_state)
+        
+        # for idx, prop in enumerate(new_state["current_properties"], 1):
+        #     cost = prop.get("totalCosts", "N/A")
+        #     cost_str = f"{cost:,}" if isinstance(cost, (int, float)) else cost
+        #     formatted.append(
+        #         f"{idx}. Type: {prop['propertyType']}, Cost: AED {cost_str}, "
+        #         f"Size: {prop.get('propertySize', 'N/A')}, Amenities: {', '.join(map(str, prop.get('amenities', []))) if prop.get('amenities') else 'N/A'}, "
+        #         f"Rental Yield: {prop.get('expectedRentalYield', 'N/A')}, "
+        #         f"Ownership: {prop.get('ownershipType', 'N/A')}\n"
+        #     )
+        aggregated_response = "Here are the property details:\n" + "\n".join(formatted)
+        # print(aggregated_response)
+        
+        connection_id = state.get("connection_id")
+        loop = state.get("loop")
+        if connection_id and loop:
+            import time
+            tokens = aggregated_response.split(" ")
+            for token in tokens:
+                asyncio.run_coroutine_threadsafe(
+                    manager_socket.send_message(connection_id, token + " "),
+                    loop
+                )
+                time.sleep(0.05)
+            new_state["response"] = ""
+        else:
+            new_state["response"] = aggregated_response
+    elif "response" in new_state:
+        connection_id = state.get("connection_id")
+        loop = state.get("loop")
+        if connection_id and loop:
+            import time
+            tokens = str(new_state["response"]).split(" ")
+            for token in tokens:
+                asyncio.run_coroutine_threadsafe(
+                    manager_socket.send_message(connection_id, token + " "),
+                    loop
+                )
+                time.sleep(0.05)
+        new_state["response"] = str(new_state["response"])
+        
+    return new_state
+
+
+
+nodes = [
+    ("route_query", route_query),
+    ("hybrid_extract", hybrid_extract),
+    ("faiss_search", search_faiss),
+    ("apply_filters", apply_filters),
+    ("suggest_properties", suggest_properties),
+    ("handle_out_of_domain", handle_out_of_domain),
+    ("generate_response", generate_response),
+    ("format_response", format_final_response)
+]
+
+for name, node in nodes:
+    workflow.add_node(name, node)
+
+workflow.add_edge(START, "route_query")
+workflow.add_conditional_edges(
+    "route_query",
+    lambda state: state.get("route", "general"),
+    {
+        "search": "hybrid_extract", 
+        "suggest": "suggest_properties", 
+        "detail": "generate_response", 
+        "general": "generate_response", 
+        "out_of_domain": "handle_out_of_domain"
+    }
+)
+workflow.add_edge("hybrid_extract", "faiss_search")
+workflow.add_edge("faiss_search", "apply_filters")
+workflow.add_edge("apply_filters", "format_response")
+workflow.add_edge("suggest_properties", "format_response")
+workflow.add_edge("generate_response", "format_response")
+workflow.add_edge("handle_out_of_domain", "format_response")
+workflow.add_edge("format_response", END)
+
+workflow_app = workflow.compile()
+
+# ------------------------ Conversation Manager ------------------------
+
+class ConversationManager:
+    def __init__(self):
+        # Each connection gets its own conversation history and state.
+        self.conversation_history = []
+        # current_properties stores the current property listing.
+        self.current_properties = []
+
+    def _add_message(self, role: str, content: str):
+        self.conversation_history.append({
+            "role": role,
+            "content": content,
+            "timestamp": datetime.now().isoformat()
+        })
+
+    def process_query(self, query: str) -> str:
+        # For greeting messages, reset history/state. // post request
+        if query.strip().lower() in {"hi", "hello", "hey"}:
+            self.conversation_history = []
+            self.current_properties = []
+            greeting_response = "Hello! How can I assist you today with your real estate inquiries?"
+            self._add_message("assistant", greeting_response)
+            return greeting_response
+
+        try:
+            self._add_message("user", query)
+            initial_state = {
+                "messages": self.conversation_history.copy(),
+                "query": query,
+                "route": "general",
+                "filters": {},
+                "current_properties": self.current_properties
+            }
+            for event in workflow_app.stream(initial_state, stream_mode="values"):
+                final_state = event
+            # Only update property listings if a new listing is fetched
+            # if 'final_results' in final_state:
+            #     self.current_properties = final_state['final_results']
+            # elif 'suggestions' in final_state:
+            #     self.current_properties = final_state['suggestions']
+            self.current_properties = final_state.get("current_properties", [])
+            
+            if final_state.get("route") == "general":
+                response_text = final_state.get("response", "")
+                self._add_message("assistant", response_text)
+                return response_text
+            else:
+                response = final_state.get("response", "I couldn't process that request.")
+                self._add_message("assistant", response)
+                return response
+        except Exception as e:
+            print(f"Processing error: {e}")
+            return "Sorry, I encountered an error processing your request."
+
+
+
+conversation_managers = {}
+
+# ------------------------ FastAPI Backend with WebSockets ------------------------
+
+app = FastAPI()
+
+class ConnectionManager:
+    def __init__(self):
+        self.active_connections = {}
+
+    async def connect(self, websocket: WebSocket):
+        await websocket.accept()
+        connection_id = str(uuid.uuid4())
+        self.active_connections[connection_id] = websocket
+        print(f"New connection: {connection_id}")
+        return connection_id
+
+    def disconnect(self, connection_id: str):
+        if connection_id in self.active_connections:
+            del self.active_connections[connection_id]
+            print(f"Disconnected: {connection_id}")
+
+    async def send_message(self, connection_id: str, message: str):
+        websocket = self.active_connections.get(connection_id)
+        if websocket:
+            await websocket.send_text(message)
+
+manager_socket = ConnectionManager()
+
+def stream_query(query: str, connection_id: str, loop):
+    conv_manager = conversation_managers.get(connection_id)
+    if conv_manager is None:
+        print(f"No conversation manager found for connection {connection_id}")
+        return
+
+    if query.strip().lower() in {"hi", "hello", "hey"}:
+        conv_manager.conversation_history = []
+        conv_manager.current_properties = []
+        greeting_response = "Hello! How can I assist you today with your real estate inquiries?"
+        conv_manager._add_message("assistant", greeting_response)
+        sendTokenViaSocket(
+            state={"connection_id": connection_id, "loop": loop},
+            manager_socket=manager_socket,
+            message=greeting_response
+        )
+        # asyncio.run_coroutine_threadsafe(
+        #     manager_socket.send_message(connection_id, greeting_response),
+        #     loop
+        # )
+        return
+
+    conv_manager._add_message("user", query)
+    initial_state = {
+        "messages": conv_manager.conversation_history.copy(),
+        "query": query,
+        "route": "general",
+        "filters": {},
+        "current_properties": conv_manager.current_properties,
+        "connection_id": connection_id,
+        "loop": loop
+    }
+    # try:
+    #     workflow_app.invoke(initial_state)
+    # except Exception as e:
+    #     error_msg = f"Error processing query: {str(e)}"
+    #     asyncio.run_coroutine_threadsafe(
+    #         manager_socket.send_message(connection_id, error_msg),
+    #         loop
+    #     )
+    try:
+        # Capture all states during execution
+        # final_state = None
+        # for event in workflow_app.stream(initial_state, stream_mode="values"):
+        #     final_state = event
+        
+        # # Update conversation manager with final state
+        # if final_state:
+        #     conv_manager.current_properties = final_state.get("current_properties", [])
+        #     if final_state.get("response"):
+        #         conv_manager._add_message("assistant", final_state["response"])
+        final_state = None
+        for event in workflow_app.stream(initial_state, stream_mode="values"):
+            final_state = event
+        
+        if final_state:
+            # Always update current_properties from final state
+            conv_manager.current_properties = final_state.get("current_properties", [])
+            # Keep conversation history bounded
+            conv_manager.conversation_history = conv_manager.conversation_history[-12:]  # Last 6 exchanges
+            
+    except Exception as e:
+        error_msg = f"Error processing query: {str(e)}"
+        asyncio.run_coroutine_threadsafe(
+            manager_socket.send_message(connection_id, error_msg),
+            loop
+        )
+        
+        
+
+@app.websocket("/ws")
+async def websocket_endpoint(websocket: WebSocket):
+    connection_id = await manager_socket.connect(websocket)
+    # Each connection maintains its own conversation manager.
+    conversation_managers[connection_id] = ConversationManager()
+    try:
+        while True:
+            query = await websocket.receive_text()
+            loop = asyncio.get_event_loop()
+            threading.Thread(
+                target=stream_query,
+                args=(query, connection_id, loop),
+                daemon=True
+            ).start()
+    except WebSocketDisconnect:
+        conv_manager = conversation_managers.get(connection_id)
+        if conv_manager:
+            filename = f"conversations/conversation_{connection_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
+            with open(filename, "w") as f:
+                json.dump(conv_manager.conversation_history, f, indent=4)
+            del conversation_managers[connection_id]
+        manager_socket.disconnect(connection_id)
+
+
+
+@app.post("/query")
+async def post_query(query: str):
+    conv_manager = ConversationManager()
+    response = conv_manager.process_query(query)
+    return {"response": response}
+
+
+
+
+model_url = "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF/resolve/main/qwen2.5-1.5b-instruct-q4_k_m.gguf"
+async def async_download():
+    import aiohttp
+    async with aiohttp.ClientSession() as session:
+        async with session.get(model_url) as response:
+            with open(model_path, "wb") as f:
+                while True:
+                    chunk = await response.content.read(1024)
+                    if not chunk:
+                        break
+                    f.write(chunk)
+
+@app.middleware("http")
+async def check_model_middleware(request: Request, call_next):
+    if not os.path.exists(model_path):
+        await async_download()
+        print("successfully downloaded")
+    else:
+        print("already downloaded")
+    return await call_next(request)
+
+
+@app.get("/")
+async def home():
+    return PlainTextResponse("Space is running. Model ready!")

index.html

@@ -49,6 +49,7 @@
   <script>
     // Create a WebSocket connection to your backend
     const ws = new WebSocket("ws://localhost:8000/ws");
+    // const ws = new WebSocket("wss://pathakdev10-estateguru.hf.space/ws");
 
     // This variable holds the current assistant message element for live updating.
     let currentAssistantMessageEl = null;