Add experimental keyword search

app.py

@@ -1,4 +1,5 @@
 from io import StringIO
+import bm25s
 import numpy as np
 import pandas as pd
 import requests
@@ -18,11 +19,15 @@ from dotenv import load_dotenv
 import warnings
 from fastapi import FastAPI, HTTPException
 from fastapi.middleware.cors import CORSMiddleware
+import nltk
+from nltk.stem import WordNetLemmatizer
 from fastapi.responses import FileResponse
 from fastapi.staticfiles import StaticFiles
 from pydantic import BaseModel
 from typing import Any, Dict, List, Literal, Optional
 
+from sklearn.preprocessing import MinMaxScaler
+nltk.download("wordnet")
 load_dotenv()
 
 warnings.filterwarnings("ignore")
@@ -221,6 +226,13 @@ class BatchDocResponse(BaseModel):
     missing: List[str]
     search_time: float
 
+class KeywordRequest2(BaseModel):
+    keywords: Optional[str] = ""
+    threshold: Optional[int] = 60
+    release: Optional[str] = None
+    working_group: Optional[str] = None
+    spec_type: Optional[Literal["TS", "TR"]] = None
+
 class KeywordRequest(BaseModel):
     keywords: Optional[str] = ""
     search_mode: Literal["quick", "deep"]
@@ -441,11 +453,99 @@ class SpecDocFinder:
 
 finder_tsg = TsgDocFinder()
 finder_spec = SpecDocFinder()
+lemmatizer = WordNetLemmatizer()
+
+if os.path.exists("bm25s.zip"):
+    with zipfile.ZipFile("bm25s.zip", 'r') as zip_ref:
+        zip_ref.extractall(".")
+    bm25_engine = bm25s.BM25.load("3gpp_bm25_docs", load_corpus=True)
 
 @app.get("/")
 async def main_menu():
     return FileResponse(os.path.join("templates", "index.html"))
 
+@app.post("/search-spec/experimental", response_model=KeywordResponse)
+def search_spec_bm25(request: KeywordRequest2):
+    start_time = time.time()
+    release = request.release
+    working_group = request.working_group
+    spec_type = request.spec_type
+    threshold = request.threshold
+    query = lemmatizer.lemmatize(request.keywords)
+
+    results_out = []
+    query_tokens = bm25s.tokenize(query)
+    results, scores = bm25_engine.retrieve(query_tokens, k=len(bm25_engine.corpus))
+
+    def calculate_boosted_score(metadata, score, query):
+        title = {lemmatizer.lemmatize(metadata['title']).lower()}
+        q = {query.lower()}
+        spec_id_presence = 0.5 if len(q & {metadata['id']}) > 0 else 0
+        booster = len(q & title) * 0.5
+        return score + spec_id_presence + booster
+
+    spec_scores = {}
+    spec_indices = {}
+    spec_details = {}
+
+    for i in range(results.shape[1]):
+        doc = results[0, i]
+        score = scores[0, i]
+        spec = doc["metadata"]["id"]
+
+        boosted_score = calculate_boosted_score(doc['metadata'], score, query)
+
+        if spec not in spec_scores or boosted_score > spec_scores[spec]:
+            spec_scores[spec] = boosted_score
+            spec_indices[spec] = i
+            spec_details[spec] = {
+                'original_score': score,
+                'boosted_score': boosted_score,
+                'doc': doc
+            }
+
+    def normalize_scores(scores_dict):
+        if not scores_dict:
+            return {}
+        
+        scores_array = np.array(list(scores_dict.values())).reshape(-1, 1)
+        scaler = MinMaxScaler()
+        normalized_scores = scaler.fit_transform(scores_array).flatten()
+        
+        normalized_dict = {}
+        for i, spec in enumerate(scores_dict.keys()):
+            normalized_dict[spec] = normalized_scores[i]
+        
+        return normalized_dict
+
+    normalized_scores = normalize_scores(spec_scores)
+
+    for spec in spec_details:
+        spec_details[spec]["normalized_score"] = normalized_scores[spec]
+
+    unique_specs = sorted(normalized_scores.keys(), key=lambda x: normalized_scores[x], reverse=True)
+
+    for rank, spec in enumerate(unique_specs, 1):
+        details = spec_details[spec]
+        metadata = details['doc']['metadata']
+        if metadata.get('version', None) is None or (release is not None and metadata["version"].split(".")[0] != str(release)):
+            continue
+        if metadata.get('type', None) is None or (spec_type is not None and metadata["type"] != spec_type):
+            continue
+        if metadata.get('working_group', None) is None or (working_group is not None and metadata["working_group"] != working_group):
+            continue
+        if details['normalized_score'] < threshold / 100:
+            break
+        results_out.append(metadata)
+    
+    if len(results_out) > 0:
+        return KeywordResponse(
+            results=results_out,
+            search_time=time.time() - start_time
+        )
+    else:
+        raise HTTPException(status_code=404, detail="Specifications not found")
+
 @app.post("/search-spec", response_model=KeywordResponse)
 def search_spec(request: KeywordRequest):
     start_time = time.time()
@@ -486,7 +586,7 @@ def search_spec(request: KeywordRequest):
                 if search_mode == "deep":
                     if docValid:
                         for chapter in list(doc.keys())[1:]:
-                            if "references" not in chapter.lower() and "void" not in chapter.lower() and "annexe" not in doc[chapter].lower():
+                            if "references" not in chapter.lower() and "void" not in chapter.lower() and "annex" not in doc[chapter].lower():
                                 if all(kw in caseSensitive(doc[chapter], booleanLowered) for kw in kws):
                                     put = True
                                     contents.append(chapter)    
@@ -496,7 +596,7 @@ def search_spec(request: KeywordRequest):
                 if search_mode == "deep":
                     if docValid:
                         for chapter in list(doc.keys())[1:]:
-                            if "references" not in chapter.lower() and "void" not in chapter.lower() and "annexe" not in doc[chapter].lower():
+                            if "references" not in chapter.lower() and "void" not in chapter.lower() and "annex" not in doc[chapter].lower():
                                 if any(kw in caseSensitive(doc[chapter], booleanLowered) for kw in kws):
                                     put = True
                                     contents.append(chapter)  

static/script.js

@@ -5,14 +5,18 @@ const dynamicTitle = document.getElementById("dynamicTitle");
 const singleModeBtn = document.getElementById('single-mode-btn');
 const batchModeBtn = document.getElementById('batch-mode-btn');
 const keywordModeBtn = document.getElementById("keyword-mode-btn");
+const expModeBtn = document.getElementById("exp-mode-btn");
 
-const singleInput = document.querySelector('.single-input');
-const batchInput = document.querySelector('.batch-input');
-const keywordSearchInput = document.querySelector(".keyword-input");
+const singleInputField = document.querySelector('.single-input');
+const batchInputField = document.querySelector('.batch-input');
+const keywordInputField = document.querySelector(".keyword-input");
+const expKeywordInputField = document.querySelector(".experimental-input");
 
 const docIdInput = document.getElementById('doc-id');
 const batchIdsInput = document.getElementById('batch-ids');
 const keywordInput = document.getElementById("keywords");
+const expKeywordInput = document.getElementById("exp-keywords")
+const thresholdInput = document.getElementById("threshold");
 
 const releaseFilter = document.querySelector("input[name=release]")
 const modeFilter = document.querySelector("select[name=mode]")
@@ -21,9 +25,14 @@ const workingGroupFilter = document.querySelector("select[name=working_group]")
 const caseSensitiveFilter = document.querySelector("input[name=case_sensitive]")
 const searchMode = document.querySelector("select[name=search_mode]")
 
+const releaseFilter2 = document.querySelector("input[name=release2]")
+const specTypeFilter2 = document.querySelector("select[name=spec_type2]")
+const workingGroupFilter2 = document.querySelector("select[name=working_group2]")
+
 const searchBtn = document.getElementById('search-btn');
 const batchSearchBtn = document.getElementById('batch-search-btn');
 const keywordSearchBtn = document.getElementById("keyword-search-btn");
+const expKeywordSearchBtn = document.getElementById("exp-search-btn");
 
 const loader = document.getElementById('loader');
 const resultsContainer = document.getElementById('results-container');
@@ -45,10 +54,12 @@ singleModeBtn.addEventListener('click', () => {
     singleModeBtn.classList.add('active');
     keywordModeBtn.classList.remove("active");
     batchModeBtn.classList.remove('active');
+    expModeBtn.classList.remove('active');
 
-    singleInput.style.display = 'block';
-    batchInput.style.display = 'none';
-    keywordSearchInput.style.display = "none";
+    singleInputField.style.display = 'block';
+    batchInputField.style.display = 'none';
+    keywordInputField.style.display = "none";
+    expKeywordInputField.style.display = "none";
 });
 
 batchModeBtn.addEventListener('click', () => {
@@ -58,10 +69,12 @@ batchModeBtn.addEventListener('click', () => {
     batchModeBtn.classList.add('active');
     keywordModeBtn.classList.remove("active");
     singleModeBtn.classList.remove('active');
+    expModeBtn.classList.remove('active');
 
-    batchInput.style.display = 'block';
-    keywordSearchInput.style.display = "none";
-    singleInput.style.display = 'none';
+    batchInputField.style.display = 'block';
+    keywordInputField.style.display = "none";
+    singleInputField.style.display = 'none';
+    expKeywordInputField.style.display = "none";
 });
 
 keywordModeBtn.addEventListener('click', () => {
@@ -71,10 +84,27 @@ keywordModeBtn.addEventListener('click', () => {
     keywordModeBtn.classList.add("active");
     singleModeBtn.classList.remove('active');
     batchModeBtn.classList.remove("active");
+    expModeBtn.classList.remove('active');
 
-    singleInput.style.display = "none";
-    batchInput.style.display = "none";
-    keywordSearchInput.style.display = "block";
+    singleInputField.style.display = "none";
+    batchInputField.style.display = "none";
+    expKeywordInputField.style.display = "none";
+    keywordInputField.style.display = "block";
+})
+
+expModeBtn.addEventListener('click', () => {
+    dynamicTitle.textContent = "[EXPERIMENTAL] Search 3GPP specifications";
+    dynamicDesc.textContent = "With keywords and filters, find all of 3GPP's specifications that matches your needs (with keywords, specification number, release or even working group (C1, S5, SP, CP: always the first letter of the group followed by the workgroup number)";
+
+    keywordModeBtn.classList.remove("active");
+    singleModeBtn.classList.remove('active');
+    batchModeBtn.classList.remove("active");
+    expModeBtn.classList.add('active');
+
+    singleInputField.style.display = "none";
+    batchInputField.style.display = "none";
+    expKeywordInputField.style.display = "block";
+    keywordInputField.style.display = "none";
 })
 
 document.getElementById('toggleFilters').onclick = function() {
@@ -82,6 +112,58 @@ document.getElementById('toggleFilters').onclick = function() {
   target.style.display = (target.style.display === 'none' || target.style.display === '') ? 'flex' : 'none';
 };
 
+document.getElementById('toggleFilters2').onclick = function() {
+  var target = document.getElementById('filtersForm2');
+  target.style.display = (target.style.display === 'none' || target.style.display === '') ? 'flex' : 'none';
+};
+
+expKeywordSearchBtn.addEventListener("click", async ()=>{
+    let keywords = expKeywordInput.value.trim();
+    let release = releaseFilter2.value.trim();
+    let wg = workingGroupFilter2.value.trim();
+    let specType = specTypeFilter2.value.trim();
+    let threshold = thresholdInput.value.trim();
+
+    if (!keywords){
+        showError("Please enter at least one keyword");
+        return;
+    }
+
+    showLoader();
+    hideError();
+
+    try{
+        let body = {
+            keywords,
+            threshold
+        };
+        if (release != ""){body["release"] = release}
+        if (wg != ""){body["working_group"] = wg}
+        if (specType != ""){body["spec_type"] = specType}
+        const response = await fetch("/search-spec/experimental", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json"
+            },
+            body: JSON.stringify(body)
+        });
+
+        const data = await response.json();
+        if (response.ok){
+            displayKeywordResults(data, "");
+        } else if (response.status == 404) {
+            showError('No specification has been found');
+        } else {
+            showError(`Error processing keyword request: ${data.detail}`)
+        }
+    } catch (error) {
+        showError('Error connecting to the server. Please check if the API is running.');
+        console.error('Error:', error);
+    } finally {
+        hideLoader();
+    }
+})
+
 keywordSearchBtn.addEventListener("click", async ()=>{
     let keywords = keywordInput.value.trim();
     let release = releaseFilter.value;
@@ -448,4 +530,10 @@ keywordInput.addEventListener('keypress', (event)=>{
     if (event.key === "Enter"){
         keywordSearchBtn.click();
     }
+})
+
+expKeywordInput.addEventListener('keypress', (event)=>{
+    if (event.key === "Enter"){
+        keywordSearchBtn.click();
+    }
 })

static/style.css

@@ -217,6 +217,16 @@ header {
     z-index: 100;
 }
 
+.input-field input.filter-input{
+    flex: none;
+    padding: 7px 10px;
+    border: 1px solid #ddd;
+    border-radius: 4px;
+    background: #fff;
+    color: #333;
+    font-size: 15px;
+}
+
 .header-content {
     display: flex;
     align-items: center;
@@ -340,11 +350,7 @@ header {
     border-bottom: 2px solid var(--primary-color);
 }
 
-.batch-input {
-    display: none;
-}
-
-.keyword-input {
+.batch-input, .keyword-input, .experimental-input {
     display: none;
 }
 

templates/index.html

@@ -30,6 +30,7 @@
                 <button id="single-mode-btn" class="active">Single Document</button>
                 <button id="batch-mode-btn">Batch Search</button>
                 <button id="keyword-mode-btn">Keyword Search</button>
+                <button id="exp-mode-btn">Experimental Search (Using BM25)</button>
             </div>
 
             <div class="search-form">
@@ -106,6 +107,49 @@
                         </form>
                     </div>
                 </div>
+                <div class="input-group experimental-input">
+                    <div class="input-field">
+                        <input type="number" name="threshold" id="threshold" class="filter-input" min="30" max="100" placeholder="Min %">
+                        <input type="text" id="exp-keywords" placeholder="Enter keywords separated by spaces">
+                        <button id="exp-search-btn" class="btn">Search</button>
+                    </div>
+                    <div class="filter-tab-container">
+                        <button type="button" id="toggleFilters2" class="filter-toggle-btn">
+                            <span>Filters</span>
+                            <svg width="16" height="16" style="vertical-align:middle" fill="currentColor">
+                                <path d="M4 7l4 4 4-4"></path>
+                            </svg>
+                        </button>
+                        <form id="filtersForm2" class="filters-row" style="display:none;">
+                            <input type="number" min="0" max="21" name="release2" placeholder="Release"
+                                class="filter-input">
+
+                            <select name="spec_type2" class="filter-select">
+                                <option value="">All types</option>
+                                <option value="TR">Technical Report (TR)</option>
+                                <option value="TS">Technical Specification (TS)</option>
+                            </select>
+
+                            <select name="working_group2" class="filter-select">
+                                <option value="">Working Group</option>
+                                <option value="CP">CP</option>
+                                <option value="C1">C1</option>
+                                <option value="C2">C2</option>
+                                <option value="C3">C3</option>
+                                <option value="C4">C4</option>
+                                <option value="C5">C5</option>
+                                <option value="C6">C6</option>
+                                <option value="SP">SP</option>
+                                <option value="S1">S1</option>
+                                <option value="S2">S2</option>
+                                <option value="S3">S3</option>
+                                <option value="S4">S4</option>
+                                <option value="S5">S5</option>
+                                <option value="S6">S6</option>
+                            </select>
+                        </form>
+                    </div>
+                </div>
             </div>
 
             <div class="error-message" id="error-message"></div>