Upload 23 files

.gitattributes

@@ -32,3 +32,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+lucene-index.miracl-v1.0-bn.20221004.2b2856/_0_Lucene90_0.doc filter=lfs diff=lfs merge=lfs -text
+lucene-index.miracl-v1.0-bn.20221004.2b2856/_0_Lucene90_0.dvd filter=lfs diff=lfs merge=lfs -text
+lucene-index.miracl-v1.0-bn.20221004.2b2856/_0_Lucene90_0.pos filter=lfs diff=lfs merge=lfs -text
+lucene-index.miracl-v1.0-bn.20221004.2b2856/_0_Lucene90_0.tim filter=lfs diff=lfs merge=lfs -text
+lucene-index.miracl-v1.0-bn.20221004.2b2856/_0.fdt filter=lfs diff=lfs merge=lfs -text
+lucene-index.miracl-v1.0-bn.20221004.2b2856/_0.tvd filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,10 +1,10 @@
 ---
-title: Miracl Search - Bengali
-emoji: ⚡
-colorFrom: yellow
-colorTo: purple
+title: Miracl Search
+emoji: 🌍🙌🌏
+colorFrom: red
+colorTo: pink
 sdk: streamlit
-sdk_version: 1.17.0
+sdk_version: 1.10.0
 app_file: app.py
 pinned: false
 license: apache-2.0

app.py

@@ -0,0 +1,201 @@
+import http.client as http_client
+import json
+import logging
+import os
+import pprint
+import re
+import time
+import string
+
+import streamlit as st
+
+import streamlit.components.v1 as components
+from typing import Callable, Optional, Tuple, Union
+from pyserini import util
+from pyserini.search import LuceneSearcher, FaissSearcher, AutoQueryEncoder
+
+
+VERSION = '1.0'
+st.set_page_config(page_title="Miracl Search - Bengali", layout="wide")
+
+os.makedirs(os.path.join(os.getcwd(),".streamlit"), exist_ok = True)
+with open(os.path.join(os.getcwd(),".streamlit/config.toml"), "w") as file:
+    file.write(
+        '[theme]\nbase="light"'
+    )
+
+Searcher = Union[FaissSearcher, LuceneSearcher]
+LANG_MAPPING = {'Bengali':'bn'}
+
+
+st.sidebar.markdown(
+"""
+<style>
+.aligncenter {
+    text-align: center;
+    font-weight: bold;
+    font-size: 30px;
+}
+</style>
+<p class="aligncenter">MIRACL Bengali Demo</p>
+<p class="aligncenter">🌍🙌🌏</p>
+<p style="text-align: center;"> MIRACL is a multilingual dataset for ad hoc retrieval that consists of 18 different languages, collectively encompassing over three billion native speakers around the world.</p>
+""",
+unsafe_allow_html=True,
+)
+
+st.sidebar.markdown(
+"""
+<style>
+.aligncenter {
+    text-align: center;
+}
+</style>
+<p style='text-align: center'>
+<a href="https://github.com/project-miracl" >GitHub</a> | <a href="https://arxiv.org/abs/2210.09984" >Paper</a>
+</p>
+""",
+unsafe_allow_html=True,
+)
+
+query = st.sidebar.text_input(label='Search query', value='')
+language = 'Bengali'
+
+max_results = st.sidebar.slider(
+    "Maximum Number of Results",
+    min_value=1,
+    max_value=1000,
+    step=1,
+    value=10,
+    help="Maximum Number of Documents to return",
+)
+
+
+def _load_sparse_searcher(language: str, k1: Optional[float]=None, b: Optional[float]=None) -> (Searcher):
+    searcher = LuceneSearcher(f'lucene-index.miracl-v1.0-{language}.20221004.2b2856')
+    searcher.set_language(language)
+    if k1 is not None and b is not None:
+        searcher.set_bm25(k1, b)
+        retriever_name = f'BM25 (k1={k1}, b={b})'
+    else:
+        retriever_name = 'BM25'
+
+    return searcher
+
+def search(query, language, num_results=10):
+    searcher = _load_sparse_searcher(language=LANG_MAPPING[language])
+
+    t_0 = time.time()
+    search_results = searcher.search(query, k=num_results)
+    search_time = time.time() - t_0
+
+    results_dict ={"docs": [], "doc_ids": [], "score":[], "lang": language}
+    for i, result in enumerate(search_results):
+        result = json.loads(result.raw)
+        results_dict["docs"].append(result["text"])
+        results_dict["doc_ids"].append(result["docid"])
+        results_dict["score"].append(search_results[i].score)
+
+    return results_dict, search_time
+
+
+
+def highlight_string(paragraph: str, highlight_terms: list) -> str:
+    for term in highlight_terms:
+        paragraph = re.sub(f"\\b{term}\\b", f"<b>{term}</b>", paragraph, flags=re.I)
+    return paragraph
+
+def process_results(hits: dict, highlight_terms: list) -> str:
+    hit_list = []
+    for i in range(len(hits['doc_ids'])):
+        res_head = f"""
+                    <div class='searchresult'>
+                        <h2>{i+1}. Document ID: {hits['doc_ids'][i]}</h2>
+                        <p>Language: <string>{hits['lang']}</string>, Score: {round(hits['score'][i], 2)}</p>              
+                        <p>{highlight_string(hits['docs'][i], highlight_terms)}</p>
+                    </div>
+                    <hr>
+                    """
+        hit_list.append(res_head)
+    return " ".join(hit_list)
+
+
+
+if st.sidebar.button("Search"):
+    hits, search_time = search(query, language, max_results)
+    html_results = process_results(hits, [])
+    rendered_results = f"""
+            <div id="searchresultsarea">
+                <br>
+                <p id="searchresultsnumber">About {max_results} results</p>
+                {html_results}
+            </div>
+                        """
+    st.markdown("""
+                <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet" 
+                    integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
+                """,
+                unsafe_allow_html=True)
+    st.markdown(
+        """
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/all.min.css">                                  
+        """,
+        unsafe_allow_html=True)
+    st.markdown(
+        f"""
+        <div class="row no-gutters mt-3 align-items-center">
+            <h2> Search Results </h2>
+        </div> 
+        """,
+        unsafe_allow_html=True)
+    components.html(
+        """
+        <style>
+        #searchresultsarea {
+            font-family: 'Arial';
+        }
+        
+        #searchresultsnumber {
+            font-size: 0.8rem;
+            color: gray;
+        }
+        
+        .searchresult h2 {
+            font-size: 19px;
+            line-height: 18px;
+            font-weight: normal;
+            color: rgb(7, 111, 222);
+            margin-bottom: 0px;
+            margin-top: 25px;
+        }
+        
+        .searchresult a {
+            font-size: 12px;
+            line-height: 12px;
+            color: green;
+            margin-bottom: 0px;
+        }
+    
+        .dark-mode {
+          color: white;
+        }
+        </style>
+        <script>
+        function load_image(id){
+            console.log(id)
+            var x = document.getElementById(id);
+            console.log(x)
+            if (x.style.display === "none") {
+                x.style.display = "block";
+            } else {
+                x.style.display = "none";
+            }
+        };
+        function myFunction() {
+           var element = document.body;
+           element.classList.toggle("dark-mode");
+        }
+        </script>
+        <button onclick="myFunction()">Toggle dark mode</button>
+        """ + rendered_results, height=800, scrolling=True
+        )

lucene-index.miracl-v1.0-bn.20221004.2b2856/_0.fdt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da8c07667ca7c4b80c6e2f27b6841bde38e98b2f77bd6ba8ebae6f24431a7de1
+size 142530944

lucene-index.miracl-v1.0-bn.20221004.2b2856/_0.tvd

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d852c9f32bc07bcdcf529989adbbb942f32ed41184f15b2b4360b417e61c891
+size 102577215

lucene-index.miracl-v1.0-bn.20221004.2b2856/_0_Lucene90_0.doc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb9d967ae8cb98c78f2b49fe2e124f93befca12ebbafa2ce379ca2a0bf99acbf
+size 18965534

lucene-index.miracl-v1.0-bn.20221004.2b2856/_0_Lucene90_0.dvd

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cb27fe20fbeab3424704e3e362068e77017e66dd71ba2cb9e776e57446e4b18
+size 2841303

lucene-index.miracl-v1.0-bn.20221004.2b2856/_0_Lucene90_0.pos

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7742570f4594f271ef3305678870f08d29837b59bd4aadf3e3552069498a753f
+size 14588563

lucene-index.miracl-v1.0-bn.20221004.2b2856/_0_Lucene90_0.tim

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:551d897f92450a25e0819c4d27455f96c6e95ced78a73c064803d464950758c4
+size 7926497

packages.txt

@@ -0,0 +1 @@
+openjdk-11-jdk

requirements.txt

@@ -0,0 +1,3 @@
+pyserini
+faiss-cpu
+torch