Upload 7 files

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+pib2022_23_cleaned.csv filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,68 @@
+import faiss
+import pickle
+import pandas as pd
+import streamlit as st
+from sentence_transformers import SentenceTransformer
+from vector_engine.utils import vector_search
+
+@st.cache_data
+def read_data(pibdata="pib2022_23_cleaned.csv"):
+    """Read the pib data."""
+    return pd.read_csv(pibdata)
+
+
+@st.cache_resource
+def load_bert_model(name="pushpdeep/sbertmsmarco-en_to_indic_ur-murilv1"):
+    """Instantiate a sentence-level DistilBERT model."""
+    return SentenceTransformer(name)
+
+
+@st.cache_data
+def load_faiss_index(path_to_faiss="models/faiss_index_ip.pickle"):
+    """Load and deserialize the Faiss index."""
+    with open(path_to_faiss, "rb") as h:
+        data = pickle.load(h)
+    return faiss.deserialize_index(data)
+
+def main():
+    # Load data and models
+    data = read_data()
+    model = load_bert_model()
+    faiss_index = load_faiss_index()
+
+    st.title("Vector-based search with Sentence Transformers and Faiss")
+
+    # User search
+    user_input = st.text_area("Search box", "Aatmanirbhar Bharat")
+
+    # Filters
+    st.sidebar.markdown("**Filters**")
+    # filter_year = st.sidebar.slider("Publication year", 2010, 2021, (2010, 2021), 1)
+    # filter_citations = st.sidebar.slider("Citations", 0, 250, 0)
+    num_results = st.sidebar.slider("Number of search results", 10, 50, 10)
+
+    # Fetch results
+    if user_input:
+        # Get paper IDs
+        D, I = vector_search([user_input], model, faiss_index, num_results)
+        # Slice data on year
+        frame = data
+        # Get individual results
+        for id_ in I.flatten().tolist():
+            if id_ in set(frame.rid):
+                f = frame[(frame.rid == id_)]
+            else:
+                continue
+
+            st.write(
+                f"""
+            **Language**: {f.iloc[0].language}  
+            **Monthyear**: {f.iloc[0].posted-on}  
+            **Abstract**
+            {f.iloc[0].body}
+            """
+            )
+
+
+if __name__ == "__main__":
+    main()

models/faiss_index_ip.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:418bbb2ecd560a57a6007a1c6dfbbd6e48babe5e5aee0219d6b78c3c6ee0862e
+size 271674732

pib2022_23_cleaned.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08b9c8bc455941f30610fc05c588f46af2c769ca38d42b4919cebb895631351b
+size 619820988

requirements.txt

@@ -0,0 +1,7 @@
+torch
+transformers
+sentence-transformers
+pandas
+faiss-cpu
+numpy
+-e .

vector_engine/utils.py

@@ -0,0 +1,24 @@
+import numpy as np
+
+
+def vector_search(query, model, index, num_results=10):
+    """Tranforms query to vector using a pretrained, sentence-level 
+    DistilBERT model and finds similar vectors using FAISS.
+    Args:
+        query (str): User query that should be more than a sentence long.
+        model (sentence_transformers.SentenceTransformer.SentenceTransformer)
+        index (`numpy.ndarray`): FAISS index that needs to be deserialized.
+        num_results (int): Number of results to return.
+    Returns:
+        D (:obj:`numpy.array` of `float`): Distance between results and query.
+        I (:obj:`numpy.array` of `int`): Paper ID of the results.
+    
+    """
+    vector = model.encode(list(query))
+    D, I = index.search(np.array(vector).astype("float32"), k=num_results)
+    return D, I
+
+
+def id2details(df, I, column):
+    """Returns the paper titles based on the paper index."""
+    return [list(df[df.rid == idx][column]) for idx in I[0]]