Spaces:
Runtime error
Runtime error
File size: 9,178 Bytes
e1b1d60 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 |
# set path
import glob, os, sys;
sys.path.append('../utils')
import streamlit as st
import json
import logging
from utils.lexical_search import runLexicalPreprocessingPipeline, lexical_search
from utils.semantic_search import runSemanticPreprocessingPipeline, semantic_keywordsearch
from utils.checkconfig import getconfig
from utils.streamlitcheck import checkbox_without_preselect
# Declare all the necessary variables
config = getconfig('paramconfig.cfg')
split_by = config.get('semantic_search','SPLIT_BY')
split_length = int(config.get('semantic_search','SPLIT_LENGTH'))
split_overlap = int(config.get('semantic_search','SPLIT_OVERLAP'))
split_respect_sentence_boundary = bool(int(config.get('semantic_search',
'RESPECT_SENTENCE_BOUNDARY')))
remove_punc = bool(int(config.get('semantic_search','REMOVE_PUNC')))
embedding_model = config.get('semantic_search','RETRIEVER')
embedding_model_format = config.get('semantic_search','RETRIEVER_FORMAT')
embedding_layer = int(config.get('semantic_search','RETRIEVER_EMB_LAYER'))
embedding_dim = int(config.get('semantic_search','EMBEDDING_DIM'))
max_seq_len = int(config.get('semantic_search','MAX_SEQ_LENGTH'))
retriever_top_k = int(config.get('semantic_search','RETRIEVER_TOP_K'))
reader_model = config.get('semantic_search','READER')
reader_top_k = int(config.get('semantic_search','RETRIEVER_TOP_K'))
top_k_per_candidate = int(config.get('semantic_search','READER_TOP_K_PER_CANDIDATE'))
lexical_split_by= config.get('lexical_search','SPLIT_BY')
lexical_split_length=int(config.get('lexical_search','SPLIT_LENGTH'))
lexical_split_overlap = int(config.get('lexical_search','SPLIT_OVERLAP'))
lexical_remove_punc = bool(int(config.get('lexical_search','REMOVE_PUNC')))
lexical_top_k=int(config.get('lexical_search','TOP_K'))
def app():
with st.container():
st.markdown("<h1 style='text-align: center; \
color: black;'> Search</h1>",
unsafe_allow_html=True)
st.write(' ')
st.write(' ')
with st.expander("ℹ️ - About this app", expanded=False):
st.write(
"""
The *Search* app is an interface \
for doing contextual and keyword searches in \
policy documents. \
""")
st.write("")
st.write(""" The application allows its user to perform a search\
based on two options: a lexical search([TFIDF](https://en.wikipedia.org/wiki/Tf%E2%80%93idf))\
and semantic search. [bi-encoder](https://www.sbert.net/examples/applications/retrieve_rerank/README.html)\
The lexical search only \
displays paragraphs in the document with exact matching results, \
the semantic search shows paragraphs with meaningful connections \
(e.g., synonyms) based on the search context. Both \
methods employ a probabilistic retrieval framework in its identification\
of relevant paragraphs. By defualt the search is performed using \
'Semantic Search', and to find 'Exact/Lexical Matches' please tick the \
checkbox provided which will by-pass semantic search. Furthermore,\
the application allows the user to search for pre-defined keywords \
from different thematic buckets present in sidebar.""")
st.write("")
st.write(""" The Exact Matches gives back top {} findings, and Semantic
search provides with top {} answers.""".format(lexical_top_k, retriever_top_k))
st.write("")
st.write("")
st.markdown("Some runtime metrics tested with cpu: Intel(R) Xeon(R) CPU @ 2.20GHz, memory: 13GB")
col1,col2,col3= st.columns([2,4,4])
with col1:
st.caption("OCR File processing")
# st.markdown('<div style="text-align: center;">50 sec</div>', unsafe_allow_html=True)
st.write("50 sec")
with col2:
st.caption("Lexical Search on 200 paragraphs(~ 35 pages)")
# st.markdown('<div style="text-align: center;">12 sec</div>', unsafe_allow_html=True)
st.write("15 sec")
with col3:
st.caption("Semantic search on 200 paragraphs(~ 35 pages)")
# st.markdown('<div style="text-align: center;">120 sec</div>', unsafe_allow_html=True)
st.write("120 sec(including emebedding creation)")
with st.sidebar:
with open('docStore/sample/keywordexample.json','r') as json_file:
keywordexample = json.load(json_file)
# genre = st.radio("Select Keyword Category", list(keywordexample.keys()))
st.caption("Select Keyword Category")
genre = checkbox_without_preselect(list(keywordexample.keys()))
if genre:
keywordList = keywordexample[genre]
else:
keywordList = None
st.markdown("---")
with st.container():
type_hinting = "Please enter here your question and we \
will look for an answer in the document\
OR enter the keyword you are looking \
for and we will look for similar\
context in the document.\
You can also explore predefined sets of keywords from sidebar. "
if keywordList is not None:
# queryList = st.text_input("You selected the {} category we \
# will look for these keywords in document".format(genre)
# value="{}".format(keywordList))
queryList = st.text_input(type_hinting,
value = "{}".format(keywordList))
else:
queryList = st.text_input(type_hinting,
placeholder="Enter keyword/query here")
searchtype = st.checkbox("Show only Exact Matches")
if st.button("Find them"):
if queryList == "":
st.info("🤔 No keyword provided, if you dont have any, \
please try example sets from sidebar!")
logging.warning("Terminated as no keyword provided")
else:
if 'filepath' in st.session_state:
if searchtype:
all_documents = runLexicalPreprocessingPipeline(
file_name=st.session_state['filename'],
file_path=st.session_state['filepath'],
split_by=lexical_split_by,
split_length=lexical_split_length,
split_overlap=lexical_split_overlap,
remove_punc=lexical_remove_punc)
logging.info("performing lexical search")
with st.spinner("Performing Exact matching search \
(Lexical search) for you"):
lexical_search(query=queryList,
documents = all_documents['documents'],
top_k = lexical_top_k )
else:
all_documents = runSemanticPreprocessingPipeline(
file_path= st.session_state['filepath'],
file_name = st.session_state['filename'],
split_by=split_by,
split_length= split_length,
split_overlap=split_overlap,
remove_punc= remove_punc,
split_respect_sentence_boundary=split_respect_sentence_boundary)
if len(all_documents['documents']) > 100:
warning_msg = ": This might take sometime, please sit back and relax."
else:
warning_msg = ""
logging.info("starting semantic search")
with st.spinner("Performing Similar/Contextual search{}".format(warning_msg)):
semantic_keywordsearch(query = queryList,
documents = all_documents['documents'],
embedding_model=embedding_model,
embedding_layer=embedding_layer,
embedding_model_format=embedding_model_format,
reader_model=reader_model,reader_top_k=reader_top_k,
retriever_top_k=retriever_top_k, embedding_dim=embedding_dim,
max_seq_len=max_seq_len,
top_k_per_candidate = top_k_per_candidate)
else:
st.info("🤔 No document found, please try to upload it at the sidebar!")
logging.warning("Terminated as no document provided")
|