updated to pull files locally

local_files/astro_ph_ga_embedding_27-Jun-2023.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:476fb3dc5155a733d6427b9f3cc126134d8be881cb0d598df2876f9a61dd672d
+size 265762

local_files/astro_ph_ga_feeds_ada_embedding_27-Jun-2023.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:655af27d3c033e15ded2051d0b3a668dd429f64ec3bed5ceac3eacd969e618dd
+size 407961763

local_files/astro_ph_ga_feeds_upto_27-Jun-2023.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29237e0e973a5fcd4df826c09432a069e6a471d5725fdfc9a0f8c7c62b69e188
+size 89228171

pages/1_paper_search.py

@@ -7,6 +7,7 @@ import streamlit as st
 import feedparser
 import urllib
 import cloudpickle as cp
+import pickle
 from urllib.request import urlopen
 from summa import summarizer
 import numpy as np
@@ -19,14 +20,21 @@ os.environ["OPENAI_API_KEY"] = openai.api_key
 
 @st.cache_data
 def get_feeds_data(url):
-    data = cp.load(urlopen(url))
-    st.sidebar.success("Fetched data from API!")
+    with open(url, "rb") as fp:
+        data = pickle.load(fp)
+    st.sidebar.success("Loaded data!")
+    # data = cp.load(urlopen(url))
+    # st.sidebar.success("Fetched data from API!")
     return data
 
 embeddings = OpenAIEmbeddings()
 
-feeds_link = "https://drive.google.com/uc?export=download&id=1-IPk1voyUM9VqnghwyVrM1dY6rFnn1S_"
-embed_link = "https://dl.dropboxusercontent.com/s/ob2betm29qrtb8v/astro_ph_ga_feeds_ada_embedding_18-Apr-2023.pkl?dl=0"
+# feeds_link = "https://drive.google.com/uc?export=download&id=1-IPk1voyUM9VqnghwyVrM1dY6rFnn1S_"
+# embed_link = "https://dl.dropboxusercontent.com/s/ob2betm29qrtb8v/astro_ph_ga_feeds_ada_embedding_18-Apr-2023.pkl?dl=0"
+
+dateval = "27-Jun-2023"
+feeds_link = "local_files/astro_ph_ga_feeds_upto_"+dateval+".pkl"
+embed_link = "local_files/astro_ph_ga_feeds_ada_embedding_"+dateval+".pkl"
 gal_feeds = get_feeds_data(feeds_link)
 arxiv_ada_embeddings = get_feeds_data(embed_link)
 
@@ -174,6 +182,7 @@ model_data = [arxiv_ada_embeddings, embeddings, all_titles, all_text, all_author
 
 st.title('ArXiv similarity search:')
 st.markdown('Search for similar papers by arxiv id or phrase:')
+st.markdown('[Includes papers up to: `'+dateval+'`]')
 
 search_type = st.radio(
     "What are you searching by?",

pages/2_arxiv_embedding.py

@@ -7,33 +7,78 @@ from bokeh.palettes import OrRd
 from bokeh.plotting import figure, show
 from bokeh.plotting import ColumnDataSource, figure, output_notebook, show
 import cloudpickle as cp
+import pickle
 from scipy import stats
 from urllib.request import urlopen
 
 st.title("ArXiv+GPT3 embedding explorer")
+st.markdown('[Includes papers up to: `'+dateval+'`]')
 st.markdown("This is an explorer for astro-ph.GA papers on the arXiv (up to Apt 18th, 2023). The papers have been preprocessed with `chaotic_neural` [(link)](http://chaotic-neural.readthedocs.io/) after which the collected abstracts are run through `text-embedding-ada-002` with [langchain](https://python.langchain.com/en/latest/ecosystem/openai.html) to generate a unique vector correpsonding to each paper. These are then compressed using [umap](https://umap-learn.readthedocs.io/en/latest/) and shown here, and can be used for similarity searches with methods like [faiss](https://github.com/facebookresearch/faiss). The scatterplot here can be paired with a heatmap for more targeted searches looking at a specific topic or area (see sidebar). Upgrade to chaotic neural suggested by Jo Ciucă, thank you! More to come (hopefully) with GPT-4 and its applications!")
 st.markdown("Interpreting the UMAP plot: the algorithm creates a 2d embedding from the high-dim vector space that tries to conserve as much similarity information as possible. Nearby points in UMAP space are similar, and grow dissimiliar as you move farther away. The axes do not have any physical meaning.")
 
+@st.cache_data
+def get_feeds_data(url):
+    # data = cp.load(urlopen(url))
+    with open(url, "rb") as fp:
+        data = pickle.load(fp)
+    st.sidebar.success("Fetched data from API!")
+    return data
+
+# embeddings = OpenAIEmbeddings()
+
+dateval = "27-Jun-2023"
+feeds_link = "local_files/astro_ph_ga_feeds_upto_"+dateval+".pkl"
+embed_link = "local_files/astro_ph_ga_feeds_ada_embedding_"+dateval+".pkl"
+gal_feeds = get_feeds_data(feeds_link)
+arxiv_ada_embeddings = get_feeds_data(embed_link)
+
 @st.cache_data
 def get_embedding_data(url):
-    data = cp.load(urlopen(url))
+    # data = cp.load(urlopen(url))
+    with open(url, "rb") as fp:
+        data = pickle.load(fp)
     st.sidebar.success("Fetched data from API!")
     return data
 
-url = "https://drive.google.com/uc?export=download&id=1133tynMwsfdR1wxbkFLhbES3FwDWTPjP"
-embedding, all_text, all_titles, all_arxivid, all_links = get_embedding_data(url)
+url = "local_files/astro_ph_ga_embedding_"+dateval+".pkl"
+# e2d, _, _, _, _ = get_embedding_data(url)
+embedding = get_embedding_data(url)
+
+st.title("ArXiv+GPT3 embedding explorer")
+st.markdown('[Includes papers up to: `'+dateval+'`]')
+st.markdown("This is an explorer for astro-ph.GA papers on the arXiv (up to Apt 18th, 2023). The papers have been preprocessed with `chaotic_neural` [(link)](http://chaotic-neural.readthedocs.io/) after which the collected abstracts are run through `text-embedding-ada-002` with [langchain](https://python.langchain.com/en/latest/ecosystem/openai.html) to generate a unique vector correpsonding to each paper. These are then compressed using [umap](https://umap-learn.readthedocs.io/en/latest/) and shown here, and can be used for similarity searches with methods like [faiss](https://github.com/facebookresearch/faiss). The scatterplot here can be paired with a heatmap for more targeted searches looking at a specific topic or area (see sidebar). Upgrade to chaotic neural suggested by Jo Ciucă, thank you! More to come (hopefully) with GPT-4 and its applications!")
+st.markdown("Interpreting the UMAP plot: the algorithm creates a 2d embedding from the high-dim vector space that tries to conserve as much similarity information as possible. Nearby points in UMAP space are similar, and grow dissimiliar as you move farther away. The axes do not have any physical meaning.")
+
+from tqdm import tqdm
+ctr = -1
+num_chunks = len(gal_feeds)
+all_text = []
+all_titles = []
+all_arxivid = []
+all_links = []
+
+for nc in tqdm(range(num_chunks)):
+    for i in range(len(gal_feeds[nc].entries)):
+        text = gal_feeds[nc].entries[i].summary
+        text = text.replace('\n', ' ')
+        text = text.replace('\\', '')
+        all_text.append(text)
+        all_titles.append(gal_feeds[nc].entries[i].title)
+        all_arxivid.append(gal_feeds[nc].entries[i].id.split('/')[-1][0:-2])
+        all_links.append(gal_feeds[nc].entries[i].links[1].href)
+
 
 def density_estimation(m1, m2, xmin=0, ymin=0, xmax=15, ymax=15):
-    X, Y = np.mgrid[xmin:xmax:100j, ymin:ymax:100j]                                                     
-    positions = np.vstack([X.ravel(), Y.ravel()])                                                       
-    values = np.vstack([m1, m2])                                                                        
-    kernel = stats.gaussian_kde(values)                                                                 
+    X, Y = np.mgrid[xmin:xmax:100j, ymin:ymax:100j]
+    positions = np.vstack([X.ravel(), Y.ravel()])
+    values = np.vstack([m1, m2])
+    kernel = stats.gaussian_kde(values)
     Z = np.reshape(kernel(positions).T, X.shape)
     return X, Y, Z
-        
+
 st.sidebar.markdown('This is a widget that allows you to look for papers containing specific phrases in the dataset and show it as a heatmap. Enter the phrase of interest, then change the size and opacity of the heatmap as desired to find the high-density regions. Hover over blue points to see the details of individual papers.')
 st.sidebar.markdown('`Note`: (i) if you enter a query that is not in the corpus of abstracts, it will return an error. just enter a different query in that case. (ii) there are some empty tooltips when you hover, these correspond to the underlying hexbins, and can be ignored.')
-    
+
 st.sidebar.text_input("Search query", key="phrase", value="")
 alpha_value = st.sidebar.slider("Pick the hexbin opacity",0.0,1.0,0.1)
 size_value = st.sidebar.slider("Pick the hexbin size",0.0,2.0,0.2)
@@ -44,7 +89,7 @@ phrase_flags = np.zeros((len(all_text),))
 for i in range(len(all_text)):
     if phrase.lower() in all_text[i].lower():
         phrase_flags[i] = 1
-        
+
 
 source = ColumnDataSource(data=dict(
     x=embedding[0:,0],
@@ -61,11 +106,11 @@ ID: $index
 @link <br> <br>
 </div>
 """
-        
+
 p = figure(width=700, height=583, tooltips=TOOLTIPS, x_range=(0, 15), y_range=(2.5,15),
            title="UMAP projection of trained ArXiv corpus | heatmap keyword: "+phrase)
 
-p.hexbin(embedding[phrase_flags==1,0],embedding[phrase_flags==1,1], size=size_value, 
+p.hexbin(embedding[phrase_flags==1,0],embedding[phrase_flags==1,1], size=size_value,
          palette = np.flip(OrRd[8]), alpha=alpha_value)
 p.circle('x', 'y', size=3, source=source, alpha=0.3)
 

pages/3_qa_sources.py

@@ -7,6 +7,7 @@ import streamlit as st
 import feedparser
 import urllib
 import cloudpickle as cp
+import pickle
 from urllib.request import urlopen
 from summa import summarizer
 import numpy as np
@@ -26,25 +27,34 @@ os.environ["OPENAI_API_KEY"] = openai.api_key
 
 @st.cache_data
 def get_feeds_data(url):
-    data = cp.load(urlopen(url))
-    st.sidebar.success("Fetched data from API!")
+    # data = cp.load(urlopen(url))
+    with open(url, "rb") as fp:
+        data = pickle.load(fp)
+    st.sidebar.success("Loaded data")
     return data
 
 embeddings = OpenAIEmbeddings()
 
-feeds_link = "https://drive.google.com/uc?export=download&id=1-IPk1voyUM9VqnghwyVrM1dY6rFnn1S_"
-embed_link = "https://dl.dropboxusercontent.com/s/ob2betm29qrtb8v/astro_ph_ga_feeds_ada_embedding_18-Apr-2023.pkl?dl=0"
+# feeds_link = "https://drive.google.com/uc?export=download&id=1-IPk1voyUM9VqnghwyVrM1dY6rFnn1S_"
+# embed_link = "https://dl.dropboxusercontent.com/s/ob2betm29qrtb8v/astro_ph_ga_feeds_ada_embedding_18-Apr-2023.pkl?dl=0"
+dateval = "27-Jun-2023"
+feeds_link = "local_files/astro_ph_ga_feeds_upto_"+dateval+".pkl"
+embed_link = "local_files/astro_ph_ga_feeds_ada_embedding_"+dateval+".pkl"
 gal_feeds = get_feeds_data(feeds_link)
 arxiv_ada_embeddings = get_feeds_data(embed_link)
 
 @st.cache_data
 def get_embedding_data(url):
-    data = cp.load(urlopen(url))
+    # data = cp.load(urlopen(url))
+    with open(url, "rb") as fp:
+        data = pickle.load(fp)
     st.sidebar.success("Fetched data from API!")
     return data
 
-url = "https://drive.google.com/uc?export=download&id=1133tynMwsfdR1wxbkFLhbES3FwDWTPjP"
-e2d, _, _, _, _ = get_embedding_data(url)
+# url = "https://drive.google.com/uc?export=download&id=1133tynMwsfdR1wxbkFLhbES3FwDWTPjP"
+url = "local_files/astro_ph_ga_embedding_"+dateval+".pkl"
+e2d = get_embedding_data(url)
+# e2d, _, _, _, _ = get_embedding_data(url)
 
 ctr = -1
 num_chunks = len(gal_feeds)
@@ -286,6 +296,7 @@ def run_query(query, return_n = 3, show_pure_answer = False, show_all_sources =
     return output
 
 st.title('ArXiv-based question answering')
+st.markdown('[Includes papers up to: `'+dateval+'`]')
 st.markdown('Concise answers for questions using arxiv abstracts + GPT-4. Please use sparingly because it costs me money right now. You might need to wait for a few seconds for the GPT-4 query to return an answer (check top right corner to see if it is still running).')
 
 query = st.text_input('Your question here:', value="What sersic index does a disk galaxy have?")