Add database integration

app.py

@@ -11,7 +11,6 @@ from langchain_text_splitters import TokenTextSplitter
 from stqdm import stqdm
 from tempfile import NamedTemporaryFile
 from utils import *
-
 from process import Process
 from validate import Validation
 
@@ -119,13 +118,16 @@ if uploaded_files and submit:
                 df, df_clean = validation.validate(dataframe, api)
                 df.drop_duplicates(['Genes', 'rsID'], ignore_index=True, inplace=True)
 
+                # Integrate with Database
+                df = integrate(df)
+
                 st.write("Success in ", round((datetime.now().timestamp() - start_time.timestamp()) / 60, 2), "minutes")
-                st.write(f"Extracted {len(df)} rows")
+                st.write(f"Extracted **{len(df)}** rows")
                 st.dataframe(df)
 
                 with pd.ExcelWriter(buffer, engine='xlsxwriter') as writer:
-                    df.to_excel(writer, sheet_name='Result Cleaned Validated')
-                    df_clean.to_excel(writer, sheet_name='Result Cleaned')
+                    df.to_excel(writer, sheet_name='Validated + Database')
+                    df_clean.to_excel(writer, sheet_name='Cleaned')
                     dataframe.to_excel(writer, sheet_name='Original')
                     writer.close()
 

utils.py

@@ -1,10 +1,11 @@
-import json
 import os
 import pandas as pd
+import re
 import requests
 import time
 
 from collections import defaultdict
+from io import StringIO
 
 # Common mistakes need to be maintained
 mistakes = {
@@ -50,3 +51,60 @@ def call(url):
             print(e)
 
     return res
+
+raw_url = "https://www.ebi.ac.uk/gwas/api/search/downloads/alternative"
+gwas_path = "resources/gwas_catalog.tsv"
+
+# Load Raw GWAS files
+if os.path.exists(gwas_path):
+    gwas = pd.read_csv(gwas_path, delimiter='\t')[['DISEASE/TRAIT', 'CHR_ID', 'MAPPED_GENE', 'SNPS', 'P-VALUE', 'OR or BETA']]
+else:
+    data = requests.get(raw_url).content.decode('utf-8')
+    gwas = pd.read_csv(StringIO(data), delimiter='\t')[['DISEASE/TRAIT', 'CHR_ID', 'MAPPED_GENE', 'SNPS', 'P-VALUE', 'OR or BETA']]
+
+# Load Genes and SNPs from GWAS
+gwas_gene_rsid = gwas[['MAPPED_GENE', 'SNPS']]
+gwas_gene_rsid.dropna(inplace=True, ignore_index=True)
+gwas_gene_rsid['MAPPED_GENE'] = gwas_gene_rsid['MAPPED_GENE'].apply(lambda x: x.replace(' ', '').upper())
+
+# Generate Genes and SNPs mapping
+ground_truth = defaultdict(list)
+for i in gwas_gene_rsid.index:
+    gene = gwas_gene_rsid.loc[i, 'MAPPED_GENE']
+    snp = gwas_gene_rsid.loc[i, 'SNPS']
+
+    pattern = r"[,\-]"
+    genes = re.split(pattern, gene)
+    for gene in genes:
+        ground_truth[gene].append(snp)
+        ground_truth[snp].append(gene)
+
+def integrate(df):
+
+    # Loop through extractor result
+    df_db = pd.DataFrame()
+    for i in df.index:
+        gene, snp = df.loc[i, 'Genes'], df.loc[i, 'rsID']
+        df_gwas = gwas[(gwas['MAPPED_GENE'].str.contains(gene, na=False)) & \
+                       (gwas['SNPS'].str.contains(snp, na=False))]
+
+        df_db = pd.concat([df_db, df_gwas])
+
+    # Adjust new column
+    df_db.rename(columns={
+        'DISEASE/TRAIT': 'Traits',
+        'MAPPED_GENE': 'Genes',
+        'SNPS': 'rsID',
+        'P-VALUE': 'P Value',
+        'OR or BETA': 'OR Value'
+    }, inplace=True)
+    df_db.drop(columns=['CHR_ID'], inplace=True)
+    df_db['Beta Value'] = df_db['OR Value']
+    df_db['Source'] = 'Database'
+
+    # Combine raw and database
+    df_db = df_db[df.columns]
+    df = pd.concat([df, df_db])
+    df.reset_index(drop=True, inplace=True)
+
+    return df