File size: 5,893 Bytes
8503206 2b126e8 8503206 1bd03b4 8503206 5dd932f 8503206 5dd932f 8503206 5dd932f dac867a 93c3874 8503206 36bfc91 8503206 93c3874 8503206 36bfc91 8503206 6001e52 93c3874 8503206 1bd03b4 8503206 36bfc91 8503206 1bd03b4 36bfc91 1bd03b4 36bfc91 1bd03b4 36bfc91 5dd932f 36bfc91 8503206 1bd03b4 36bfc91 8503206 36bfc91 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 |
from dotenv import load_dotenv
from langchain_google_genai import ChatGoogleGenerativeAI
from langchain_openai import ChatOpenAI
from prompt import *
from utils import *
import os
import re
load_dotenv()
class Validation():
def __init__(self, llm):
if llm.startswith('gpt'):
self.llm = ChatOpenAI(temperature=0, model_name=llm)
elif llm.startswith('gemini'):
self.llm = ChatGoogleGenerativeAI(temperature=0, model=llm)
else:
self.llm = ChatOpenAI(temperature=0, model_name=llm, api_key=os.environ['PERPLEXITY_API_KEY'], base_url="https://api.perplexity.ai")
def validate(self, df, text, api):
df = df.fillna('')
df['Genes'] = df['Genes'].str.replace(' ', '').str.upper()
df['rsID'] = df['rsID'].str.replace(' ', '').str.lower()
# Check if there are multiple Genes
sym = [',', '/', '|', '-', '(', ')']
i = 0
while i < len(df):
gene = df.loc[i, 'Genes']
for s in sym:
if s in gene:
genes = gene.split(s)
df.loc[i + 0.1], df.loc[i + 0.9] = df.loc[i], df.loc[i]
df = df.sort_index().reset_index(drop=True)
df.loc[i + 1, 'Genes'], df.loc[i + 2, 'Genes'] = genes[0], s.join(genes[1:])
break
i += 1
df.reset_index(drop=True, inplace=True)
# Check if there are SNPs not well captured
for i in df.index:
safe = True
snp = df.loc[i, 'rsID']
snp = snp.replace('l', '1')
if re.fullmatch('rs(\d)+|', snp):
pass
elif re.fullmatch('ts(\d)+', snp):
snp = 'r' + snp[1:]
elif re.fullmatch('s(\d)+', snp):
snp = 'r' + snp
elif re.fullmatch('(\d)+', snp):
snp = 'rs' + snp
elif re.fullmatch('r(\d)+', snp):
snp = 'rs' + snp[1:]
if snp[2] == '5':
snp += f',rs{snp[3:]}'
else:
safe = False
df = df.drop(i)
if safe:
df.loc[i, 'rsID'] = snp
df.reset_index(drop=True, inplace=True)
# Check if there are multiple rsIDs
i = 0
while i < len(df):
rsid = df.loc[i, 'rsID']
if ',' in rsid:
rsids = rsid.split(',')
df.loc[i + 0.1], df.loc[i + 0.9] = df.loc[i], df.loc[i]
df = df.sort_index().reset_index(drop=True)
df.loc[i + 1, 'rsID'], df.loc[i + 2, 'rsID'] = rsids[0], s.join(rsids[1:])
i += 1
df.reset_index(drop=True, inplace=True)
df_clean = df.copy()
# WARNING: DEPRECATED
# Validate genes and SNPs with APIs
if api:
dbsnp = {}
for i in df.index:
snp = df.loc[i, 'SNPs']
gene = df.loc[i, 'Genes']
if snp not in dbsnp:
try:
res = call(f'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=snp&retmode=json&id={snp[2:]}').json()['result'][snp[2:]]
if 'error' not in res:
dbsnp[snp].extend([r['name'] for r in res['genes']])
except Exception as e:
print("Error at API", e)
pass
dbsnp[snp] = list(set(dbsnp[snp]))
if gene not in dbsnp[snp]:
for other in permutate(gene):
if other in dbsnp[snp]:
df.loc[i, 'Genes'] = other
print(f'{gene} corrected to {other}')
break
else:
df = df.drop(i)
# WARNING: DEPRECATED
# Check with GWAS ground truth
if False:
for i in df.index:
gene = df.loc[i, 'Genes']
snp = df.loc[i, 'rsID']
perms = permutate(gene)
for perm in perms:
if perm in ground_truth and snp in ground_truth[perm]:
df.loc[i, 'Genes'] = perm
if gene != perm:
print(f'{gene} corrected to {perm} with {snp}')
else:
print(f'{gene} and {snp} safe')
break
else:
print(f'{gene} and {snp} not found')
df = df.drop(i)
# Check with Text
for i in df.index:
gene = df.loc[i, 'Genes']
snp = df.loc[i, 'rsID']
perms = permutate(gene)
for perm in perms:
if perm in text and snp in text:
df.loc[i, 'Genes'] = perm
if gene != perm:
print(f'{gene} corrected to {perm} with {snp}')
else:
print(f'{gene} and {snp} safe')
break
else:
print(f'{gene} and {snp} not found')
df = df.drop(i)
# Drop (duplicate) entries with empty values
genes = []
snps = []
for i in df.index:
gene = df.loc[i, 'Genes']
snp = df.loc[i, 'rsID']
if len(gene) == 0 and len(snp) == 0:
df = df.drop(i)
elif len(gene) == 0:
if snp in snps:
df = df.drop(i)
elif len(snp) == 0:
if gene in genes:
df = df.drop(i)
else:
genes.append(gene)
snps.append(snp)
df.reset_index(drop=True, inplace=True)
return df, df_clean |