fetch_data.py

from arvix import extract_arxiv_data
from pmc import extract_pmc_data
import json
import dotenv
import os
from concurrent.futures import ThreadPoolExecutor

dotenv.load_dotenv()
ACCESS_KEY = os.getenv("ACCESS_KEY")

def fetch_arxiv_data():
    return json.loads(extract_arxiv_data())

def fetch_pmc_data():
    return json.loads(extract_pmc_data())

def fetch_data(user_access_key):
    if user_access_key != ACCESS_KEY:
        papers_data = {"error": "Invalid access key"}
    else:
        papers_data = {}
        with ThreadPoolExecutor() as executor:
            arxiv_future = executor.submit(fetch_arxiv_data)
            pmc_future = executor.submit(fetch_pmc_data)
            arxiv_data = arxiv_future.result()
            pmc_data = pmc_future.result()
        for topic, topic_data in arxiv_data.items():
            if topic_data['count'] == 0:
                continue
            else:
                papers_data[topic] = {}
                papers_data[topic]['ids'] = topic_data['ids']
        for topic, topic_data in pmc_data.items():
            if topic_data['count'] == 0:
                continue
            else:
                papers_data[topic] = {}
                papers_data[topic]['ids'] = topic_data['ids']
    data = json.dumps(papers_data, indent=4, ensure_ascii=False)
    return data

if __name__ == '__main__':
    data = fetch_data()
    with open('data.json', 'w') as f:
        f.write(data)