load_data.py

import os
from app.vectorizer import Vectorizer

os.environ['HF_HOME'] = '/app/cache/hub'

if __name__ == "__main__":
    vectorizer = Vectorizer()
    vectorizer.store_from_dataset(store_data=True)  # Run this once to load the dataset into Pinecone