Spaces:

alexpantex
/

prompt-search-app

Sleeping

alexpantex commited on Jan 10

Commit

2c84f1d

verified ·

1 Parent(s): 290938c

Upload scripts/preprocess.py with huggingface_hub

Files changed (1) hide show

scripts/preprocess.py CHANGED Viewed

@@ -29,7 +29,7 @@ def clean_corpus():
     """
     if not os.path.isfile(os.path.join(PROCESSED_DATA_PATH, 'prompt_corpus_clean.parquet')): # to speed up the process
         os.makedirs(PROCESSED_DATA_PATH, exist_ok=True)
-        df = pd.read_parquet(PROMPTS_URL).sample(10000, random_state=123)
         assert 'prompt' in df.columns, "Parquet file must contain a 'prompt' column."
         df = df[df['prompt'].notna()][['prompt']]     # drop missing rows
         df['prompt'] = df['prompt'].apply(preprocess_text)              # preprocess each prompt

     """
     if not os.path.isfile(os.path.join(PROCESSED_DATA_PATH, 'prompt_corpus_clean.parquet')): # to speed up the process
         os.makedirs(PROCESSED_DATA_PATH, exist_ok=True)
+        df = pd.read_parquet(PROMPTS_URL).sample(100000, random_state=123)
         assert 'prompt' in df.columns, "Parquet file must contain a 'prompt' column."
         df = df[df['prompt'].notna()][['prompt']]     # drop missing rows
         df['prompt'] = df['prompt'].apply(preprocess_text)              # preprocess each prompt