Update tasks/audio.py

tasks/audio.py

@@ -35,7 +35,7 @@ async def evaluate_audio(request: AudioEvaluationRequest):
     
     # Load dataset
     logging.info("Chargement des données")
-    dataset = load_dataset(request.dataset_name, token=os.getenv("HF_TOKEN"))
+    dataset = load_dataset(request.dataset_name, streaming=True,token=os.getenv("HF_TOKEN"))
     logging.info("Données chargées")
     
     test_dataset = dataset["test"]
@@ -51,7 +51,7 @@ async def evaluate_audio(request: AudioEvaluationRequest):
         audio_arrays = [x["array"] for x in examples["audio"]]
         return feature_extractor(audio_arrays, sampling_rate=feature_extractor.sampling_rate, padding="longest", max_length=16000, truncation=True, return_tensors="pt")
     
-    encoded_data_test = test_dataset.map(preprocess_function, remove_columns="audio", batched=True, streaming=True, keep_in_memory=False)
+    encoded_data_test = test_dataset.map(preprocess_function, remove_columns="audio", batched=True, keep_in_memory=False)
     del feature_extractor
     del audio_arrays
     # Pipeline de classification