update eval code & src readme

src/bash_runners/eval_cv11_test.sh

@@ -7,4 +7,6 @@ python src/run_eval_whisper_streaming.py \
     --text_column="sentence" \
     --device="0" \
     --batch_size="32" \
-    --streaming="True"
+    --streaming="True" \
+    --push_to_hub="True" \
+    --save_predictions="True"

src/bash_runners/eval_fleurs_test.sh

@@ -7,4 +7,6 @@ python src/run_eval_whisper_streaming.py \
     --text_column="transcription" \
     --device="0" \
     --batch_size="32" \
-    --streaming="True"
+    --streaming="True" \
+    --push_to_hub="True" \
+    --save_predictions="True"

src/readme.md

@@ -23,14 +23,6 @@ The code in this repository is a modified version of code from
   --logging_steps="50"
   --eval_steps="1000"
   ```
-* on the next run:
-  * download the whole dataset before the launch. 
-    this will probably save some time for data processing,
-    and allow to load and prepare data in a parallel fashion
-  * can also decrease eval batch size. currently it's probably causing GPU to wait for CPU to prepare a next batch
-* perform evaluation of fine-tuned model on CommonVoice test set
-* add [Whisper fine-tuning Event repo](https://github.com/huggingface/community-events/tree/main/whisper-fine-tuning-event)
-  to remotes and merge updates from this original event repo
 * Learning rate:
   * max learning rate is not the same as LR passed as a parameter to training script. it is actually lower.
   * when resuming training, LR scheduling behaves incorrectly
@@ -69,6 +61,7 @@ When resuming training from existing checkpoint:
 ## Questions:
 * What checkpoint (best, I guess) is saved in the `output_dir`? 
   How is it overwritten when resuming training from existing checkpoint?
+* why dataset loading crashes when using `num_proc > 0`?
 * does `ShuffleCallback` work with StreamingDataset? it reshuffles data `on_epoch_begin()`,
   but does StreamingDataset have any epochs?
 * does streaming mode support parallel data load and processing?<br>
@@ -98,6 +91,9 @@ When resuming training from existing checkpoint:
 * Log tracking in Jupyter (not working) and in bash (works as expected with `tee`)
 * Loggers in `run_speech.....py` do not control `transformers` and `datasets` loggers. 
   can't redirect their outputs using handlers. it's better and easier to redirect output in a bash
+* to evaluate on `google/fleurs` dataset had to downgrade `numba` from `0.56.4` to `0.56.3`, then install `librosa`
+  (strange, because `librosa` should have been installed when `pip install -r ~/whisper-finetuning-be/requirements.txt`
+  was run) and then upgrade back to `numba==0.56.4` because couldn't `import numba` when it was `0.56.3`
 * Need to set `use_cache` to False since we're using gradient checkpointing, and the two are incompatible
 * Default Linear scheduler is used 
 * Default Adam optimizer is used

src/run_eval_whisper_streaming.py

@@ -2,6 +2,9 @@ import argparse
 import logging
 import sys
 import datetime
+import os
+
+import pandas as pd
 
 from transformers import pipeline
 from transformers.models.whisper.english_normalizer import BasicTextNormalizer
@@ -27,7 +30,7 @@ logger.setLevel(logging.INFO)
 
 
 wer_metric = evaluate.load("wer")
-whisper_norm = BelarusianTextNormalizer()
+text_normalizer = BelarusianTextNormalizer()
 
 
 def is_target_text_in_range(ref):
@@ -38,18 +41,22 @@ def is_target_text_in_range(ref):
 
 
 def normalise(sample, text_column: str):
-    sample["norm_text"] = whisper_norm(sample[text_column])
+    sample["reference_norm"] = text_normalizer(sample[text_column])
     return sample
 
 
-def data(dataset):
+def data(dataset,text_column: str):
     for i, item in enumerate(dataset):
-        yield {**item["audio"], "reference": item["norm_text"]}
+        yield {**item["audio"], "reference_norm": item["reference_norm"], 'reference': item[text_column]}
+
+
+def clean_filename(filename: str):
+    return filename.replace(os.path.sep, '_')
 
 
 def main(args):
     logger.info(f'running evaluation script with following parameters: {args}')
-    logger.info(f'using following text normalier: {whisper_norm}')
+    logger.info(f'using following text normalier: {text_normalizer}')
 
     batch_size = args.batch_size
     whisper_asr = pipeline("automatic-speech-recognition", model=args.model_id, device=args.device)
@@ -72,40 +79,64 @@ def main(args):
     # Only uncomment for debugging
     dataset = dataset.take(args.max_eval_samples)
 
+    # TODO: probably no need in cast, because pipelien migh handle resampling internally. need to check
     dataset = dataset.cast_column("audio", Audio(sampling_rate=16000))
     dataset = dataset.map(normalise, fn_kwargs=dict(text_column=args.text_column))
-    dataset = dataset.filter(is_target_text_in_range, input_columns=["norm_text"])
+    dataset = dataset.filter(is_target_text_in_range, input_columns=["reference_norm"])
 
     predictions = []
+    predictions_norm = []
     references = []
+    references_norm = []
+    audio_paths = []
 
     logger.info('running inference')
-    for out in whisper_asr(data(dataset), batch_size=batch_size):
-        predictions.append(whisper_norm(out["text"]))
+    for out in whisper_asr(data(dataset, text_column=args.text_column), batch_size=batch_size):
+        predictions.append(out["text"])
+        predictions_norm.append(text_normalizer(out["text"]))
         references.append(out["reference"][0])
+        references_norm.append(out["reference_norm"][0])
+        audio_paths.append(out['path'][0])
 
     logger.info('computing metrics')
-    wer = wer_metric.compute(references=references, predictions=predictions)
+    wer = wer_metric.compute(references=references_norm, predictions=predictions_norm)
     wer = wer * 100
 
     logger.info('metrics computed')
     logger.info(f'WER: {wer}')
 
-    evaluate.push_to_hub(
-        model_id=args.model_id,
-
-        metric_value=wer,
-        metric_type="wer",
-        metric_name="WER",
-
-        dataset_name=args.dataset,
-        dataset_type=args.dataset,
-        dataset_config=args.config,
-        dataset_split=args.split,
-        
-        task_type="automatic-speech-recognition",
-        task_name="Automatic Speech Recognition"
-    )
+    if args.save_predictions is True:
+        preds_fp = f'preds_{args.dataset}_{args.config}_{args.split}_{now_str}.tsv'
+        preds_fp = clean_filename(preds_fp)
+        logger.info(f'saving predictions to: "{preds_fp}"')
+        preds_df = pd.DataFrame({
+            'audio_path': audio_paths, 
+            'prediction_norm': predictions_norm, 'reference_norm': references_norm,
+            'prediction': predictions, 'reference': references, 
+        })
+        preds_df.to_csv(preds_fp, sep='\t', index=False)
+    else:
+        logger.info('save_predictions is False. will not save predictions to a file')
+
+    if args.push_to_hub is True:
+        logger.info(f'updating model card and pushing to HuggingFace Hub')
+        evaluate.push_to_hub(
+            model_id=args.model_id,
+
+            metric_value=wer,
+            metric_type="wer",
+            metric_name="WER",
+
+            dataset_name=args.dataset,
+            dataset_type=args.dataset,
+            dataset_config=args.config,
+            dataset_split=args.split,
+            
+            task_type="automatic-speech-recognition",
+            task_name="Automatic Speech Recognition"
+        )
+    else:
+        logger.info('push_to_hub is False. will not update model card and push to HuggingFace Hub')
 
 
 if __name__ == "__main__":
@@ -171,6 +202,18 @@ if __name__ == "__main__":
         required=True,
         help="Two letter language code for the transcription language, e.g. use 'en' for English.",
     )
+    parser.add_argument(
+        '--push_to_hub',
+        type=bool,
+        default=True,
+        help="Whether to update model card and push changes to HuggingFace Hub"
+    )
+    parser.add_argument(
+        '--save_predictions',
+        type=bool,
+        default=True,
+        help="Whether to store predictions and target transcriptions to a file"
+    )
     args = parser.parse_args()
 
     main(args)