upd eval code

src/bash_runners/{run_eval_cv11.sh → eval_cv11_test.sh}

@@ -1,9 +1,10 @@
-python src/run_eval_whisper_streaming \
-	--model_id="." \
+python src/run_eval_whisper_streaming.py \
+	--model_id="$1" \
 	--language="be" \
 	--dataset="mozilla-foundation/common_voice_11_0" \
     --config="be" \
     --split="test" \
+    --text_column="sentence" \
     --device="0" \
     --batch_size="32" \
     --streaming="True"

src/bash_runners/eval_fleurs_test.sh

@@ -0,0 +1,10 @@
+python src/run_eval_whisper_streaming.py \
+	--model_id="$1" \
+	--language="be" \
+	--dataset="google/fleurs" \
+    --config="be_by" \
+    --split="test" \
+    --text_column="transcription" \
+    --device="0" \
+    --batch_size="32" \
+    --streaming="True"

src/run_eval_whisper_streaming.py

@@ -1,4 +1,7 @@
 import argparse
+import logging
+import sys
+import datetime
 
 from transformers import pipeline
 from transformers.models.whisper.english_normalizer import BasicTextNormalizer
@@ -8,7 +11,23 @@ import evaluate
 from belarusian_text_normalizer import BelarusianTextNormalizer
 
 
+now_str = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
+
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(
+    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+    datefmt="%m/%d/%Y %H:%M:%S",
+    handlers=[
+        logging.StreamHandler(sys.stdout),
+        logging.FileHandler(filename=f'eval_{now_str}.log', mode='w')
+    ],
+)
+logger.setLevel(logging.INFO)
+
+
 wer_metric = evaluate.load("wer")
+whisper_norm = BelarusianTextNormalizer()
 
 
 def is_target_text_in_range(ref):
@@ -18,30 +37,9 @@ def is_target_text_in_range(ref):
         return ref.strip() != ""
 
 
-def get_text(sample):
-    if "text" in sample:
-        return sample["text"]
-    elif "sentence" in sample:
-        return sample["sentence"]
-    elif "normalized_text" in sample:
-        return sample["normalized_text"]
-    elif "transcript" in sample:
-        return sample["transcript"]
-    elif "transcription" in sample:
-        return sample["transcription"]
-    else:
-        raise ValueError(
-            f"Expected transcript column of either 'text', 'sentence', 'normalized_text' or 'transcript'. Got sample of "
-            ".join{sample.keys()}. Ensure a text column name is present in the dataset."
-        )
-
-
-whisper_norm = BelarusianTextNormalizer()
-
-
-def normalise(batch):
-    batch["norm_text"] = whisper_norm(get_text(batch))
-    return batch
+def normalise(sample, text_column: str):
+    sample["norm_text"] = whisper_norm(sample[text_column])
+    return sample
 
 
 def data(dataset):
@@ -50,10 +48,11 @@ def data(dataset):
 
 
 def main(args):
+    logger.info(f'running evaluation script with following parameters: {args}')
+    logger.info(f'using following text normalier: {whisper_norm}')
+
     batch_size = args.batch_size
-    whisper_asr = pipeline(
-        "automatic-speech-recognition", model=args.model_id, device=args.device
-    )
+    whisper_asr = pipeline("automatic-speech-recognition", model=args.model_id, device=args.device)
 
     whisper_asr.model.config.forced_decoder_ids = (
         whisper_asr.tokenizer.get_decoder_prompt_ids(
@@ -61,6 +60,7 @@ def main(args):
         )
     )
 
+    logger.info('loading dataset')
     dataset = load_dataset(
         args.dataset,
         args.config,
@@ -73,30 +73,36 @@ def main(args):
     dataset = dataset.take(args.max_eval_samples)
 
     dataset = dataset.cast_column("audio", Audio(sampling_rate=16000))
-    dataset = dataset.map(normalise)
+    dataset = dataset.map(normalise, fn_kwargs=dict(text_column=args.text_column))
     dataset = dataset.filter(is_target_text_in_range, input_columns=["norm_text"])
 
     predictions = []
     references = []
 
-    # run streamed inference
+    logger.info('running inference')
     for out in whisper_asr(data(dataset), batch_size=batch_size):
         predictions.append(whisper_norm(out["text"]))
         references.append(out["reference"][0])
 
+    logger.info('computing metrics')
     wer = wer_metric.compute(references=references, predictions=predictions)
-    wer = round(100 * wer, 2)
+    wer = wer * 100
+
+    logger.info('metrics computed')
+    logger.info(f'WER: {wer}')
 
-    print("WER:", wer)
     evaluate.push_to_hub(
         model_id=args.model_id,
+
         metric_value=wer,
         metric_type="wer",
         metric_name="WER",
+
         dataset_name=args.dataset,
         dataset_type=args.dataset,
-        dataset_split=args.split,
         dataset_config=args.config,
+        dataset_split=args.split,
+        
         task_type="automatic-speech-recognition",
         task_name="Automatic Speech Recognition"
     )
@@ -129,7 +135,12 @@ if __name__ == "__main__":
         default="test",
         help="Split of the dataset. *E.g.* `'test'`",
     )
-
+    parser.add_argument(
+        "--text_column",
+        type=str,
+        required=True,
+        help="Dataset column name containing target transcription of an audiofile"
+    )
     parser.add_argument(
         "--device",
         type=int,