Add model

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "roberta-base",
+  "architectures": [
+    "RobertaForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.28.0",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 50265
+}

eval_results.txt

@@ -0,0 +1,12 @@
+accuracy = 0.9996004794246903
+auprc = 0.9997711321330485
+auroc = 0.9999888917688007
+eval_loss = 0.002696692644352147
+f1 = 0.9955555555555555
+fn = 1
+fp = 0
+mcc = 0.9953571764069896
+precision = 0.9911504424778761
+recall = 1.0
+tn = 2390
+tp = 112

handler.py

@@ -0,0 +1,138 @@
+from simpletransformers.classification import ClassificationModel, ClassificationArgs
+from typing import Dict, List, Any
+import pandas as pd
+import webvtt
+from datetime import datetime
+import torch
+import spacy
+
+nlp = spacy.load("en_core_web_sm")
+tokenizer = nlp.tokenizer
+token_limit = 200
+
+class Utterance(object):
+
+    def __init__(self, starttime, endtime, speaker, text, 
+                 idx, prev_utterance, prev_prev_utterance):
+        self.starttime = starttime
+        self.endtime = endtime
+        self.speaker = speaker
+        self.text = text
+        self.idx = idx
+        self.prev = prev_utterance
+        self.prev_prev = prev_prev_utterance
+
+class EndpointHandler():
+    def __init__(self, path="."):
+        print("Loading models...")
+        cuda_available = torch.cuda.is_available()
+        self.model = ClassificationModel(
+            "roberta", path, use_cuda=cuda_available
+        )
+
+    def utterance_to_str(self, utterance: Utterance) -> str:
+        # connecting only uses text
+        doc = nlp(utterance.text)
+        if len(doc) > token_limit:
+            return self.handle_long_utterances(doc)
+        return utterance.text
+        
+    def handle_long_utterances(self, doc: str) -> List[str]:
+        split_count = 1
+        total_sent = len([x for x in doc.sents])
+        sent_count = 0
+        token_count = 0
+        split_utterance = ''
+        utterances = []
+        for sent in doc.sents:
+            # add a sentence to split
+            split_utterance = split_utterance + ' ' + sent.text
+            token_count += len(sent)
+            sent_count +=1
+            if token_count >= token_limit or sent_count == total_sent: 
+                # save utterance segment
+                utterances.append(split_utterance)
+
+                # restart count
+                split_utterance = ''
+                token_count = 0
+                split_count += 1
+
+        return utterances
+
+        
+    def convert_time(self, time_str):
+        time = datetime.strptime(time_str, "%H:%M:%S.%f")
+        return 1000 * (3600 * time.hour + 60 * time.minute + time.second) + time.microsecond / 1000
+        
+    def process_vtt_transcript(self, vttfile) -> List[Utterance]:
+        """Process raw vtt file."""
+
+        utterances_list = []
+        text = ""
+        prev_speaker = None
+        prev_start = "00:00:00.000"
+        prev_end = "00:00:00.000"
+        idx = 0
+        prev_utterance = None
+        prev_prev_utterance = None
+        for caption in webvtt.read(vttfile):
+
+            # Get speaker
+            check_for_speaker = caption.text.split(":")
+            if len(check_for_speaker) > 1:  # the speaker was changed or restated
+                speaker = check_for_speaker[0]
+            else:
+                speaker = prev_speaker
+
+            # Get utterance
+            new_text = check_for_speaker[1] if len(check_for_speaker) > 1 else check_for_speaker[0]
+
+            # If speaker was changed, start new batch
+            if (prev_speaker is not None) and (speaker != prev_speaker):
+                utterance = Utterance(starttime=self.convert_time(prev_start),
+                              endtime=self.convert_time(prev_end),
+                              speaker=prev_speaker,
+                              text=text.strip(),
+                              idx=idx,
+                              prev_utterance=prev_utterance,
+                              prev_prev_utterance=prev_prev_utterance)
+
+                utterances_list.append(utterance)
+
+                # Start new batch
+                prev_start = caption.start
+                text = ""
+                prev_prev_utterance = prev_utterance
+                prev_utterance = utterance
+                idx+=1
+            text += new_text + " "
+            prev_end = caption.end
+            prev_speaker = speaker
+
+        # Append last one
+        if prev_speaker is not None:
+            utterance = Utterance(starttime=self.convert_time(prev_start),
+                                  endtime=self.convert_time(prev_end),
+                                  speaker=prev_speaker,
+                                  text=text.strip(),
+                                  idx=idx,
+                                  prev_utterance=prev_utterance,
+                                  prev_prev_utterance=prev_prev_utterance)
+            utterances_list.append(utterance)
+
+        print(utterances_list)
+        return utterances_list
+
+
+    def __call__(self, data_file: str) -> List[Dict[str, Any]]:
+        ''' data_file is a str pointing to filename of type .vtt '''
+
+        utterances_list = []
+        for utterance in self.process_vtt_transcript(data_file):
+            #TODO: filter out to only have SL utterances
+            utterances_list.append(self.utterance_to_str(utterance))
+           
+        predictions, raw_outputs = self.model.predict(utterances_list)
+        
+        return predictions

handler.py~

@@ -0,0 +1,27 @@
+from simpletransformers.classification import ClassificationModel, ClassificationArgs
+from typing import Dict, List, Any
+import pandas as pd
+import webvtt
+from datetime import datetime
+import torch
+import spacy
+
+nlp = spacy.load("en_core_web_sm")
+tokenizer = nlp.tokenizer
+token_limit = 200
+
+class EndpointHandler():
+    def __init__(self, path="."):
+        print("Loading models...")
+        cuda_available = torch.cuda.is_available()
+        self.model = ClassificationModel(
+            "roberta", path, use_cuda=cuda_available
+        )
+
+    def __call__(self, data_file: str) -> List[Dict[str, Any]]:
+        ''' data_file is a str pointing to filename of type .vtt '''
+
+        utterances_list = []   
+        predictions, raw_outputs = self.model.predict(utterances_list)
+        
+        return predictions

model_args.json

@@ -0,0 +1 @@
+{"adafactor_beta1": null, "adafactor_clip_threshold": 1.0, "adafactor_decay_rate": -0.8, "adafactor_eps": [1e-30, 0.001], "adafactor_relative_step": true, "adafactor_scale_parameter": true, "adafactor_warmup_init": true, "adam_betas": [0.9, 0.999], "adam_epsilon": 1e-08, "best_model_dir": "outputs/roberta/connecting_FINAL_MODEL/best_model_all_transcripts", "cache_dir": "outputs/roberta/connecting_FINAL_MODEL/cache", "config": {}, "cosine_schedule_num_cycles": 0.5, "custom_layer_parameters": [], "custom_parameter_groups": [], "dataloader_num_workers": 0, "do_lower_case": false, "dynamic_quantize": false, "early_stopping_consider_epochs": false, "early_stopping_delta": 0, "early_stopping_metric": "eval_loss", "early_stopping_metric_minimize": true, "early_stopping_patience": 3, "encoding": null, "eval_batch_size": 8, "evaluate_during_training": true, "evaluate_during_training_silent": true, "evaluate_during_training_steps": 348, "evaluate_during_training_verbose": false, "evaluate_each_epoch": true, "fp16": false, "gradient_accumulation_steps": 2, "learning_rate": 4e-05, "local_rank": -1, "logging_steps": 50, "loss_type": null, "loss_args": {}, "manual_seed": null, "max_grad_norm": 1.0, "max_seq_length": 512, "model_name": "roberta-base", "model_type": "roberta", "multiprocessing_chunksize": -1, "n_gpu": 1, "no_cache": false, "no_save": false, "not_saved_args": [], "num_train_epochs": 5, "optimizer": "AdamW", "output_dir": "outputs/roberta/connecting_FINAL_MODEL", "overwrite_output_dir": true, "polynomial_decay_schedule_lr_end": 1e-07, "polynomial_decay_schedule_power": 1.0, "process_count": 1, "quantized_model": false, "reprocess_input_data": true, "save_best_model": true, "save_eval_checkpoints": false, "save_model_every_epoch": false, "save_optimizer_and_scheduler": true, "save_steps": 2000, "scheduler": "linear_schedule_with_warmup", "silent": false, "skip_special_tokens": true, "tensorboard_dir": "outputs/roberta/connecting_FINAL_MODEL/tensorboard", "thread_count": null, "tokenizer_name": "roberta-base", "tokenizer_type": null, "train_batch_size": 8, "train_custom_parameters_only": false, "use_cached_eval_features": false, "use_early_stopping": false, "use_hf_datasets": false, "use_multiprocessing": false, "use_multiprocessing_for_evaluation": false, "wandb_kwargs": {"reinit": true}, "wandb_project": "connecting_all_transcripts", "warmup_ratio": 0.06, "warmup_steps": 53, "weight_decay": 0.0, "model_class": "ClassificationModel", "labels_list": [0, 1], "labels_map": {}, "lazy_delimiter": "\t", "lazy_labels_column": 1, "lazy_loading": false, "lazy_loading_start_line": 1, "lazy_text_a_column": null, "lazy_text_b_column": null, "lazy_text_column": 0, "onnx": false, "regression": false, "sliding_window": false, "special_tokens_list": [], "stride": 0.8, "tie_value": 1}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbb13cc66ca8a6202d9de0c2f7f7e060d79764bfe005a269ef31074120544e15
+size 498662069

requirements.txt

@@ -0,0 +1,5 @@
+pandas==1.1.1
+scikit_learn==1.1.3
+scipy==1.7.1
+simpletransformers==0.63.6
+torch==1.6.0

special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

test_run_handler.py

@@ -0,0 +1,13 @@
+from handler import EndpointHandler
+
+# init handler
+my_handler = EndpointHandler(path=".")
+
+# prepare sample payload
+test_payload = 'test.transcript.vtt'
+
+# test the handler
+test_pred=my_handler(test_payload)
+
+# show results
+print("test_pred", test_pred)

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "do_lower_case": false,
+  "eos_token": "</s>",
+  "errors": "replace",
+  "mask_token": "<mask>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "RobertaTokenizer",
+  "trim_offsets": true,
+  "unk_token": "<unk>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7a7eb9bd523295c1f076863d6e51ca1a99210220944f2a8573aa5e6a556be9c
+size 3451

training_progress_scores.csv

@@ -0,0 +1,8 @@
+global_step,train_loss,mcc,tp,tn,fp,fn,auroc,auprc,accuracy,precision,recall,f1,eval_loss
+174,0.336158812046051,0.0,0,2390,0,113,0.9098048654052654,0.499127409757406,0.954854174990012,0.0,0.0,0.0,0.16167433786030394
+348,0.010986842215061188,0.785426877747522,104,2343,47,9,0.9911689561965416,0.9236932727451965,0.9776268477826608,0.9203539823008849,0.6887417218543046,0.7878787878787877,0.08293315396103383
+348,0.07023407518863678,0.785426877747522,104,2343,47,9,0.9911689561965416,0.9236932727451965,0.9776268477826608,0.9203539823008849,0.6887417218543046,0.7878787878787877,0.08293315396103383
+522,0.0005581587320193648,0.9180271585164762,110,2374,16,3,0.9986596067686155,0.9899529045270361,0.9924091090691171,0.9734513274336283,0.873015873015873,0.9205020920502092,0.027681814245459364
+696,0.0003248823923058808,0.9775600823633134,113,2385,5,0,0.9999814862813344,0.9996191363565474,0.9980023971234518,1.0,0.9576271186440678,0.9783549783549783,0.005484335490429146
+696,0.00022003523190505803,0.9775600823633134,113,2385,5,0,0.9999814862813344,0.9996191363565474,0.9980023971234518,1.0,0.9576271186440678,0.9783549783549783,0.005484335490429146
+870,0.00024747333372943103,0.9953571764069896,112,2390,0,1,0.9999888917688007,0.9997711321330485,0.9996004794246903,0.9911504424778761,1.0,0.9955555555555555,0.002696692644352147