add new output features

handler.py

@@ -9,7 +9,6 @@ from utils import MultiHeadModel, BertInputBuilder, get_num_words
 
 import transformers
 from transformers import BertTokenizer, BertForSequenceClassification
-import psutil
 from transformers.utils import logging
 from edu_toolkit import language_analysis
 
@@ -30,7 +29,13 @@ class Utterance:
         self.endtime = endtime
         self.transcript = weakref.ref(transcript) if transcript else None
         self.props = kwargs
+        self.role = None
+        self.word_count = self.get_num_words(text)
+        self.timestamp = [starttime, endtime]
+        self.unit_measure = endtime - starttime
+        self.aggregate_unit_measure = endtime
 
+        # moments
         self.uptake = None
         self.reasoning = None
         self.question = None
@@ -56,6 +61,18 @@ class Utterance:
             **self.props
         }
 
+    def to_talk_timeline_dict(self):
+        return{
+            'speaker': self.speaker,
+            'text': self.text,
+            'role': self.role,
+            'timestamp': self.timestamp,
+            'moments': {'reasoning': self.reasoning, 'questioning': self.question, 'uptake': self.uptake},
+            'unitMeasure': self.unit_measure,
+            'aggregateUnitMeasure': self.aggregate_unit_measure,
+            'wordCount': self.word_count
+        }
+
     def __repr__(self):
         return f"Utterance(speaker='{self.speaker}'," \
                f"text='{self.text}', uid={self.uid}," \
@@ -85,6 +102,56 @@ class Transcript:
     def length(self):
         return len(self.utterances)
 
+    def update_utterance_roles(self, uptake_speaker):
+        for utt in self.utterances:
+            if (utt.speaker == uptake_speaker):
+                utt.role = 'teacher'
+            else:
+                utt.role = 'student'
+
+    def get_talk_distribution_and_length(self, uptake_speaker):
+        if ((uptake_speaker is None)):
+            return None
+        teacher_words = 0
+        student_words = 0
+        for utt in self.utterances:
+            if (utt.speaker == uptake_speaker):
+                utt.role = 'teacher'
+                teacher_words += utt.get_num_words()
+            else:
+                utt.role = 'student'
+                student_words += utt.get_num_words()
+        teacher_percentage = round(
+            (teacher_words / (teacher_words + student_words)) * 100)
+        student_percentage = 100 - teacher_percentage
+        return {'talk_distribution': {'teacher': teacher_percentage, 'student': student_percentage}}, {'talk_length': {'teacher': teacher_words, 'student': student_words}}
+
+    def get_word_cloud_dicts(self):
+        teacher_dict = {}
+        student_dict = {}
+        for utt in self.utterances.get_clean_text():
+            words = (utt.get_clean_text(remove_punct=True)).split(' ')
+            for word in words:
+                if utt.role == 'teacher':
+                    if word not in teacher_dict:
+                        teacher_dict[word] = 0
+                    teacher_dict[word] += 1
+                else:
+                    if word not in student_dict:
+                        student_dict[word] = 0
+                    student_dict[word] += 1
+        dict_list = []
+        for word in teacher_dict.keys():
+            dict_list.append(
+                {'text': word, 'value': teacher_dict[word], 'category': 'teacher'})
+        for word in student_dict.keys():
+            dict_list.append(
+                {'text': word, 'value': student_dict[word], 'category': 'student'})
+        return dict_list
+
+    def get_talk_timeline(self):
+        return [utterance.to_talk_timeline_dict() for utterance in self.utterances]
+
     def to_dict(self):
         return {
             'utterances': [utterance.to_dict() for utterance in self.utterances],
@@ -243,27 +310,30 @@ class EndpointHandler():
             transcript.add_utterance(Utterance(**utt))
 
         print("Running inference on %d examples..." % transcript.length())
-        cpu_percent = psutil.cpu_percent()
+        # cpu_percent = psutil.cpu_percent()
         logging.set_verbosity_info()
-        logger = logging.get_logger("transformers")
-        logger.info(f"CPU Usage before models loaded: {cpu_percent}%")
-        mem_info = psutil.virtual_memory()
-        used_mem = mem_info.used / (1024 ** 3)  # Convert to gigabytes
-        total_mem = mem_info.total / (1024 ** 3)  # Convert to gigabytes
-        logger.info(
-            f"Used Memory before models loaded: {used_mem:.2f} GB, Total RAM: {total_mem:.2f} GB")
+        # logger = logging.get_logger("transformers")
+        # logger.info(f"CPU Usage before models loaded: {cpu_percent}%")
+        # mem_info = psutil.virtual_memory()
+        # used_mem = mem_info.used / (1024 ** 3)  # Convert to gigabytes
+        # total_mem = mem_info.total / (1024 ** 3)  # Convert to gigabytes
+        # logger.info(
+        #     f"Used Memory before models loaded: {used_mem:.2f} GB, Total RAM: {total_mem:.2f} GB")
+
         # Uptake
         uptake_model = UptakeModel(
             self.device, self.tokenizer, self.input_builder)
+        uptake_speaker = params.pop("uptake_speaker", None)
         uptake_model.run_inference(transcript, min_prev_words=params['uptake_min_num_words'],
-                                   uptake_speaker=params.pop("uptake_speaker", None))
-        cpu_percent = psutil.cpu_percent()
-        mem_info = psutil.virtual_memory()
-        used_mem = mem_info.used / (1024 ** 3)  # Convert to gigabytes
-        total_mem = mem_info.total / (1024 ** 3)  # Convert to gigabytes
-        logger.info(
-            f"Used Memory after model 1 loaded: {used_mem:.2f} GB, Total Mem: {total_mem:.2f} GB")
-        logger.info(f"CPU Usage after model 1 loaded: {cpu_percent}%")
+                                   uptake_speaker=uptake_speaker)
+
+        # cpu_percent = psutil.cpu_percent()
+        # mem_info = psutil.virtual_memory()
+        # used_mem = mem_info.used / (1024 ** 3)  # Convert to gigabytes
+        # total_mem = mem_info.total / (1024 ** 3)  # Convert to gigabytes
+        # logger.info(
+        #     f"Used Memory after model 1 loaded: {used_mem:.2f} GB, Total Mem: {total_mem:.2f} GB")
+        # logger.info(f"CPU Usage after model 1 loaded: {cpu_percent}%")
         # del uptake_model
         # cpu_percent = psutil.cpu_percent()
         # mem_info = psutil.virtual_memory()
@@ -275,16 +345,16 @@ class EndpointHandler():
         reasoning_model = ReasoningModel(
             self.device, self.tokenizer, self.input_builder)
         reasoning_model.run_inference(transcript)
-        cpu_percent = psutil.cpu_percent()
-        mem_info = psutil.virtual_memory()
-        used_mem = mem_info.used / (1024 ** 3)  # Convert to gigabytes
-        total_mem = mem_info.total / (1024 ** 3)  # Convert to gigabytes
-        logger.info(
-            f"Used Memory after model 2 loaded: {used_mem:.2f} GB, Total Mem: {total_mem:.2f} GB")
-        logger.info(f"CPU Usage after model 2 loaded: {cpu_percent}%")
-        # print(f"CPU Usage after model 2 loaded: {cpu_percent}%")
-        # del reasoning_model
-        cpu_percent = psutil.cpu_percent()
+        # cpu_percent = psutil.cpu_percent()
+        # mem_info = psutil.virtual_memory()
+        # used_mem = mem_info.used / (1024 ** 3)  # Convert to gigabytes
+        # total_mem = mem_info.total / (1024 ** 3)  # Convert to gigabytes
+        # logger.info(
+        #     f"Used Memory after model 2 loaded: {used_mem:.2f} GB, Total Mem: {total_mem:.2f} GB")
+        # logger.info(f"CPU Usage after model 2 loaded: {cpu_percent}%")
+        # # print(f"CPU Usage after model 2 loaded: {cpu_percent}%")
+        # # del reasoning_model
+        # cpu_percent = psutil.cpu_percent()
         # mem_info = psutil.virtual_memory()
         # used_mem = mem_info.used / (1024 ** 3)  # Convert to gigabytes
         # total_mem = mem_info.total / (1024 ** 3)  # Convert to gigabytes
@@ -295,20 +365,40 @@ class EndpointHandler():
         question_model = QuestionModel(
             self.device, self.tokenizer, self.input_builder)
         question_model.run_inference(transcript)
-        cpu_percent = psutil.cpu_percent()
-        logger.info(f"CPU Usage after model 3 loaded: {cpu_percent}%")
-        mem_info = psutil.virtual_memory()
-        used_mem = mem_info.used / (1024 ** 3)  # Convert to gigabytes
-        total_mem = mem_info.total / (1024 ** 3)  # Convert to gigabytes
-        logger.info(
-            f"Used Memory after model 3 loaded: {used_mem:.2f} GB, Total Mem: {total_mem:.2f} GB")
+        # cpu_percent = psutil.cpu_percent()
+        # logger.info(f"CPU Usage after model 3 loaded: {cpu_percent}%")
+        # mem_info = psutil.virtual_memory()
+        # used_mem = mem_info.used / (1024 ** 3)  # Convert to gigabytes
+        # total_mem = mem_info.total / (1024 ** 3)  # Convert to gigabytes
+        # logger.info(
+        #     f"Used Memory after model 3 loaded: {used_mem:.2f} GB, Total Mem: {total_mem:.2f} GB")
         # print(f"CPU Usage after model 3 loaded: {cpu_percent}%")
         # del question_model
-        cpu_percent = psutil.cpu_percent()
+        # cpu_percent = psutil.cpu_percent()
         # logger.info(f"CPU Usage after model 3 deleted: {cpu_percent}%")
         # mem_info = psutil.virtual_memory()
         # used_mem = mem_info.used / (1024 ** 3)  # Convert to gigabytes
         # total_mem = mem_info.total / (1024 ** 3)  # Convert to gigabytes
         # logger.info(f"Used Memory after model 3 deleted: {used_mem:.2f} GB, Total Mem: {total_mem:.2f} GB")
         # print(f"CPU Usage after model 3 deleted: {cpu_percent}%")
-        return transcript.to_dict()
+        transcript.update_utterance_roles
+        talk_dist, talk_len = transcript.get_talk_distribution_and_length(
+            self, uptake_speaker)
+        talk_timeline = transcript.get_talk_timeline()
+        word_cloud = transcript.get_word_cloud_dicts()
+
+        return transcript.to_dict(), talk_dist, talk_len, talk_timeline, word_cloud
+
+
+# {
+#  "inputs": [
+#    {"uid": "1", "speaker": "Alice", "text": "How much is the fish?" },
+#    {"uid": "2", "speaker": "Bob", "text": "I do not know about the fish. Because you put a long side and it’s a long side. What do you think." },
+# {"uid": "3", "speaker": "Alice", "text": "OK, thank you Bob." }
+#  ],
+#  "parameters": {
+#    "uptake_min_num_words": 5,
+#    "uptake_speaker": "Bob",
+#     "filename": "sample.csv"
+#  }
+# }