bugs

app.py

@@ -5,43 +5,55 @@ import re
 # https://stackoverflow.com/questions/22800401/how-to-capitalize-the-first-letter-of-every-sentence
 def cap(match):
     return(match.group().capitalize())
-    
-
-def predict(brakes, transcript):
 
+def remove_filler_words(transcript):
+    
+    # preserve line brakes
+    transcript_hash = " # ".join(transcript.strip().splitlines())
+    print('transcript_hash')
+    print(transcript_hash)
     # preprocess the text by removing filler words
     # Define a list of filler words to remove
     filler_words = ["um", "uh", "hmm", "ha", "er", "ah", "yeah"]
-    words = transcript.split()
+    words = transcript_hash.split()
     clean_words = [word for word in words if word.lower() not in filler_words]
-    input_text = " ".join(clean_words)
+    input_text_clean = ' '.join(clean_words)
+    # restore the line brakes
+    input_text= input_text_clean.replace(' # ','\n')
+    return input_text
     # Define a regular expression pattern that matches any filler word surrounded by whitespace or punctuation
     #pattern = r"(?<=\s|\b)(" + "|".join(fillers) + r")(?=\s|\b)"
     # Use re.sub to replace the filler words with empty strings
-    #clean_input_text = re.sub(pattern, "", input_text)
+    #clean_input_text = re.sub(pattern, "", input_text)    
+
+def predict(brakes, transcript):
 
+    input_text = remove_filler_words(transcript)
     # Do the punctuation restauration
     model = PunctuationModel()
     output_text = model.restore_punctuation(input_text)
-    
-    srt_file = input_text
-    punctuated = output_text
 
     # if any of the line brake methods are implemented,
     # return the text as a single line
     pcnt_file_cr = output_text
 
     if 'textlines' in brakes:
-        # restore the carrige returns 
-        srt_file_strip=srt_file.strip()
-        srt_file_sub=re.sub('\s*\n\s*','# ',srt_file_strip)
-        srt_file_array=srt_file_sub.split(' ')
-        pcnt_file_array=punctuated.split(' ')
-    
+
+        # preserve line brakes
+        srt_file_hash = '# '.join(input_text.strip().splitlines())
+        #srt_file_sub=re.sub('\s*\n\s*','# ',srt_file_strip)
+        srt_file_array=srt_file_hash.split()
+        pcnt_file_array=output_text.split()
+
+        print('pcnt_file_array')
+        print(pcnt_file_array)
+        print('srt_file_array')
+        print(srt_file_array)
         # goal: restore the break points i.e. the same number of lines as the srt file
         # this is necessary, because each line in the srt file corresponds to a frame from the video
         if len(srt_file_array)!=len(pcnt_file_array):
             return "AssertError: The length of the transcript and the punctuated file should be the same: ",len(srt_file_array),len(pcnt_file_array)
+        
         pcnt_file_array_hash = []
         for idx, item in enumerate(srt_file_array):
             if item.endswith('#'):
@@ -73,7 +85,7 @@ Model restores punctuation and case i.e. of the following punctuations -- [! ? .
     examples = [['sentences', "my name is clara i live in berkeley california"]]
 
     interface = gr.Interface(fn = predict,
-                         inputs = [gr.Radio(["no breaks","sentences", "textlines"], default="no breaks", label="line brakes"),
+                         inputs = [gr.Radio(["no brakes","sentences", "textlines"], value="no brakes", label="line brakes"),
                                    "text"],
                          outputs = ["text"],
                          title = title,

metrics.py

@@ -0,0 +1,60 @@
+# Import nltk library for natural language processing
+import nltk
+
+from transformers import AutoTokenizer
+
+# Define a function that takes some text as input and returns the number of tokens
+def token_count(text):
+    # Import the Encoder class from bpe
+    from bpe import Encoder
+    # Create an encoder object with a vocabulary size of 10
+    encoder = Encoder(vocab_size=14735746)
+
+    # Train the encoder on the text
+    encoder.fit(text.split())
+
+    # Encode the text into tokens
+    tokens = encoder.tokenize(text)
+
+    # Return the number of tokens
+    return tokens
+
+def num_tokens(text):
+
+    tokenizer = AutoTokenizer.from_pretrained("gpt2")
+
+    token_ids = tokenizer.encode(text)
+
+    token_size = len(token_ids)
+
+    return token_size
+
+def num_words(text):
+    sentences = nltk.sent_tokenize(text)
+    # Tokenize each sentence into words using nltk.word_tokenize()
+    words = []
+    for sentence in sentences:
+        words.extend(nltk.word_tokenize(sentence))
+    
+    num_words = len(words)
+
+    return num_words
+
+def num_sentences(text):
+    # Tokenize the text into sentences using nltk.sent_tokenize()
+    sentences = nltk.sent_tokenize(text)
+    num_sentences = len(sentences)
+    return num_sentences
+
+
+def num_chars(text):
+    num_characters = len(text)
+    return num_characters
+
+
+# Print out the results
+# print(f"Number of sentences: {num_sentences}")
+# print(f"Number of words: {num_words}")
+# print(f"Number of tokens: {num_tokens}")
+# print(f"Number of trans_tokens: {trans_tokens}")
+# print(f"Number of characters: {num_characters}")