Initial commmit

.gitattributes

@@ -22,6 +22,8 @@
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text
+*.keras filter=lfs diff=lfs merge=lfs -text
+functions/*.keras filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +1,12 @@
 ---
 title: Sponsoredbye
-emoji: 😻
-colorFrom: green
-colorTo: blue
+emoji: 💎
+colorFrom: blue
+colorTo: purple
 sdk: gradio
-sdk_version: 4.31.5
+sdk_version: 4.31.4
 app_file: app.py
-pinned: false
+pinned: true
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

RNN_model.keras

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:642ec9499996ca6dcd3c8f2874ae3c5d9ca0095064d2f8faae1f12b2fea1e020
+size 3974964

app.py

@@ -0,0 +1,47 @@
+from os import pipe
+import re
+import gradio as gr
+from functions.punctuation import punctuate
+from functions.model_infer import predict_from_document
+from functions.convert_time import match_mask_and_transcript
+
+
+title = "sponsoredBye - never listen to sponsors again"
+description = "Sponsored sections in videos are annoying and take up a lot of time. Improve your YouTube watching experience, by filling in the youtube url and figure out what segments to skip."
+article = "Check out [the original Rick and Morty Bot](https://huggingface.co/spaces/kingabzpro/Rick_and_Morty_Bot) that this demo is based off of."
+
+
+def pipeline(video_url):
+    video_id = video_url.split("?v=")[-1]
+    punctuated_text, transcript = punctuate(video_id)
+    sentences = re.split(r"[\.\!\?]\s", punctuated_text)
+    classification, probs = predict_from_document(sentences)
+    #    return punctuated_text
+    times, timestamps = match_mask_and_transcript(sentences, transcript, classification)
+    return [{"begin": time[0], "end": time[1]} for time in times]
+    #    return [
+    #        {
+    #            "start": "12:05",
+    #            "end": "12:52",
+    #            "classification": str(classification),
+    #            "probabilities": probs,
+    #            "times": times,
+    #            "timestamps": timestamps,
+    #        }
+    #    ]
+
+
+# print(pipeline("VL5M5ZihJK4"))
+demo = gr.Interface(
+    fn=pipeline,
+    title=title,
+    description=description,
+    inputs="text",
+    #    outputs=gr.Label(num_top_classes=3),
+    outputs="json",
+    examples=[
+        "https://www.youtube.com/watch?v=UjtOGPJ0URM",
+        "https://www.youtube.com/watch?v=TrZyuCh9df0",
+    ],
+)
+demo.launch(share=True)

functions/RNN_model.keras

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:642ec9499996ca6dcd3c8f2874ae3c5d9ca0095064d2f8faae1f12b2fea1e020
+size 3974964

functions/convert_time.py

@@ -0,0 +1,52 @@
+import re
+from thefuzz import fuzz
+import numpy as np
+
+
+def match_mask_and_transcript(split_punct, transcript, classification):
+    """
+    Input:
+        split_punct: the punctuated text, split on ?/!/.\s,
+        transcript: original transcript with timestamps
+        classification: classification object (list of numbers 0,1)
+    Output: times
+    """
+
+    # Get the sponsored part
+    sponsored_segment = []
+    for i, val in enumerate(classification):
+        if val == 1:
+            sponsored_segment.append(split_punct[i])
+
+    segment = " ".join(sponsored_segment)
+    sim_scores = list()
+
+    # Check the similarity scores between the sponsored part and the transcript parts
+    for elem in transcript:
+        sim_scores.append(fuzz.partial_ratio(segment, elem["text"]))
+
+    # Get the scores and check if they are above mean + 2*stdev
+    scores = np.array(sim_scores)
+    timestamp_mask = (scores > np.mean(scores) + np.std(scores) * 2).astype(int)
+    timestamps = [
+        (transcript[i]["start"], transcript[i]["duration"])
+        for i, elem in enumerate(timestamp_mask)
+        if elem == 1
+    ]
+
+    # Get the timestamp segments
+    times = []
+    current = -1
+    current_time = 0
+    for elem in timestamps:
+        # Threshold of 5 to see if it is a jump to another segment (also to make sure smaller segments are added together
+        if elem[0] > (current_time + 15):
+            current += 1
+            times.append((elem[0], elem[0] + elem[1]))
+            current_time = elem[0] + elem[1]
+        else:
+            times[current] = (times[current][0], elem[0] + elem[1])
+            current_time = elem[0] + elem[1]
+
+    return_times = [x for x in times if (x[1] - x[0]) > 10]
+    return return_times, timestamps

functions/model_infer.py

@@ -0,0 +1,49 @@
+from keras.preprocessing.sequence import pad_sequences
+import numpy as np
+import re
+
+# import tensorflow as tf
+import os
+import requests
+from keras.models import load_model
+
+headers = {"Authorization": f"Bearer {os.environ['HF_Token']}"}
+
+model = load_model("./RNN_model.keras")
+
+
+def query_embeddings(texts):
+    payload = {"inputs": texts, "options": {"wait_for_model": True}}
+
+    model_id = "sentence-transformers/sentence-t5-base"
+    API_URL = (
+        f"https://api-inference.huggingface.co/pipeline/feature-extraction/{model_id}"
+    )
+    response = requests.post(API_URL, headers=headers, json=payload)
+    return response.json()
+
+
+def preprocess(sentences):
+    max_len = 1682
+    embeddings = query_embeddings(sentences)
+
+    if len(sentences) > max_len:
+        X = embeddings[:max_len]
+    else:
+        X = embeddings
+    X_padded = pad_sequences([X], maxlen=max_len, dtype="float32", padding="post")
+    return X_padded
+
+
+def predict_from_document(sentences):
+    preprop = preprocess(sentences)
+    prediction = model.predict(preprop)
+    # Set the prediction threshold to 0.8 instead of 0.5, now use mean
+    if np.mean(prediction) < 0.5:
+        output = (prediction.flatten()[: len(sentences)] >= 0.5).astype(int)
+    else:
+        output = (
+            prediction.flatten()[: len(sentences)]
+            >= np.mean(prediction) * 1.20  # + np.std(prediction)
+        ).astype(int)
+    return output, prediction.flatten()[: len(sentences)]

functions/punctuation.py

@@ -0,0 +1,58 @@
+import requests
+from youtube_transcript_api import YouTubeTranscriptApi
+import json
+import os
+
+headers = {
+    "Authorization": f"Bearer {os.environ['HF_Token']}"
+}  # NOTE: put this somewhere else
+
+
+def retrieve_transcript(vid_id):
+    try:
+        transcript = YouTubeTranscriptApi.get_transcript(vid_id)
+        return transcript
+    except Exception as e:
+        return None
+
+
+def split_transcript(transcript, chunk_size=40):
+    sentences = []
+    for i in range(0, len(transcript), chunk_size):
+        to_add = [x["text"] for x in transcript[i : i + chunk_size]]
+        sentences.append(" ".join(to_add))
+    return sentences
+
+
+def query_punctuation(splits):
+    payload = {"inputs": splits}
+    API_URL = "https://api-inference.huggingface.co/models/oliverguhr/fullstop-punctuation-multilang-large"
+    response = requests.post(API_URL, headers=headers, json=payload)
+    return response.json()
+
+
+def parse_output(output, comb):
+    total = []
+
+    # loop over the response from the huggingface api
+    for i, o in enumerate(output):
+        added = 0
+        tt = comb[i]
+        for elem in o:
+            # Loop over the output chunks and add the . and ?
+            if elem["entity_group"] not in ["0", ",", ""]:
+                split = elem["end"] + added
+                tt = tt[:split] + elem["entity_group"] + tt[split:]
+                added += 1
+        total.append(tt)
+    return " ".join(total)
+
+
+def punctuate(video_id):
+    transcript = retrieve_transcript(video_id)
+    splits = split_transcript(
+        transcript
+    )  # Get the transcript from the YoutubeTranscriptApi
+    resp = query_punctuation(splits)  # Get the response from the Inference API
+    punctuated_transcript = parse_output(resp, splits)
+    return punctuated_transcript, transcript

requirements.txt

@@ -0,0 +1,6 @@
+youtube_transcript_api
+thefuzz
+numpy
+tensorflow==2.15
+keras
+keras-nlp