Adding skeleton highlights creator

README.md

@@ -10,4 +10,13 @@ pinned: false
 short_description: Soccer Word-Based-Search Football Highlights Generator
 ---
 
+This is the main app module for a word-search-based football highlights extractor.
+
+This starts out with a gradio web app to upload a video.
+The video then gets labelled using a variety of input + a labeller LLM hosted on TogetherAI.
+Then the app calls a model space hosted on HuggingFace and embeds the video and the labels with InternV
+Then through the gradio web app UI search, we can query the mf.
+
+I get by with a little help from my friends
+
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,27 @@
+import gradio as gr
+import json
+from src.pipeline import run_pipeline, search_highlights
+
+def extract(video_file, game_card_str):
+    video_file.save("uploaded.mp4")
+    result = run_pipeline("uploaded.mp4", game_card_str)
+    return json.dumps(result, indent=2)
+
+def search(query):
+    return "\n".join(search_highlights(query))
+
+with gr.Blocks() as demo:
+    gr.Markdown("# Soccer Highlight Extractor")
+
+    with gr.Tab("Extract Highlights"):
+        video = gr.File(label="Upload Video")
+        game_card = gr.Textbox(label="Paste Game Card (JSON)", lines=10)
+        result = gr.Textbox(label="Pipeline Output")
+        gr.Button("Run Extraction").click(extract, [video, game_card], result)
+
+    with gr.Tab("Search Highlights"):
+        query = gr.Textbox(label="Search Query")
+        output = gr.Textbox(label="Search Results")
+        gr.Button("Search").click(search, query, output)
+
+demo.launch()

data/README.md

@@ -0,0 +1 @@
+Placeholder to save folder

requirements.txt

@@ -0,0 +1,12 @@
+gradio
+whisper
+roboflow
+requests
+pinecone-client
+ffmpeg-python
+transformers
+torch
+decord
+numpy
+python-dotenv
+together

src/embedder.py

@@ -0,0 +1,28 @@
+import torch
+import numpy as np
+from transformers import AutoProcessor, AutoModel
+import decord
+
+class InternVLEmbedder:
+    def __init__(self):
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        self.model = AutoModel.from_pretrained("OpenGVLab/InternVL2_5-8B-MPO").to(self.device)
+        self.processor = AutoProcessor.from_pretrained("OpenGVLab/InternVL2_5-8B-MPO")
+
+    def embed_video(self, video_path):
+        vr = decord.VideoReader(video_path)
+        frames = np.stack([vr[i].asnumpy() for i in np.linspace(0, len(vr)-1, 8).astype(int)])
+        tensor = torch.tensor(frames).permute(0, 3, 1, 2).unsqueeze(0).to(self.device)
+
+        with torch.no_grad():
+            video_vector = self.model.get_video_features(tensor).squeeze(0).cpu().numpy()
+
+        return video_vector / np.linalg.norm(video_vector)
+
+    def embed_text(self, text):
+        inputs = self.processor(text=[text], return_tensors="pt").to(self.device)
+
+        with torch.no_grad():
+            text_vector = self.model.get_text_features(**inputs).squeeze(0).cpu().numpy()
+
+        return text_vector / np.linalg.norm(text_vector)

src/event_card.py

@@ -0,0 +1,7 @@
+import json
+
+def parse_game_card(game_card_str):
+    try:
+        return json.loads(game_card_str)
+    except json.JSONDecodeError:
+        return {"description": game_card_str}

src/labeler.py

@@ -0,0 +1,35 @@
+import os
+from together import Together
+
+class TogetherLLMLabeler:
+    def __init__(self):
+        self.client = Together(api_key=os.getenv("TOGETHER_API_KEY"))
+
+    def generate_label(self, game_card, transcript, spatial_context, frame_urls):
+        prompt = f"""
+        Game Information:
+        {game_card}
+
+        Commentary:
+        {transcript}
+
+        Spatial Context (object detections per frame):
+        {spatial_context}
+
+        Instructions:
+        - Summarize this event in factual soccer terminology.
+        - Focus on the play's significance to the score.
+        - Avoid exaggeration.
+        """
+
+        images = [{"type": "image_url", "image_url": {"url": url}} for url in frame_urls]
+
+        content = [{"type": "text", "text": prompt}] + images
+
+        response = self.client.chat_completions.create(
+            model="meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo",
+            messages=[{"role": "user", "content": content}],
+            max_tokens=200
+        )
+
+        return response.choices[0].message["content"].strip()

src/pinecone_store.py

@@ -0,0 +1,29 @@
+import pinecone
+import os
+
+class PineconeStore:
+    def __init__(self):
+        api_key = os.getenv("PINECONE_API_KEY")
+        environment = os.getenv("PINECONE_ENV")
+        pinecone.init(api_key=api_key, environment=environment)
+
+        self.index_name = "soccer-highlights"
+        if self.index_name not in pinecone.list_indexes():
+            pinecone.create_index(
+                name=self.index_name,
+                dimension=1024,  # Ensure this matches your embedding model's output dimension
+                metric="cosine"  # Choose the appropriate metric (e.g., cosine, euclidean)
+            )
+        self.index = pinecone.Index(self.index_name)
+
+    def upsert(self, id, vector, metadata):
+        self.index.upsert([(id, vector.tolist(), metadata)])
+
+    def query(self, vector, filter_key, top_k):
+        return [
+            m["metadata"] for m in self.index.query(
+                vector.tolist(),
+                top_k=top_k,
+                include_metadata=True
+            )["matches"] if filter_key in m["id"]
+        ]

src/pipeline.py

@@ -0,0 +1,63 @@
+from src.segmenter import detect_event_segments
+from src.transcriber import transcribe_video
+from src.event_card import parse_game_card
+from src.labeler import TogetherLLMLabeler
+from src.embedder import InternVLEmbedder
+from src.pinecone_store import PineconeStore
+from src.utils import (
+    extract_key_frames, save_frames_locally,
+    generate_frame_urls, match_transcript_to_events,
+    clip_video_segment
+)
+
+labeler = TogetherLLMLabeler()
+embedder = InternVLEmbedder()
+pinecone = PineconeStore()
+
+def run_pipeline(video_path, game_card_str):
+    game_card = parse_game_card(game_card_str)
+    transcript = transcribe_video(video_path)
+    events = detect_event_segments(video_path)
+
+    matched_events = match_transcript_to_events(events, transcript)
+
+    results = []
+
+    for idx, event in enumerate(matched_events):
+        event_id = f"event-{idx}"
+
+        frames = extract_key_frames(video_path, event['start_sec'], event['end_sec'])
+        frame_paths = save_frames_locally(frames, event_id)
+        frame_urls = generate_frame_urls(frame_paths)
+
+        label = labeler.generate_label(
+            game_card=game_card,
+            transcript=event['transcript'],
+            spatial_context=event['frames'],
+            frame_urls=frame_urls
+        )
+
+        clip_path = clip_video_segment(video_path, event['start_sec'], event['end_sec'], event_id)
+
+        video_vector = embedder.embed_video(clip_path)
+        text_vector = embedder.embed_text(label)
+
+        metadata = {
+            "start_sec": event['start_sec'],
+            "end_sec": event['end_sec'],
+            "label": label
+        }
+
+        pinecone.upsert(f"{event_id}-video", video_vector, metadata)
+        pinecone.upsert(f"{event_id}-text", text_vector, metadata)
+
+        results.append(metadata)
+
+    return {"events": results}
+
+def search_highlights(query, top_k=5):
+    query_vector = embedder.embed_text(query)
+    results = pinecone.query(query_vector, filter_key="text", top_k=top_k)
+    return [
+        f"{r['label']} ({r['start_sec']}s - {r['end_sec']}s)" for r in results
+    ]

src/segmenter.py

@@ -0,0 +1,49 @@
+import cv2
+import os
+from roboflow import Roboflow
+
+## When the ball is no longer detected, we start a new segment
+
+def detect_event_segments(video_path, confidence=0.4):
+    rf = Roboflow(api_key=os.getenv("ROBOFLOW_API_KEY"))
+    project = rf.workspace().project("soccer-event-detection")
+    model = project.version(1).model
+
+    cap = cv2.VideoCapture(video_path)
+    fps = cap.get(cv2.CAP_PROP_FPS)
+
+    events = []
+    active_event = None
+    frame_data = []
+
+    while cap.isOpened():
+        ret, frame = cap.read()
+        if not ret:
+            break
+
+        frame_number = int(cap.get(cv2.CAP_PROP_POS_FRAMES))
+        detections = model.predict(frame, confidence=confidence).json().get('predictions', [])
+        frame_data.append({"frame": frame_number, "objects": detections})
+
+        ball_detected = any(obj['class'] == 'ball' for obj in detections)
+        goal_area_activity = any(obj['class'] == 'goal' for obj in detections) and ball_detected
+
+        if goal_area_activity and active_event is None:
+            active_event = {"start_frame": frame_number, "frames": []}
+
+        if active_event:
+            active_event["frames"].append(frame_data[-1])
+
+        if active_event and not ball_detected:
+            active_event["end_frame"] = frame_number
+            events.append(active_event)
+            active_event = None
+
+    cap.release()
+
+    # Convert frames to timestamps
+    for event in events:
+        event['start_sec'] = event['start_frame'] / fps
+        event['end_sec'] = event['end_frame'] / fps
+
+    return events

src/transcriber.py

@@ -0,0 +1,6 @@
+import whisper
+
+def transcribe_video(video_path, model_size="large"):
+    model = whisper.load_model(model_size)
+    result = model.transcribe(video_path)
+    return [{"start_sec": seg["start"], "end_sec": seg["end"], "text": seg["text"]} for seg in result["segments"]]

src/utils.py

@@ -0,0 +1,56 @@
+import cv2
+import os
+
+def extract_key_frames(video_path, start_sec, end_sec):
+    cap = cv2.VideoCapture(video_path)
+    fps = cap.get(cv2.CAP_PROP_FPS)
+
+    start_frame = int(start_sec * fps)
+    end_frame = int(end_sec * fps)
+    mid_frame = (start_frame + end_frame) // 2
+
+    frames = []
+    for frame_number in [start_frame, mid_frame, end_frame]:
+        cap.set(cv2.CAP_PROP_POS_FRAMES, frame_number)
+        ret, frame = cap.read()
+        if ret:
+            frames.append(frame)
+
+    cap.release()
+    return frames
+
+def save_frames_locally(frames, event_id):
+    os.makedirs("/data", exist_ok=True)
+    frame_paths = []
+    for idx, frame in enumerate(frames):
+        path = f"/data/frame_{event_id}_{idx}.jpg"
+        cv2.imwrite(path, frame)
+        frame_paths.append(path)
+    return frame_paths
+
+def generate_frame_urls(frame_paths):
+    base_url = os.getenv("SPACE_URL", "http://localhost:8000")
+    return [f"{base_url}/data/{os.path.basename(path)}" for path in frame_paths]
+
+def match_transcript_to_events(events, transcript):
+    for event in events:
+        matched_lines = [
+            line["text"] for line in transcript
+            if line["start_sec"] <= event["end_sec"] and line["end_sec"] >= event["start_sec"]
+        ]
+        event["transcript"] = "\n".join(matched_lines) or "(No matching commentary)"
+    return events
+
+def clip_video_segment(video_path, start_sec, end_sec, event_id):
+    output_path = f"/data/clip_{event_id}.mp4"
+    duration = end_sec - start_sec
+
+    command = [
+        "ffmpeg", "-y",
+        "-ss", str(start_sec),
+        "-i", video_path,
+        "-t", str(duration),
+        "-c", "copy", output_path
+    ]
+    os.system(" ".join(command))
+    return output_path