second modification

Dockerfile

@@ -22,6 +22,7 @@ COPY app/ ./app/
 COPY config/ ./config/
 COPY data/ ./data/
 COPY grafana/ ./grafana/
+COPY .env ./
 
 # Make port 8501 available to the world outside this container
 EXPOSE 8501

app/data_processor.py

@@ -1,3 +1,4 @@
+import logging
 from minsearch import Index
 from sentence_transformers import SentenceTransformer
 import numpy as np
@@ -5,124 +6,198 @@ from sklearn.metrics.pairwise import cosine_similarity
 import re
 from elasticsearch import Elasticsearch
 import os
+import json
+from transcript_extractor import get_transcript
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
 
 def clean_text(text):
-    # Remove special characters and extra whitespace
-    text = re.sub(r'[^\w\s]', '', text)
-    text = re.sub(r'\s+', ' ', text).strip()
-    return text
+    if not isinstance(text, str):
+        logger.warning(f"Non-string input to clean_text: {type(text)}")
+        return ""
+    cleaned = re.sub(r'[^\w\s.,!?]', ' ', text)
+    cleaned = re.sub(r'\s+', ' ', cleaned).strip()
+    logger.info(f"Cleaned text: '{cleaned[:100]}...'")
+    return cleaned
 
 class DataProcessor:
     def __init__(self, text_fields=["content", "title", "description"], 
-                 keyword_fields=["video_id", "start_time", "author", "upload_date"], 
+                 keyword_fields=["video_id", "author", "upload_date"], 
                  embedding_model="all-MiniLM-L6-v2"):
         self.text_index = Index(text_fields=text_fields, keyword_fields=keyword_fields)
         self.embedding_model = SentenceTransformer(embedding_model)
         self.documents = []
         self.embeddings = []
+        self.index_built = False
+        self.current_index_name = None
         
-        # Use environment variables for Elasticsearch configuration
         elasticsearch_host = os.getenv('ELASTICSEARCH_HOST', 'localhost')
         elasticsearch_port = int(os.getenv('ELASTICSEARCH_PORT', 9200))
         
-        # Initialize Elasticsearch client with explicit scheme
         self.es = Elasticsearch([f'http://{elasticsearch_host}:{elasticsearch_port}'])
+        logger.info(f"DataProcessor initialized with Elasticsearch at {elasticsearch_host}:{elasticsearch_port}")
 
     def process_transcript(self, video_id, transcript_data):
+        if not transcript_data or 'metadata' not in transcript_data or 'transcript' not in transcript_data:
+            logger.error(f"Invalid transcript data for video {video_id}")
+            return None
+
         metadata = transcript_data['metadata']
         transcript = transcript_data['transcript']
 
-        for i, segment in enumerate(transcript):
-            cleaned_text = clean_text(segment['text'])
-            doc = {
-                "video_id": video_id,
-                "content": cleaned_text,
-                "start_time": segment['start'],
-                "duration": segment['duration'],
-                "segment_id": f"{video_id}_{i}",
-                "title": metadata['title'],
-                "author": metadata['author'],
-                "upload_date": metadata['upload_date'],
-                "view_count": metadata['view_count'],
-                "like_count": metadata['like_count'],
-                "comment_count": metadata['comment_count'],
-                "video_duration": metadata['duration']
-            }
-            self.documents.append(doc)
-            self.embeddings.append(self.embedding_model.encode(cleaned_text + " " + metadata['title']))
+        logger.info(f"Processing transcript for video {video_id}")
+        logger.info(f"Number of transcript segments: {len(transcript)}")
+
+        full_transcript = " ".join([segment.get('text', '') for segment in transcript])
+        cleaned_transcript = clean_text(full_transcript)
+
+        if not cleaned_transcript:
+            logger.warning(f"Empty cleaned transcript for video {video_id}")
+            return None
+
+        doc = {
+            "video_id": video_id,
+            "content": cleaned_transcript,
+            "segment_id": f"{video_id}_full",
+            "title": clean_text(metadata.get('title', '')),
+            "author": metadata.get('author', ''),
+            "upload_date": metadata.get('upload_date', ''),
+            "view_count": metadata.get('view_count', 0),
+            "like_count": metadata.get('like_count', 0),
+            "comment_count": metadata.get('comment_count', 0),
+            "video_duration": metadata.get('duration', '')
+        }
+        self.documents.append(doc)
+        self.embeddings.append(self.embedding_model.encode(cleaned_transcript + " " + metadata.get('title', '')))
+
+        logger.info(f"Processed transcript for video {video_id}")
+        return f"video_{video_id}_{self.embedding_model.get_sentence_embedding_dimension()}"
 
     def build_index(self, index_name):
-        self.text_index.fit(self.documents)
+        if not self.documents:
+            logger.error("No documents to index")
+            return None
+
+        logger.info(f"Building index with {len(self.documents)} documents")
+        try:
+            self.text_index.fit(self.documents)
+            self.index_built = True
+            logger.info("Text index built successfully")
+        except Exception as e:
+            logger.error(f"Error building text index: {str(e)}")
+            raise
+
         self.embeddings = np.array(self.embeddings)
         
-        # Create Elasticsearch index
-        if not self.es.indices.exists(index=index_name):
-            self.es.indices.create(index=index_name, body={
-                "mappings": {
-                    "properties": {
-                        "embedding": {"type": "dense_vector", "dims": self.embeddings.shape[1]},
-                        "content": {"type": "text"},
-                        "video_id": {"type": "keyword"},
-                        "segment_id": {"type": "keyword"},
-                        "start_time": {"type": "float"},
-                        "duration": {"type": "float"},
-                        "title": {"type": "text"},
-                        "author": {"type": "keyword"},
-                        "upload_date": {"type": "date"},
-                        "view_count": {"type": "integer"},
-                        "like_count": {"type": "integer"},
-                        "comment_count": {"type": "integer"},
-                        "video_duration": {"type": "text"}
+        try:
+            if not self.es.indices.exists(index=index_name):
+                self.es.indices.create(index=index_name, body={
+                    "mappings": {
+                        "properties": {
+                            "embedding": {"type": "dense_vector", "dims": self.embeddings.shape[1]},
+                            "content": {"type": "text"},
+                            "video_id": {"type": "keyword"},
+                            "segment_id": {"type": "keyword"},
+                            "title": {"type": "text"},
+                            "author": {"type": "keyword"},
+                            "upload_date": {"type": "date"},
+                            "view_count": {"type": "integer"},
+                            "like_count": {"type": "integer"},
+                            "comment_count": {"type": "integer"},
+                            "video_duration": {"type": "text"}
+                        }
                     }
-                }
-            })
+                })
+                logger.info(f"Created Elasticsearch index: {index_name}")
+
+            for doc, embedding in zip(self.documents, self.embeddings):
+                doc_with_embedding = doc.copy()
+                doc_with_embedding['embedding'] = embedding.tolist()
+                self.es.index(index=index_name, body=doc_with_embedding, id=doc['segment_id'])
+            
+            logger.info(f"Successfully indexed {len(self.documents)} documents in Elasticsearch")
+            self.current_index_name = index_name
+            return index_name
+        except Exception as e:
+            logger.error(f"Error building Elasticsearch index: {str(e)}")
+            raise
 
-        # Index documents in Elasticsearch
-        for doc, embedding in zip(self.documents, self.embeddings):
-            doc['embedding'] = embedding.tolist()
-            self.es.index(index=index_name, body=doc, id=doc['segment_id'])
+    def ensure_index_built(self, video_id, embedding_model):
+        index_name = f"video_{video_id}_{embedding_model.replace('-', '_')}".lower()
+        if not self.es.indices.exists(index=index_name):
+            logger.info(f"Index {index_name} does not exist. Building now...")
+            transcript_data = get_transcript(video_id)
+            if transcript_data:
+                self.process_transcript(video_id, transcript_data)
+                return self.build_index(index_name)
+            else:
+                logger.error(f"Failed to retrieve transcript for video {video_id}")
+                return None
+        return index_name
 
     def search(self, query, filter_dict={}, boost_dict={}, num_results=10, method='hybrid', index_name=None):
+        if not index_name:
+            logger.error("No index name provided for search.")
+            raise ValueError("No index name provided for search.")
+        
+        if not self.es.indices.exists(index=index_name):
+            logger.error(f"Index {index_name} does not exist.")
+            raise ValueError(f"Index {index_name} does not exist.")
+        
+        logger.info(f"Performing {method} search for query: {query} in index: {index_name}")
+        
         if method == 'text':
-            return self.text_search(query, filter_dict, boost_dict, num_results)
+            return self.text_search(query, filter_dict, boost_dict, num_results, index_name)
         elif method == 'embedding':
             return self.embedding_search(query, num_results, index_name)
         else:  # hybrid search
-            text_results = self.text_search(query, filter_dict, boost_dict, num_results)
+            text_results = self.text_search(query, filter_dict, boost_dict, num_results, index_name)
             embedding_results = self.embedding_search(query, num_results, index_name)
             return self.combine_results(text_results, embedding_results, num_results)
 
-    def text_search(self, query, filter_dict={}, boost_dict={}, num_results=10):
-        return self.text_index.search(query, filter_dict, boost_dict, num_results)
+    def text_search(self, query, filter_dict={}, boost_dict={}, num_results=10, index_name=None):
+        if not index_name:
+            logger.error("No index name provided for text search.")
+            raise ValueError("No index name provided for text search.")
+        
+        # Perform text search using Elasticsearch
+        search_body = {
+            "query": {
+                "multi_match": {
+                    "query": query,
+                    "fields": ["content", "title"]
+                }
+            },
+            "size": num_results
+        }
+        response = self.es.search(index=index_name, body=search_body)
+        return [hit['_source'] for hit in response['hits']['hits']]
 
     def embedding_search(self, query, num_results=10, index_name=None):
-        if index_name:
-            # Use Elasticsearch for embedding search
-            query_vector = self.embedding_model.encode(query).tolist()
-            script_query = {
-                "script_score": {
-                    "query": {"match_all": {}},
-                    "script": {
-                        "source": "cosineSimilarity(params.query_vector, 'embedding') + 1.0",
-                        "params": {"query_vector": query_vector}
-                    }
+        if not index_name:
+            logger.error("No index name provided for embedding search.")
+            raise ValueError("No index name provided for embedding search.")
+        
+        query_vector = self.embedding_model.encode(query).tolist()
+        script_query = {
+            "script_score": {
+                "query": {"match_all": {}},
+                "script": {
+                    "source": "cosineSimilarity(params.query_vector, 'embedding') + 1.0",
+                    "params": {"query_vector": query_vector}
                 }
             }
-            response = self.es.search(
-                index=index_name,
-                body={
-                    "size": num_results,
-                    "query": script_query,
-                    "_source": {"excludes": ["embedding"]}
-                }
-            )
-            return [hit['_source'] for hit in response['hits']['hits']]
-        else:
-            # Use in-memory embedding search
-            query_embedding = self.embedding_model.encode(query)
-            similarities = cosine_similarity([query_embedding], self.embeddings)[0]
-            top_indices = np.argsort(similarities)[::-1][:num_results]
-            return [self.documents[i] for i in top_indices]
+        }
+        response = self.es.search(
+            index=index_name,
+            body={
+                "size": num_results,
+                "query": script_query,
+                "_source": {"excludes": ["embedding"]}
+            }
+        )
+        return [hit['_source'] for hit in response['hits']['hits']]
 
     def combine_results(self, text_results, embedding_results, num_results):
         combined = []
@@ -142,4 +217,8 @@ class DataProcessor:
         return deduped[:num_results]
 
     def process_query(self, query):
-        return clean_text(query)
+        return clean_text(query)
+
+    def set_embedding_model(self, model_name):
+        self.embedding_model = SentenceTransformer(model_name)
+        logger.info(f"Embedding model set to: {model_name}")

app/database.py

@@ -6,6 +6,7 @@ class DatabaseHandler:
         self.db_path = db_path
         self.conn = None
         self.create_tables()
+        self.update_schema()
 
     def create_tables(self):
         with sqlite3.connect(self.db_path) as conn:
@@ -48,13 +49,44 @@ class DatabaseHandler:
             ''')
             conn.commit()
 
-    def add_video(self, youtube_id, title, channel_name):
+    def update_schema(self):
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            # Check if columns exist, if not, add them
+            cursor.execute("PRAGMA table_info(videos)")
+            columns = [column[1] for column in cursor.fetchall()]
+            
+            new_columns = [
+                ("upload_date", "TEXT"),
+                ("view_count", "INTEGER"),
+                ("like_count", "INTEGER"),
+                ("comment_count", "INTEGER"),
+                ("video_duration", "TEXT")
+            ]
+            
+            for col_name, col_type in new_columns:
+                if col_name not in columns:
+                    cursor.execute(f"ALTER TABLE videos ADD COLUMN {col_name} {col_type}")
+            
+            conn.commit()
+
+    def add_video(self, video_data):
         with sqlite3.connect(self.db_path) as conn:
             cursor = conn.cursor()
             cursor.execute('''
-                INSERT OR IGNORE INTO videos (youtube_id, title, channel_name)
-                VALUES (?, ?, ?)
-            ''', (youtube_id, title, channel_name))
+                INSERT OR REPLACE INTO videos 
+                (youtube_id, title, channel_name, upload_date, view_count, like_count, comment_count, video_duration)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            ''', (
+                video_data['video_id'],
+                video_data['title'],
+                video_data['author'],
+                video_data['upload_date'],
+                video_data['view_count'],
+                video_data['like_count'],
+                video_data['comment_count'],
+                video_data['video_duration']
+            ))
             conn.commit()
             return cursor.lastrowid
 
@@ -92,12 +124,75 @@ class DatabaseHandler:
             cursor.execute('SELECT * FROM videos WHERE youtube_id = ?', (youtube_id,))
             return cursor.fetchone()
 
-    def get_elasticsearch_index(self, video_id, embedding_model_id):
+    def get_elasticsearch_index(self, video_id, embedding_model):
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                SELECT ei.index_name 
+                FROM elasticsearch_indices ei
+                JOIN embedding_models em ON ei.embedding_model_id = em.id
+                JOIN videos v ON ei.video_id = v.id
+                WHERE v.youtube_id = ? AND em.model_name = ?
+            ''', (video_id, embedding_model))
+            result = cursor.fetchone()
+            return result[0] if result else None
+        
+    def get_all_videos(self):
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                SELECT youtube_id, title, channel_name, upload_date
+                FROM videos
+                ORDER BY upload_date DESC
+            ''')
+            return cursor.fetchall()
+
+    def get_elasticsearch_index_by_youtube_id(self, youtube_id, embedding_model):
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                SELECT ei.index_name 
+                FROM elasticsearch_indices ei
+                JOIN embedding_models em ON ei.embedding_model_id = em.id
+                JOIN videos v ON ei.video_id = v.id
+                WHERE v.youtube_id = ? AND em.model_name = ?
+            ''', (youtube_id, embedding_model))
+            result = cursor.fetchone()
+            return result[0] if result else None
+        
+    def get_transcript_content(self, youtube_id):
+        # This method assumes you're storing the transcript content in the database
+        # If you're not, you'll need to modify this to retrieve the transcript from wherever it's stored
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                SELECT transcript_content
+                FROM videos
+                WHERE youtube_id = ?
+            ''', (youtube_id,))
+            result = cursor.fetchone()
+            return result[0] if result else None
+
+    # If you're not already storing the transcript content, you'll need to add a method to do so:
+    def add_transcript_content(self, youtube_id, transcript_content):
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                UPDATE videos
+                SET transcript_content = ?
+                WHERE youtube_id = ?
+            ''', (transcript_content, youtube_id))
+            conn.commit()
+            
+    def get_elasticsearch_index_by_youtube_id(self, youtube_id, embedding_model):
         with sqlite3.connect(self.db_path) as conn:
             cursor = conn.cursor()
             cursor.execute('''
-                SELECT index_name FROM elasticsearch_indices
-                WHERE video_id = ? AND embedding_model_id = ?
-            ''', (video_id, embedding_model_id))
+                SELECT ei.index_name 
+                FROM elasticsearch_indices ei
+                JOIN embedding_models em ON ei.embedding_model_id = em.id
+                JOIN videos v ON ei.video_id = v.id
+                WHERE v.youtube_id = ? AND em.model_name = ?
+            ''', (youtube_id, embedding_model))
             result = cursor.fetchone()
             return result[0] if result else None

app/main.py

@@ -12,28 +12,43 @@ import json
 import requests
 from tqdm import tqdm
 import sqlite3
+import logging
+import ollama
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
 
-# Initialize components
 @st.cache_resource
 def init_components():
-    db_handler = DatabaseHandler()
-    data_processor = DataProcessor()
-    rag_system = RAGSystem(data_processor)
-    query_rewriter = QueryRewriter()
-    evaluation_system = EvaluationSystem(data_processor, db_handler)
-    return db_handler, data_processor, rag_system, query_rewriter, evaluation_system
-
-db_handler, data_processor, rag_system, query_rewriter, evaluation_system = init_components()
+    try:
+        db_handler = DatabaseHandler()
+        data_processor = DataProcessor()
+        rag_system = RAGSystem(data_processor)
+        query_rewriter = QueryRewriter()
+        evaluation_system = EvaluationSystem(data_processor, db_handler)
+        logger.info("Components initialized successfully")
+        return db_handler, data_processor, rag_system, query_rewriter, evaluation_system
+    except Exception as e:
+        logger.error(f"Error initializing components: {str(e)}")
+        st.error(f"Error initializing components: {str(e)}")
+        st.error("Please check your configuration and ensure all services are running.")
+        return None, None, None, None, None
+        
+components = init_components()
+if components:
+    db_handler, data_processor, rag_system, query_rewriter, evaluation_system = components
+else:
+    st.stop()
 
 # Ground Truth Generation
+
 def generate_questions(transcript):
-    OLLAMA_HOST = os.getenv('OLLAMA_HOST', 'localhost')
-    OLLAMA_PORT = os.getenv('OLLAMA_PORT', '11434')
     prompt_template = """
     You are an AI assistant tasked with generating questions based on a YouTube video transcript.
-    Formulate 10 questions that a user might ask based on the provided transcript.
+    Formulate atleast 10 questions that a user might ask based on the provided transcript.
     Make the questions specific to the content of the transcript.
     The questions should be complete and not too short. Use as few words as possible from the transcript.
+    It is important that the questions are relevant to the content of the transcript and are atleast 10 in number.
 
     The transcript:
 
@@ -47,34 +62,44 @@ def generate_questions(transcript):
     prompt = prompt_template.format(transcript=transcript)
 
     try:
-        response = requests.post(f'http://{OLLAMA_HOST}:{OLLAMA_PORT}/api/generate', json={
-            'model': 'phi3.5',
-            'prompt': prompt
-        })
-        response.raise_for_status()
-        return json.loads(response.json()['response'])
-    except requests.RequestException as e:
-        st.error(f"Error generating questions: {str(e)}")
+        response = ollama.chat(
+            model='phi3.5',
+            messages=[{"role": "user", "content": prompt}]
+        )
+        print("Printing the response from OLLAMA: " + response['message']['content'])
+        return json.loads(response['message']['content'])
+    except Exception as e:
+        logger.error(f"Error generating questions: {str(e)}")
+        return None
+    
+def generate_ground_truth(video_id=None, existing_transcript=None):
+    if video_id is None and existing_transcript is None:
+        st.error("Please provide either a video ID or an existing transcript.")
         return None
 
-def generate_ground_truth(video_id):
-    transcript_data = get_transcript(video_id)
+    if video_id:
+        transcript_data = get_transcript(video_id)
+        if transcript_data and 'transcript' in transcript_data:
+            full_transcript = " ".join([entry['text'] for entry in transcript_data['transcript']])
+        else:
+            logger.error("Failed to retrieve transcript for the provided video ID.")
+            st.error("Failed to retrieve transcript for the provided video ID.")
+            return None
+    else:
+        full_transcript = existing_transcript
+
+    questions = generate_questions(full_transcript)
     
-    if transcript_data and 'transcript' in transcript_data:
-        full_transcript = " ".join([entry['text'] for entry in transcript_data['transcript']])
-        questions = generate_questions(full_transcript)
+    if questions and 'questions' in questions:
+        df = pd.DataFrame([(video_id if video_id else "custom", q) for q in questions['questions']], columns=['video_id', 'question'])
         
-        if questions and 'questions' in questions:
-            df = pd.DataFrame([(video_id, q) for q in questions['questions']], columns=['video_id', 'question'])
-            
-            os.makedirs('data', exist_ok=True)
-            df.to_csv('data/ground-truth-retrieval.csv', index=False)
-            st.success("Ground truth data generated and saved to data/ground-truth-retrieval.csv")
-            return df
-        else:
-            st.error("Failed to generate questions.")
+        os.makedirs('data', exist_ok=True)
+        df.to_csv('data/ground-truth-retrieval.csv', index=False)
+        st.success("Ground truth data generated and saved to data/ground-truth-retrieval.csv")
+        return df
     else:
-        st.error("Failed to generate ground truth data due to transcript retrieval error.")
+        logger.error("Failed to generate questions.")
+        st.error("Failed to generate questions.")
     return None
 
 # RAG Evaluation
@@ -82,6 +107,7 @@ def evaluate_rag(sample_size=200):
     try:
         ground_truth = pd.read_csv('data/ground-truth-retrieval.csv')
     except FileNotFoundError:
+        logger.error("Ground truth file not found. Please generate ground truth data first.")
         st.error("Ground truth file not found. Please generate ground truth data first.")
         return None
 
@@ -111,14 +137,38 @@ def evaluate_rag(sample_size=200):
     progress_bar = st.progress(0)
     for i, (_, row) in enumerate(sample.iterrows()):
         question = row['question']
-        answer_llm = rag_system.query(question)
+        video_id = row['video_id']
+        
+        # Get the index name for the video (you might need to adjust this based on your setup)
+        index_name = db_handler.get_elasticsearch_index_by_youtube_id(video_id, "all-MiniLM-L6-v2")  # Assuming you're using this embedding model
+        
+        if not index_name:
+            logger.warning(f"No index found for video {video_id}. Skipping this question.")
+            continue
+
+        try:
+            answer_llm, _ = rag_system.query(question, index_name=index_name)
+        except ValueError as e:
+            logger.error(f"Error querying RAG system: {str(e)}")
+            continue
+
         prompt = prompt_template.format(question=question, answer_llm=answer_llm)
-        evaluation = rag_system.query(prompt)  # Assuming rag_system can handle this type of query
         try:
-            evaluation_json = json.loads(evaluation)
-            evaluations.append((row['video_id'], question, answer_llm, evaluation_json['Relevance'], evaluation_json['Explanation']))
-        except json.JSONDecodeError:
-            st.warning(f"Failed to parse evaluation for question: {question}")
+            response = ollama.chat(
+                model='phi3.5',
+                messages=[{"role": "user", "content": prompt}]
+            )
+            evaluation_json = json.loads(response['message']['content'])
+            evaluations.append((
+                str(video_id),
+                str(question),
+                str(answer_llm),
+                str(evaluation_json.get('Relevance', 'UNKNOWN')),
+                str(evaluation_json.get('Explanation', 'No explanation provided'))
+            ))
+        except Exception as e:
+            logger.warning(f"Failed to evaluate question: {question}. Error: {str(e)}")
+            st.warning(f"Failed to evaluate question: {question}")
         progress_bar.progress((i + 1) / len(sample))
 
     # Store RAG evaluations in the database
@@ -140,39 +190,163 @@ def evaluate_rag(sample_size=200):
     conn.commit()
     conn.close()
 
+    logger.info("Evaluation complete. Results stored in the database.")
     st.success("Evaluation complete. Results stored in the database.")
     return evaluations
 
+@st.cache_data
+def process_single_video(video_id, embedding_model):
+    # Check if the video has already been processed with the current embedding model
+    existing_index = db_handler.get_elasticsearch_index_by_youtube_id(video_id, embedding_model)
+    if existing_index:
+        logger.info(f"Video {video_id} has already been processed with {embedding_model}. Using existing index: {existing_index}")
+        return existing_index
+
+    transcript_data = get_transcript(video_id)
+    if transcript_data is None:
+        logger.error(f"Failed to retrieve transcript for video {video_id}")
+        return None
+
+    # Store video metadata in the database
+    video_data = {
+        'video_id': video_id,
+        'title': transcript_data['metadata'].get('title', 'Unknown Title'),
+        'author': transcript_data['metadata'].get('author', 'Unknown Author'),
+        'upload_date': transcript_data['metadata'].get('upload_date', 'Unknown Date'),
+        'view_count': int(transcript_data['metadata'].get('view_count', 0)),
+        'like_count': int(transcript_data['metadata'].get('like_count', 0)),
+        'comment_count': int(transcript_data['metadata'].get('comment_count', 0)),
+        'video_duration': transcript_data['metadata'].get('duration', 'Unknown Duration')
+    }
+    try:
+        db_handler.add_video(video_data)
+    except Exception as e:
+        logger.error(f"Error adding video to database: {str(e)}")
+        return None
+
+    # Process transcript for RAG system
+    try:
+        data_processor.process_transcript(video_id, transcript_data)
+    except Exception as e:
+        logger.error(f"Error processing transcript: {str(e)}")
+        return None
+    
+    # Create Elasticsearch index
+    index_name = f"video_{video_id}_{embedding_model}".lower()
+    try:
+        index_name = data_processor.build_index(index_name)
+        logger.info(f"Successfully built index: {index_name}")
+    except Exception as e:
+        logger.error(f"Error building index: {str(e)}")
+        return None
+    
+    # Add embedding model to the database
+    embedding_model_id = db_handler.add_embedding_model(embedding_model, "Description of the model")
+    
+    # Get the video ID from the database
+    video_db_record = db_handler.get_video_by_youtube_id(video_id)
+    if video_db_record is None:
+        logger.error(f"Failed to retrieve video record from database for video {video_id}")
+        return None
+    video_db_id = video_db_record[0]  # Assuming the ID is the first column
+    
+    # Store Elasticsearch index information
+    db_handler.add_elasticsearch_index(video_db_id, index_name, embedding_model_id)
+    
+    logger.info(f"Processed and indexed transcript for video {video_id}")
+    return index_name
+
+@st.cache_data
+def process_multiple_videos(video_ids, embedding_model):
+    indices = []
+    for video_id in video_ids:
+        index = process_single_video(video_id, embedding_model)
+        if index:
+            indices.append(index)
+    logger.info(f"Processed and indexed transcripts for {len(indices)} videos")
+    st.success(f"Processed and indexed transcripts for {len(indices)} videos")
+    return indices
+
 def main():
     st.title("YouTube Transcript RAG System")
 
+    components = init_components()
+    if not all(components):
+        st.error("Failed to initialize one or more components. Please check the logs and your configuration.")
+        return
+
+    db_handler, data_processor, rag_system, query_rewriter, evaluation_system = components
+
     tab1, tab2, tab3 = st.tabs(["RAG System", "Ground Truth Generation", "Evaluation"])
 
     with tab1:
         st.header("RAG System")
-        # Input section
+        
+        # Video selection section
+        st.subheader("Select a Video")
+        videos = db_handler.get_all_videos()
+        if not videos:
+            st.warning("No videos available. Please process some videos first.")
+        else:
+            video_df = pd.DataFrame(videos, columns=['youtube_id', 'title', 'channel_name', 'upload_date'])
+            
+            # Allow filtering by channel name
+            channels = sorted(video_df['channel_name'].unique())
+            selected_channel = st.selectbox("Filter by Channel", ["All"] + channels)
+            
+            if selected_channel != "All":
+                video_df = video_df[video_df['channel_name'] == selected_channel]
+            
+            # Display videos and allow selection
+            st.dataframe(video_df)
+            selected_video_id = st.selectbox("Select a Video", video_df['youtube_id'].tolist(), format_func=lambda x: video_df[video_df['youtube_id'] == x]['title'].iloc[0])
+            
+            # Embedding model selection
+            embedding_model = st.selectbox("Select embedding model:", ["all-MiniLM-L6-v2", "all-mpnet-base-v2"])
+            
+            # Get the index name for the selected video and embedding model
+            index_name = db_handler.get_elasticsearch_index_by_youtube_id(selected_video_id, embedding_model)
+            
+            if index_name:
+                st.success(f"Using index: {index_name}")
+            else:
+                st.warning("No index found for the selected video and embedding model. The index will be built when you search.")
+        
+        # Process new video section
+        st.subheader("Process New Video")
         input_type = st.radio("Select input type:", ["Video URL", "Channel URL", "YouTube ID"])
         input_value = st.text_input("Enter the URL or ID:")
-        embedding_model = st.selectbox("Select embedding model:", ["all-MiniLM-L6-v2", "all-mpnet-base-v2"])
-
+        
         if st.button("Process"):
             with st.spinner("Processing..."):
                 data_processor.embedding_model = SentenceTransformer(embedding_model)
                 if input_type == "Video URL":
                     video_id = extract_video_id(input_value)
                     if video_id:
-                        process_single_video(video_id, embedding_model)
+                        index_name = process_single_video(video_id, embedding_model)
+                        if index_name is None:
+                            st.error(f"Failed to process video {video_id}")
+                        else:
+                            st.success(f"Successfully processed video {video_id}")
                     else:
                         st.error("Failed to extract video ID from the URL")
                 elif input_type == "Channel URL":
                     channel_videos = get_channel_videos(input_value)
                     if channel_videos:
-                        process_multiple_videos([video['video_id'] for video in channel_videos], embedding_model)
+                        index_names = process_multiple_videos([video['video_id'] for video in channel_videos], embedding_model)
+                        if not index_names:
+                            st.error("Failed to process any videos from the channel")
+                        else:
+                            st.success(f"Successfully processed {len(index_names)} videos from the channel")
                     else:
                         st.error("Failed to retrieve videos from the channel")
                 else:
-                    process_single_video(input_value, embedding_model)
-
+                    index_name = process_single_video(input_value, embedding_model)
+                    if index_name is None:
+                        st.error(f"Failed to process video {input_value}")
+                    else:
+                        st.success(f"Successfully processed video {input_value}")
+        
         # Query section
         st.subheader("Query the RAG System")
         query = st.text_input("Enter your query:")
@@ -180,108 +354,147 @@ def main():
         search_method = st.radio("Search method:", ["Hybrid", "Text-only", "Embedding-only"])
 
         if st.button("Search"):
-            with st.spinner("Searching..."):
-                if rewrite_method == "Chain of Thought":
-                    query = query_rewriter.rewrite_cot(query)
-                elif rewrite_method == "ReAct":
-                    query = query_rewriter.rewrite_react(query)
-
-                search_method_map = {"Hybrid": "hybrid", "Text-only": "text", "Embedding-only": "embedding"}
-                response = rag_system.query(query, search_method=search_method_map[search_method])
-                st.write("Response:", response)
-
-            # Feedback
-            feedback = st.radio("Provide feedback:", ["+1", "-1"])
-            if st.button("Submit Feedback"):
-                db_handler.add_user_feedback("all_videos", query, 1 if feedback == "+1" else -1)
-                st.success("Feedback submitted successfully!")
+            if not selected_video_id:
+                st.error("Please select a video before searching.")
+            else:
+                with st.spinner("Searching..."):
+                    rewritten_query = query
+                    rewrite_prompt = ""
+                    if rewrite_method == "Chain of Thought":
+                        rewritten_query, rewrite_prompt = query_rewriter.rewrite_cot(query)
+                    elif rewrite_method == "ReAct":
+                        rewritten_query, rewrite_prompt = query_rewriter.rewrite_react(query)
+
+                    st.subheader("Query Processing")
+                    st.write("Original query:", query)
+                    if rewrite_method != "None":
+                        st.write("Rewritten query:", rewritten_query)
+                        st.text_area("Query rewriting prompt:", rewrite_prompt, height=100)
+                        if rewritten_query == query:
+                            st.warning("Query rewriting failed. Using original query.")
+
+                    search_method_map = {"Hybrid": "hybrid", "Text-only": "text", "Embedding-only": "embedding"}
+                    try:
+                        # Ensure index is built before searching
+                        if not index_name:
+                            st.info("Building index for the selected video...")
+                            index_name = process_single_video(selected_video_id, embedding_model)
+                            if not index_name:
+                                st.error("Failed to build index for the selected video.")
+                                return
+
+                        response, final_prompt = rag_system.query(rewritten_query, search_method=search_method_map[search_method], index_name=index_name)
+                        
+                        st.subheader("RAG System Prompt")
+                        if final_prompt:
+                            st.text_area("Prompt sent to LLM:", final_prompt, height=300)
+                        else:
+                            st.warning("No prompt was generated. This might indicate an issue with the RAG system.")
+                        
+                        st.subheader("Response")
+                        if response:
+                            st.write(response)
+                        else:
+                            st.error("No response generated. Please try again or check the system logs for errors.")
+                    except ValueError as e:
+                        logger.error(f"Error during search: {str(e)}")
+                        st.error(f"Error during search: {str(e)}")
+                    except Exception as e:
+                        logger.error(f"An unexpected error occurred: {str(e)}")
+                        st.error(f"An unexpected error occurred: {str(e)}")
 
     with tab2:
         st.header("Ground Truth Generation")
-        video_id = st.text_input("Enter YouTube Video ID for ground truth generation:")
-        if st.button("Generate Ground Truth"):
-            with st.spinner("Generating ground truth..."):
-                ground_truth_df = generate_ground_truth(video_id)
-                if ground_truth_df is not None:
-                    st.dataframe(ground_truth_df)
-                    csv = ground_truth_df.to_csv(index=False)
-                    st.download_button(
-                        label="Download Ground Truth CSV",
-                        data=csv,
-                        file_name="ground_truth.csv",
-                        mime="text/csv",
-                    )
+        use_existing_transcript = st.checkbox("Use existing transcript")
+        
+        if use_existing_transcript:
+            # Get all available videos (assuming all videos have transcripts)
+            videos = db_handler.get_all_videos()
+            if not videos:
+                st.warning("No videos available. Please process some videos first.")
+            else:
+                video_df = pd.DataFrame(videos, columns=['youtube_id', 'title', 'channel_name', 'upload_date'])
+                
+                # Allow filtering by channel name
+                channels = sorted(video_df['channel_name'].unique())
+                selected_channel = st.selectbox("Filter by Channel", ["All"] + channels, key="gt_channel_filter")
+                
+                if selected_channel != "All":
+                    video_df = video_df[video_df['channel_name'] == selected_channel]
+                
+                # Display videos and allow selection
+                st.dataframe(video_df)
+                selected_video_id = st.selectbox("Select a Video", video_df['youtube_id'].tolist(), 
+                                                 format_func=lambda x: video_df[video_df['youtube_id'] == x]['title'].iloc[0],
+                                                 key="gt_video_select")
+                
+                if st.button("Generate Ground Truth from Existing Transcript"):
+                    with st.spinner("Generating ground truth..."):
+                        # Retrieve the transcript content (you'll need to implement this method)
+                        transcript_data = get_transcript(selected_video_id)
+                        if transcript_data and 'transcript' in transcript_data:
+                            full_transcript = " ".join([entry['text'] for entry in transcript_data['transcript']])
+                            ground_truth_df = generate_ground_truth(video_id=selected_video_id, existing_transcript=full_transcript)
+                            if ground_truth_df is not None:
+                                st.dataframe(ground_truth_df)
+                                csv = ground_truth_df.to_csv(index=False)
+                                st.download_button(
+                                    label="Download Ground Truth CSV",
+                                    data=csv,
+                                    file_name=f"ground_truth_{selected_video_id}.csv",
+                                    mime="text/csv",
+                                )
+                        else:
+                            st.error("Failed to retrieve transcript content.")
+        else:
+            video_id = st.text_input("Enter YouTube Video ID for ground truth generation:")
+            if st.button("Generate Ground Truth"):
+                with st.spinner("Generating ground truth..."):
+                    ground_truth_df = generate_ground_truth(video_id=video_id)
+                    if ground_truth_df is not None:
+                        st.dataframe(ground_truth_df)
+                        csv = ground_truth_df.to_csv(index=False)
+                        st.download_button(
+                            label="Download Ground Truth CSV",
+                            data=csv,
+                            file_name=f"ground_truth_{video_id}.csv",
+                            mime="text/csv",
+                        )
 
     with tab3:
         st.header("RAG Evaluation")
-        sample_size = st.number_input("Enter sample size for evaluation:", min_value=1, max_value=1000, value=200)
-        if st.button("Run Evaluation"):
-            with st.spinner("Running evaluation..."):
-                evaluation_results = evaluate_rag(sample_size)
-                if evaluation_results:
-                    st.write("Evaluation Results:")
-                    st.dataframe(pd.DataFrame(evaluation_results, columns=['Video ID', 'Question', 'Answer', 'Relevance', 'Explanation']))
-
-@st.cache_data
-def process_single_video(video_id, embedding_model):
-    # Check if the video has already been processed with the current embedding model
-    existing_index = db_handler.get_elasticsearch_index(video_id, embedding_model)
-    if existing_index:
-        st.info(f"Video {video_id} has already been processed with {embedding_model}. Using existing index: {existing_index}")
-        return existing_index
 
-    transcript_data = get_transcript(video_id)
-    if transcript_data:
-        # Store video metadata in the database
-        video_data = {
-            'video_id': video_id,
-            'title': transcript_data['metadata'].get('title', 'Unknown Title'),
-            'author': transcript_data['metadata'].get('author', 'Unknown Author'),
-            'upload_date': transcript_data['metadata'].get('upload_date', 'Unknown Date'),
-            'view_count': int(transcript_data['metadata'].get('view_count', 0)),
-            'like_count': int(transcript_data['metadata'].get('like_count', 0)),
-            'comment_count': int(transcript_data['metadata'].get('comment_count', 0)),
-            'video_duration': transcript_data['metadata'].get('duration', 'Unknown Duration')
-        }
-        db_handler.add_video(video_data)
+        # Load ground truth data
+        try:
+            ground_truth_df = pd.read_csv('data/ground-truth-retrieval.csv')
+            ground_truth_available = True
+        except FileNotFoundError:
+            ground_truth_available = False
 
-        # Store transcript segments in the database
-        for i, segment in enumerate(transcript_data['transcript']):
-            segment_data = {
-                'segment_id': f"{video_id}_{i}",
-                'video_id': video_id,
-                'content': segment.get('text', ''),
-                'start_time': segment.get('start', 0),
-                'duration': segment.get('duration', 0)
-            }
-            db_handler.add_transcript_segment(segment_data)
-
-        # Process transcript for RAG system
-        data_processor.process_transcript(video_id, transcript_data)
-        
-        # Create Elasticsearch index
-        index_name = f"video_{video_id}_{embedding_model}"
-        data_processor.build_index(index_name)
-        
-        # Store Elasticsearch index information
-        db_handler.add_elasticsearch_index(video_id, index_name, embedding_model)
-        
-        st.success(f"Processed and indexed transcript for video {video_id}")
-        st.write("Metadata:", transcript_data['metadata'])
-        return index_name
-    else:
-        st.error(f"Failed to retrieve transcript for video {video_id}")
-        return None
+        if ground_truth_available:
+            st.write("Evaluation will be run on the following ground truth data:")
+            st.dataframe(ground_truth_df)
+            st.info("The evaluation will use this ground truth data to assess the performance of the RAG system.")
 
-@st.cache_data
-def process_multiple_videos(video_ids, embedding_model):
-    indices = []
-    for video_id in video_ids:
-        index = process_single_video(video_id, embedding_model)
-        if index:
-            indices.append(index)
-    st.success(f"Processed and indexed transcripts for {len(indices)} videos")
-    return indices
+            sample_size = st.number_input("Enter sample size for evaluation:", min_value=1, max_value=len(ground_truth_df), value=min(200, len(ground_truth_df)))
+            
+            if st.button("Run Evaluation"):
+                with st.spinner("Running evaluation..."):
+                    evaluation_results = evaluate_rag(sample_size)
+                    if evaluation_results:
+                        st.write("Evaluation Results:")
+                        st.dataframe(pd.DataFrame(evaluation_results, columns=['Video ID', 'Question', 'Answer', 'Relevance', 'Explanation']))
+        else:
+            st.warning("No ground truth data available. Please generate ground truth data first.")
+            st.button("Run Evaluation", disabled=True)
+
+        # Add a section to generate ground truth if it's not available
+        if not ground_truth_available:
+            st.subheader("Generate Ground Truth")
+            st.write("You need to generate ground truth data before running the evaluation.")
+            if st.button("Go to Ground Truth Generation"):
+                st.session_state.active_tab = "Ground Truth Generation"
+                st.experimental_rerun()
 
 if __name__ == "__main__":
     main()

app/query_rewriter.py

@@ -1,8 +1,24 @@
+import os
 import ollama
+import logging
+
+logger = logging.getLogger(__name__)
 
 class QueryRewriter:
     def __init__(self):
-        self.model = "phi"  # Using Phi-3.5 model
+        self.model = os.getenv('OLLAMA_MODEL', "phi3")
+        self.ollama_host = os.getenv('OLLAMA_HOST', 'http://ollama:11434')
+
+    def generate(self, prompt):
+        try:
+            response = ollama.chat(
+                model=self.model,
+                messages=[{"role": "user", "content": prompt}]
+            )
+            return response['message']['content']
+        except Exception as e:
+            logger.error(f"Error generating response: {e}")
+            return f"Error: {str(e)}"
 
     def rewrite_cot(self, query):
         prompt = f"""
@@ -11,8 +27,11 @@ class QueryRewriter:
         
         Rewritten query:
         """
-        response = ollama.generate(model=self.model, prompt=prompt)
-        return response['response'].strip()
+        rewritten_query = self.generate(prompt)
+        if rewritten_query.startswith("Error:"):
+            logger.error(f"Error in CoT rewriting: {rewritten_query}")
+            return query, prompt  # Return original query if rewriting fails
+        return rewritten_query, prompt
 
     def rewrite_react(self, query):
         prompt = f"""
@@ -29,5 +48,8 @@ class QueryRewriter:
         
         Final rewritten query:
         """
-        response = ollama.generate(model=self.model, prompt=prompt)
-        return response['response'].strip()
+        rewritten_query = self.generate(prompt)
+        if rewritten_query.startswith("Error:"):
+            logger.error(f"Error in ReAct rewriting: {rewritten_query}")
+            return query, prompt  # Return original query if rewriting fails
+        return rewritten_query, prompt

app/rag.py

@@ -1,31 +1,108 @@
+import os
+from dotenv import load_dotenv
 import ollama
+import logging
+import time
+
+load_dotenv()
+
+logger = logging.getLogger(__name__)
 
 class RAGSystem:
     def __init__(self, data_processor):
         self.data_processor = data_processor
-        self.model = "phi3.5"  # Using Phi-3.5 model
-
-    def query(self, user_query, top_k=3, search_method='hybrid'):
-        # Retrieve relevant documents using the specified search method
-        relevant_docs = self.data_processor.search(user_query, num_results=top_k, method=search_method)
+        self.model = os.getenv('OLLAMA_MODEL', 'phi3')
+        self.ollama_host = os.getenv('OLLAMA_HOST', 'http://ollama:11434')
+        self.timeout = int(os.getenv('OLLAMA_TIMEOUT', 240))
+        self.max_retries = int(os.getenv('OLLAMA_MAX_RETRIES', 3))
         
-        # Construct the prompt
+        self.check_ollama_service()
+
+    def check_ollama_service(self):
+        try:
+            ollama.list()
+            logger.info("Ollama service is accessible.")
+            self.pull_model()
+        except Exception as e:
+            logger.error(f"An error occurred while connecting to Ollama: {e}")
+            logger.error(f"Please ensure Ollama is running and accessible at {self.ollama_host}")
+
+    def pull_model(self):
+        try:
+            ollama.pull(self.model)
+            logger.info(f"Successfully pulled model {self.model}.")
+        except Exception as e:
+            logger.error(f"Error pulling model {self.model}: {e}")
+
+    def generate(self, prompt):
+        for attempt in range(self.max_retries):
+            try:
+                response = ollama.chat(
+                    model=self.model,
+                    messages=[{"role": "user", "content": prompt}]
+                )
+                print("Printing the response from OLLAMA: "+response['message']['content'])
+                return response['message']['content']
+            except Exception as e:
+                logger.error(f"Error generating response on attempt {attempt + 1}: {e}")
+                if attempt == self.max_retries - 1:
+                    logger.error("All retries exhausted. Unable to generate response.")
+                    return None
+                time.sleep(2 ** attempt)  # Exponential backoff
+
+    def get_prompt(self, user_query, relevant_docs):
         context = "\n".join([doc['content'] for doc in relevant_docs])
-        prompt = f"Context: {context}\n\nQuestion: {user_query}\n\nAnswer:"
-        
-        # Generate response using Ollama
-        response = ollama.generate(model=self.model, prompt=prompt)
+        prompt = f"""You are AI Youtube transcript assistant that analyses youtube transcripts and responds back to the user query based on the Context shared with you. Please ensure that the answers are correct, meaningful, and help in answering the query.
+
+Context: {context}
+
+Question: {user_query}
+
+Answer:"""
+        return prompt
+
+    def query(self, user_query, search_method='hybrid', index_name=None):
+        try:
+            if not index_name:
+                raise ValueError("No index name provided. Please select a video and ensure it has been processed.")
+
+            relevant_docs = self.data_processor.search(user_query, num_results=3, method=search_method, index_name=index_name)
+            
+            if not relevant_docs:
+                logger.warning("No relevant documents found for the query.")
+                return "I couldn't find any relevant information to answer your query.", ""
+
+            prompt = self.get_prompt(user_query, relevant_docs)
+            
+            response = ollama.chat(
+                model=self.model,
+                messages=[{"role": "user", "content": prompt}]
+            )
+            
+            answer = response['message']['content']
+            return answer, prompt
+        except Exception as e:
+            logger.error(f"An error occurred in the RAG system: {e}")
+            return f"An error occurred: {str(e)}", ""
         
-        return response['response']
-
-    def rerank_documents(self, documents, query):
-        # Implement a simple re-ranking strategy
-        # This could be improved with more sophisticated methods
-        reranked = sorted(documents, key=lambda doc: self.calculate_relevance(doc['content'], query), reverse=True)
-        return reranked
-
-    def calculate_relevance(self, document, query):
-        # Simple relevance calculation based on word overlap
-        doc_words = set(document.lower().split())
-        query_words = set(query.lower().split())
-        return len(doc_words.intersection(query_words)) / len(query_words)
+    def rewrite_cot(self, query):
+        prompt = f"""Rewrite the following query using chain-of-thought reasoning:
+
+Query: {query}
+
+Rewritten query:"""
+        response = self.generate(prompt)
+        if response:
+            return response, prompt
+        return query, prompt  # Return original query if rewriting fails
+
+    def rewrite_react(self, query):
+        prompt = f"""Rewrite the following query using ReAct (Reasoning and Acting) approach:
+
+Query: {query}
+
+Rewritten query:"""
+        response = self.generate(prompt)
+        if response:
+            return response, prompt
+        return query, prompt  # Return original query if rewriting fails

app/transcript_extractor.py

@@ -1,15 +1,35 @@
+import os
+from dotenv import load_dotenv
 from youtube_transcript_api import YouTubeTranscriptApi
 from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError
 import re
-import os
 
-# Replace with your actual API key
-API_KEY = os.environ.get('YOUTUBE_API_KEY', 'YOUR_API_KEY_HERE')
+# Get the directory of the current script
+current_dir = os.path.dirname(os.path.abspath(__file__))
+# Construct the path to the .env file (one directory up from the current script)
+dotenv_path = os.path.join(os.path.dirname(current_dir), '.env')
+print("the .env path is :" + dotenv_path)
+# Load environment variables from .env file
+load_dotenv(dotenv_path)
+
+# Get API key from environment variable
+API_KEY = os.getenv('YOUTUBE_API_KEY')
+print("the api key is :" + API_KEY)
+if not API_KEY:
+    raise ValueError("YouTube API key not found. Make sure it's set in your .env file in the parent directory of the 'app' folder.")
 
-youtube = build('youtube', 'v3', developerKey=API_KEY)
+print(f"API_KEY: {API_KEY[:5]}...{API_KEY[-5:]}")  # Print first and last 5 characters for verification
+
+try:
+    youtube = build('youtube', 'v3', developerKey=API_KEY)
+except Exception as e:
+    print(f"Error initializing YouTube API client: {str(e)}")
+    raise
 
 def extract_video_id(url):
+    if not url:
+        return None
     video_id_match = re.search(r"(?:v=|\/)([0-9A-Za-z_-]{11}).*", url)
     if video_id_match:
         return video_id_match.group(1)
@@ -30,21 +50,53 @@ def get_video_metadata(video_id):
                 'title': snippet['title'],
                 'author': snippet['channelTitle'],
                 'upload_date': snippet['publishedAt'],
-                'view_count': video['statistics']['viewCount'],
-                'like_count': video['statistics'].get('likeCount', 'N/A'),
-                'comment_count': video['statistics'].get('commentCount', 'N/A'),
+                'view_count': video['statistics'].get('viewCount', '0'),
+                'like_count': video['statistics'].get('likeCount', '0'),
+                'comment_count': video['statistics'].get('commentCount', '0'),
                 'duration': video['contentDetails']['duration']
             }
         else:
+            print(f"No video found with ID: {video_id}")
             return None
     except HttpError as e:
         print(f"An HTTP error {e.resp.status} occurred: {e.content}")
         return None
+    except Exception as e:
+        print(f"An error occurred while fetching video metadata: {str(e)}")
+        return None
 
 def get_transcript(video_id):
+    # Get the directory of the current script
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    # Construct the path to the .env file (one directory up from the current script)
+    dotenv_path = os.path.join(os.path.dirname(current_dir), '.env')
+    print("the .env path is :" + dotenv_path)
+    # Load environment variables from .env file
+    load_dotenv(dotenv_path)
+
+    # Get API key from environment variable
+    API_KEY = os.getenv('YOUTUBE_API_KEY')
+    print("the api key is :" + API_KEY)
+    if not API_KEY:
+        raise ValueError("YouTube API key not found. Make sure it's set in your .env file in the parent directory of the 'app' folder.")
+
+    print(f"API_KEY: {API_KEY[:5]}...{API_KEY[-5:]}")  # Print first and last 5 characters for verification
+
+    try:
+        youtube = build('youtube', 'v3', developerKey=API_KEY)
+    except Exception as e:
+        print(f"Error initializing YouTube API client: {str(e)}")
+        raise
+    
+    if not video_id:
+        return None
     try:
         transcript = YouTubeTranscriptApi.get_transcript(video_id)
         metadata = get_video_metadata(video_id)
+        print(f"Metadata for video {video_id}: {metadata}")
+        print(f"Transcript length for video {video_id}: {len(transcript)}")
+        if not metadata:
+            return None
         return {
             'transcript': transcript,
             'metadata': metadata
@@ -53,7 +105,11 @@ def get_transcript(video_id):
         print(f"Error extracting transcript for video {video_id}: {str(e)}")
         return None
 
-def get_channel_videos(channel_id):
+def get_channel_videos(channel_url):
+    channel_id = extract_channel_id(channel_url)
+    if not channel_id:
+        print(f"Invalid channel URL: {channel_url}")
+        return []
     try:
         request = youtube.search().list(
             part="id,snippet",
@@ -75,6 +131,15 @@ def get_channel_videos(channel_id):
     except HttpError as e:
         print(f"An HTTP error {e.resp.status} occurred: {e.content}")
         return []
+    except Exception as e:
+        print(f"An error occurred while fetching channel videos: {str(e)}")
+        return []
+
+def extract_channel_id(url):
+    channel_id_match = re.search(r"(?:channel\/|c\/|@)([a-zA-Z0-9-_]+)", url)
+    if channel_id_match:
+        return channel_id_match.group(1)
+    return None
 
 def process_videos(video_ids):
     transcripts = {}

data/ground-truth-retrieval.csv

@@ -0,0 +1,7 @@
+video_id,question
+zjkBMFhNj_g,What are Google Apps Script and its relation to user data security within a domain?
+zjkBMFhNj_g,"How can prompt injection attacks manipulate language models' outputs using shared documents like those managed by Gmail users or Microsoft Office files (Word, Excel)?"
+zjkBMFhNj_g,"In the context of AI-based systems such as large language models (LLMs), how might an attacker exploit these tools to exfiltrate sensitive user data from a Google Doc? Please provide details."
+zjkBMFhNj_g,"Can you explain prompt injection attacks and their potential impact on LLM predictions, including any specific examples provided in the discussion like using 'James Bond' as a trigger phrase for threat detection tasks or title generation?"
+zjkBMFhNj_g,Are there defenses against these types of language model (LLM) security threats similar to traditional cybersecurity measures such as prompt injection attacks and data poisoning? Please elaborate.
+zjkBMFhNj_g,"What does the future hold for LLMs considering their benefits, potential risks including adversarial exploitation like those discussed here, regulatory oversight needs due to privacy concerns (GDPR), mitigation of harmful outputs by these models in various applications?"

docker-compose.yaml

@@ -7,10 +7,14 @@ services:
       - "8501:8501"
     depends_on:
       - elasticsearch
+      - ollama
     environment:
       - ELASTICSEARCH_HOST=elasticsearch
       - ELASTICSEARCH_PORT=9200
       - YOUTUBE_API_KEY=${YOUTUBE_API_KEY}
+      - OLLAMA_HOST=http://ollama:11434
+      - OLLAMA_TIMEOUT=${OLLAMA_TIMEOUT:-120}
+      - OLLAMA_MAX_RETRIES=${OLLAMA_MAX_RETRIES:-3}
     env_file:
       - .env
     volumes:
@@ -37,6 +41,14 @@ services:
     depends_on:
       - elasticsearch
 
+  ollama:
+    image: ollama/ollama:latest
+    ports:
+      - "11434:11434"
+    volumes:
+      - ollama_data:/root/.ollama
+
 volumes:
   esdata:
-  grafana-storage:
+  grafana-storage:
+  ollama_data:

requirements.txt

@@ -11,4 +11,5 @@ elasticsearch
 ollama
 requests
 matplotlib
-tqdm
+tqdm
+python-dotenv