embedchain init code

.gitignore

@@ -0,0 +1,129 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+.coverage
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or tool, you might want to ignore these files since the code is intended to run in multiple environments; 
+#   otherwise, check in the pyenv configuration files, especially if you are in an isolated environment.
+.pyenv
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies 
+#   not cross-compatible, pipenv may install dependencies that are not in line with the rest of the team.
+Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Pycharm
+.idea/
+
+# VS Code
+.vscode/
+
+# Streamlit static files
+.streamlit/
+
+# Local environment variables
+.env
+
+# Deepface models
+. deepface_weights/
+
+# MacOS specific
+.DS_Store
+
+# Keep empty models dir
+models/*
+!models/.gitkeep
+
+# All cached movie files
+*.mp4.env
+
+# cache dir
+cache/
+

app.py

@@ -0,0 +1,229 @@
+import os
+import streamlit as st
+import re
+from datetime import datetime
+from groq import Groq
+from langchain_community.vectorstores import FAISS
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_huggingface import HuggingFaceEmbeddings
+import pandas as pd
+
+
+# Load environment variables
+try:
+    from dotenv import load_dotenv
+    load_dotenv()
+except ImportError:
+    pass  # dotenv not installed, likely running on Hugging Face Spaces
+
+# Function to get the API key
+
+
+def get_api_key():
+    api_key = os.environ.get("GROQ_API_KEY")
+    if not api_key:
+        api_key = st.secrets.get("GROQ_API_KEY")
+    if not api_key:
+        st.error(
+            "GROQ_API_KEY is not set. Please set it in your environment or Streamlit secrets.")
+        st.stop()
+    return api_key
+
+
+def parse_transcript(content):
+
+    parsed_segments = []
+    current_speaker = ""
+    current_company = ""
+    current_timestamp = ""
+
+    # Split the content into paragraphs
+    paragraphs = re.split(r'\n\s*\n', content)
+
+    for paragraph in paragraphs:
+        paragraph = paragraph.strip()
+        if not paragraph:
+            continue
+
+        # Check if the paragraph contains speaker information
+        speaker_match = re.match(r'(.*?),\s*(.*?)\((.*?)\):', paragraph)
+        if speaker_match:
+            current_speaker, current_company, current_timestamp = speaker_match.groups()
+            text = paragraph.split('\n', 1)[1] if '\n' in paragraph else ''
+
+        # Check if the paragraph contains only a timestamp
+        elif re.match(r'\((.*?)\):', paragraph):
+            current_timestamp = re.match(r'\((.*?)\):', paragraph).group(1)
+            text = re.sub(r'^\(.*?\):\s*', '', paragraph).strip()
+
+        # If it's not a speaker line or timestamp line, it's just text
+        else:
+            text = paragraph
+
+        # Add the segment
+        if text:
+            add_segment(parsed_segments, current_speaker,
+                        current_company, current_timestamp, text)
+
+    return parsed_segments
+
+
+def add_segment(parsed_segments, speaker, company, timestamp, text):
+    segment = {
+        "speaker": speaker,
+        "company": company,
+        "timestamp": timestamp,
+        "text": text
+    }
+    parsed_segments.append(segment)
+    print_segment(speaker, company, timestamp, text)
+
+
+def print_segment(speaker, company, timestamp, text):
+    print(f"Speaker: {speaker}")
+    print(f"Company: {company}")
+    print(f"Timestamp: {timestamp}")
+    print(f"Text: {text[:100]}...")  # Print first 100 characters of text
+    print("-" * 50)
+
+
+def create_searchable_segments(parsed_segments):
+    searchable_segments = []
+    for segment in parsed_segments:
+        searchable_text = (
+            f"{segment['speaker']},{segment['company']},"
+            f"{segment['timestamp']}:: {segment['text']}"
+        )
+        searchable_segments.append(searchable_text)
+    return searchable_segments
+
+
+# Load and parse the transcript
+def load_transcript(content):
+    global vectorstore
+
+    # Parse the transcript
+    parsed_transcript = parse_transcript(content)
+    searchable_segments = create_searchable_segments(parsed_transcript)
+
+    # Create text splitter and split the searchable segments
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=1000, chunk_overlap=200)
+
+    splits = text_splitter.create_documents(searchable_segments)
+
+    # Create vector store with HuggingFaceEmbeddings
+    embeddings = HuggingFaceEmbeddings()
+    vectorstore = FAISS.from_documents(documents=splits, embedding=embeddings)
+
+
+def search_transcript(query, k=30):
+    # Perform similarity search
+    docs = vectorstore.similarity_search(query=query, k=k)
+
+    # Format results
+    results = []
+    for doc in docs:
+        content = doc.page_content
+        match = re.match(r'(.*?),(.*?),(.*?)::\s*(.*)', content, re.DOTALL)
+        if match:
+            speaker, company, timestamp, text = match.groups()
+            results.append({
+                "speaker": speaker.strip(),
+                "company": company.strip(),
+                "timestamp": timestamp.strip(),
+                "text": text.strip()
+            })
+
+    return results
+
+
+# Groq client setup
+client = Groq(api_key=get_api_key())
+
+
+def generate_response(query, search_results):
+    # Prepare the prompt with search results
+    prompt = f"""You are a friendly assistant. Your job is to answer the user's question based on the transcript excerpts provided below:
+
+Transcript excerpts:
+{search_results}
+
+Question: {query}
+
+Please provide a concise and relevant answer based on the information in the transcript excerpts. If the information is not directly related to the question, say so and provide the most relevant information available."""
+
+    completion = client.chat.completions.create(
+        model="llama3-8b-8192",
+        messages=[
+            {
+                "role": "user",
+                "content": prompt
+            }
+        ],
+        temperature=0.5,
+        max_tokens=3000,
+        top_p=1,
+        stream=False,
+        stop=None,
+    )
+
+    return completion.choices[0].message.content
+
+
+# Streamlit app
+def main():
+    st.title("Transcript Search and Q&A")
+
+    st.caption("This site takes a TelecomTV video transcript and allows a chat session with it.  If no transcript is provided it defaults to this one: https://www.telecomtv.com/content/dsp-leaders-forum/enabling-the-autonomous-network-with-ai-50536/")
+
+    # File upload
+    uploaded_file = st.file_uploader("Upload a transcript file", type="txt")
+
+    if uploaded_file is None:
+        file_name = "Enabling the autonomous network with AI"
+        with open("example-transcript.txt", 'r') as file:
+            content = file.read()
+    else:
+        content = uploaded_file.getvalue().decode("utf-8")
+        file_name = uploaded_file.name
+
+    # Read and process the uploaded file
+    load_transcript(content)
+
+    st.subheader(f"Chat with {file_name}")
+
+    # User input
+    user_query = st.text_input(
+        "Enter your question:", placeholder="e.g.What are people speaking about? or List all people speaking")
+
+    # Add a slider for selecting the number of results
+    num_results = st.slider("Number of relevant transcript excerpts to show:",
+                            min_value=1, max_value=50, value=30, step=1)
+
+    if user_query:
+        search_results = search_transcript(user_query, k=num_results)
+        formatted_results = (
+            "\n\n".join([f"{result['speaker']} {result['company']} ({result['timestamp']}): "
+                         "{result['text']}" for result in search_results])
+        )
+
+        response = generate_response(user_query, formatted_results)
+
+        st.subheader("Assistant's response:")
+        st.write(response)
+
+        st.subheader("Relevant transcript excerpts:")
+
+        # Create a DataFrame from the search results
+        df = pd.DataFrame(search_results)
+
+        # Rename columns for better readability
+        df.columns = ['Speaker', 'Company', 'Timestamp', 'Quote']
+
+        # Display the DataFrame as a table
+        st.table(df)
+
+
+if __name__ == "__main__":
+    main()

dsp-urls.txt

@@ -0,0 +1,8 @@
+https://www.telecomtv.com/content/dsp-leaders-forum/directing-and-accelerating-network-innovation-50535/
+https://www.telecomtv.com/content/dsp-leaders-forum/enabling-the-autonomous-network-with-ai-50536/
+https://www.telecomtv.com/content/dsp-leaders-forum/creating-cloud-native-software-engineering-teams-50537/
+https://www.telecomtv.com/content/dsp-leaders-forum/building-digital-infrastructure-from-core-to-edge-50538/
+https://www.telecomtv.com/content/dsp-leaders-forum/unlocking-platform-opportunities-with-network-apis-50539/
+https://www.telecomtv.com/content/dsp-leaders-forum/leveraging-telco-cloud-for-advanced-operations-50540/
+https://www.telecomtv.com/content/dsp-leaders-forum/improving-network-optimisation-through-automation-50541/
+https://www.telecomtv.com/content/dsp-leaders-forum/focusing-on-the-customer-50542/

ec-app.py

@@ -0,0 +1,106 @@
+from ec_config import create_app
+from ttv_web_scraper import db_load_metadata_sets
+import streamlit as st
+import re
+
+
+@st.cache_resource
+def embedchain_bot():
+    return create_app()  # Use the create_app function from config.py
+
+
+def timestamp_to_seconds(timestamp):
+    """Convert a timestamp in the format 'hh:mm:ss' or 'mm:ss' to total seconds."""
+    parts = timestamp.split(':')
+    if len(parts) == 3:
+        h, m, s = map(int, parts)
+        ts = h * 3600 + m * 60 + s
+    elif len(parts) == 2:
+        m, s = map(int, parts)
+        ts = m * 60 + s
+    else:
+        raise ValueError(f"Invalid timestamp format: {timestamp}")
+
+    return ts
+
+
+def create_filter_panel(speakers, companies, sentiments, subjects):
+    st.sidebar.header("Filter Options")
+
+    selected_speaker = st.sidebar.selectbox(
+        "Select Speaker", [""] + list(speakers))
+    selected_company = st.sidebar.selectbox(
+        "Select Company", [""] + list(companies))
+    selected_sentiment = st.sidebar.selectbox(
+        "Select Sentiment", [""] + list(sentiments))
+    selected_subject = st.sidebar.selectbox(
+        "Select Subject", [""] + list(subjects))
+
+    where = {}
+    if selected_speaker:
+        where['speaker'] = selected_speaker
+    if selected_company:
+        where['companies'] = selected_company
+    if selected_sentiment:
+        where['sentiment'] = selected_sentiment
+    if selected_subject:
+        where['subject'] = selected_subject
+
+    return where
+
+
+# Streamlit app
+
+
+def main():
+    st.title("DSP Leaders World Forum 2024 ChatBot")
+
+    st.markdown(
+        "Trained on data from [here](https://www.telecomtv.com/content/dsp-leaders-forum-videos/)")
+
+    # Load metadata sets
+    _, speakers, companies, sentiments, subjects = db_load_metadata_sets()
+
+    # Create filter panel
+    where = create_filter_panel(speakers, companies, sentiments, subjects)
+
+    # User input
+    user_query = st.text_input(
+        "Enter your question:", placeholder="e.g. What are people speaking about? or List all people speaking")
+
+    # Add a slider for selecting the number of results
+    num_results = st.slider("Number of relevant transcript excerpts to show:",
+                            min_value=1, max_value=50, value=30, step=1)
+
+    if user_query:
+        app = embedchain_bot()
+
+        msg_placeholder = st.empty()
+        msg_placeholder.markdown("Thinking...")
+
+        # Use app.search() with the where parameter
+        search_results = app.search(
+            user_query, num_documents=num_results, where=where)
+
+        # Process and display search results
+        answer = "Here are the most relevant transcript excerpts:\n\n"
+        for i, result in enumerate(search_results, 1):
+            metadata = result['metadata']
+            ts = timestamp_to_seconds(metadata['timestamp'])
+            yt_url = f"https://youtu.be/{metadata['youtube_id']}?t={ts}"
+
+            speaker_info = (
+                f"Speaker: {metadata.get('speaker', 'Unknown')}, "
+                f"Company: {metadata.get('company', 'Unknown')}, "
+                f"Timestamp: {metadata.get('timestamp', 'Unknown')}"
+            )
+
+            answer += f"{i}. [Speaker Info: {speaker_info}]({yt_url})  \n"
+            answer += f"{metadata.get('title', 'Unknown')}  \n"
+            answer += f"\"{result['context']}\"\n\n"
+
+        msg_placeholder.markdown(answer)
+
+
+if __name__ == "__main__":
+    main()

ec_config.py

@@ -0,0 +1,60 @@
+
+import os
+import streamlit as st
+from embedchain import App
+
+# Load environment variables
+try:
+    from dotenv import load_dotenv
+    load_dotenv()
+except ImportError:
+    pass  # dotenv not installed, likely running on Hugging Face Spaces
+
+# Function to get the API key
+
+
+def get_api_key(name):
+    api_key = os.environ.get(name)
+    if not api_key:
+        api_key = st.secrets.get(name)
+    if not api_key:
+        raise ValueError(
+            f"{name} is not set. Please set it in your environment or Streamlit secrets.")
+    return api_key
+
+
+config_dict = {
+    'app': {
+        'config': {
+            'name': 'ttv-ec'
+        }
+    },
+    'llm': {
+        'provider': 'huggingface',
+        'config': {
+            'model': 'mistralai/Mistral-7B-Instruct-v0.2',
+            'top_p': 0.5,
+            'stream': False,
+            'prompt': """You are an AI assistant that answers questions based solely on the information provided in your knowledge base.
+
+Question: $query
+Context: $context
+
+If the information to answer a question is not available in your knowledge base,
+respond with 'I don't have enough information to answer that question.
+""",
+            'api_key': get_api_key('HF_TOKEN')
+        }
+    },
+    'embedder': {
+        'provider': 'huggingface',
+        'config': {
+            'model': 'sentence-transformers/all-mpnet-base-v2',
+            'api_key': get_api_key('HF_TOKEN')
+        }
+    }
+}
+
+
+def create_app():
+    return App.from_config(config=config_dict)

requirements.txt

@@ -0,0 +1,6 @@
+streamlit
+embedchain
+langchain_huggingface
+watchdog
+pyppeteer
+beautifulsoup4

ttv_web_scraper.py

@@ -0,0 +1,264 @@
+import re
+import asyncio
+import json
+import os
+import traceback
+from pyppeteer import launch
+from bs4 import BeautifulSoup
+import hashlib
+from ec_config import create_app
+
+
+CACHE_DIR = "cache/"
+if not os.path.exists(CACHE_DIR):
+    os.makedirs(CACHE_DIR)
+
+DB_METADATA_FILE = os.path.join(CACHE_DIR, "db_metadata.json")
+
+
+def db_load_metadata_sets():
+
+    content_hashes = set()
+    speakers = set()
+    companies = set()
+    sentiments = set()
+    subjects = set()
+
+    if os.path.exists(DB_METADATA_FILE):
+        with open(DB_METADATA_FILE, 'r') as f:
+            metadata = json.load(f)
+
+        content_hashes = set(metadata.get('content_hashes', []))
+        speakers = set(metadata.get('speakers', []))
+        companies = set(metadata.get('companies', []))
+        sentiments = set(metadata.get('sentiments', []))
+        subjects = set(metadata.get('subjects', []))
+
+    return content_hashes, speakers, companies, sentiments, subjects
+
+
+def save_metadata_sets(content_hashes, speakers, companies, sentiments, subjects):
+
+    metadata = {
+        'content_hashes': list(content_hashes),
+        'speakers': list(speakers),
+        'companies': list(companies),
+        'sentiments': list(sentiments),
+        'subjects': list(subjects)
+    }
+
+    with open(DB_METADATA_FILE, 'w') as f:
+        json.dump(metadata, f, indent=2)
+
+
+async def get_client_rendered_content(url):
+    browser = None
+    try:
+        browser = await launch()
+        page = await browser.newPage()
+        await page.goto(url, {'waitUntil': 'networkidle0', 'timeout': 60000})
+        await asyncio.sleep(5)
+        content = await page.content()
+        return content
+    except Exception as e:
+        raise Exception(f"Error fetching content: {str(e)}")
+    finally:
+        if browser:
+            await browser.close()
+
+
+def extract_info(html_content):
+    try:
+        soup = BeautifulSoup(html_content, 'html.parser')
+        title = soup.title.string.strip() if soup.title else None
+        date_elem = soup.find('p', class_='content-date')
+        date = date_elem.find(
+            'span', class_='ng-binding').text.strip() if date_elem else None
+        youtube_iframe = soup.find(
+            'iframe', src=lambda x: x and 'youtube.com' in x)
+        youtube_url = youtube_iframe['src'] if youtube_iframe else None
+        youtube_id = None
+        if youtube_url:
+            match = re.search(r'youtube\.com/embed/([^?]+)', youtube_url)
+            if match:
+                youtube_id = match.group(1)
+        transcript_elem = soup.find(id='transcript0')
+        transcript = transcript_elem.get_text(
+            strip=True) if transcript_elem else None
+        return {
+            'metadata': {
+                'title': title,
+                'date': date,
+                'youtube_id': youtube_id,
+            },
+            'transcript': transcript
+        }
+    except Exception as e:
+        raise Exception(f"Error extracting information: {str(e)}")
+
+
+def read_html_from_file(filename):
+    try:
+        if os.path.exists(filename):
+            with open(filename, 'r', encoding='utf-8') as f:
+                return f.read()
+        return None
+    except Exception as e:
+        raise Exception(f"Error reading file {filename}: {str(e)}")
+
+
+def read_json_from_file(filename):
+    try:
+        if os.path.exists(filename):
+            with open(filename, 'r', encoding='utf-8') as f:
+                return json.load(f)
+        return None
+    except json.JSONDecodeError as e:
+        raise Exception(f"Error decoding JSON in file {filename}: {str(e)}")
+    except Exception as e:
+        raise Exception(f"Error reading file {filename}: {str(e)}")
+
+
+def extract_speaker_info(segment):
+    try:
+        pattern = r'(?P<speaker>(?:[A-Z][a-z]+ ){1,3}[A-Z][a-z]+), (?P<company>[A-Za-z\s]+)\((?P<timestamp>(?:\d{2}:)?\d{2}:\d{2})\):'
+        match = re.match(pattern, segment)
+        if match:
+            return {key: value.strip() if value else None for key, value in match.groupdict().items()}
+        else:
+            timestamp_pattern = r'\((?P<timestamp>(?:\d{2}:)?\d{2}:\d{2})\):'
+            timestamp_match = re.match(timestamp_pattern, segment)
+            if timestamp_match:
+                return {'speaker': None, 'company': None, 'timestamp': timestamp_match.group('timestamp')}
+        return None
+    except Exception as e:
+        raise Exception(f"Error extracting speaker info: {str(e)}")
+
+
+def parse_transcript(content):
+    try:
+        parsed_segments = []
+        metadata = {}
+        pattern = r'((?:[A-Z][a-z]+ ){1,3}[A-Z][a-z]+, [A-Za-z\s]+\((?:\d{2}:)?\d{2}:\d{2}\):|\((?:\d{2}:)?\d{2}:\d{2}\):)'
+        segments = re.split(pattern, content)
+        segments = [segment.strip() for segment in segments if segment.strip()]
+        for segment in segments:
+            speaker_info = extract_speaker_info(segment)
+            if (speaker_info):
+                if speaker_info['speaker']:
+                    metadata = speaker_info.copy()
+                else:
+                    metadata = metadata.copy()
+                    metadata['timestamp'] = speaker_info['timestamp']
+            else:
+                parsed_segments.append({
+                    'metadata': metadata,
+                    "text": segment
+                })
+        return parsed_segments
+    except Exception as e:
+        raise Exception(f"Error parsing transcript: {str(e)}")
+
+
+def get_cached_filename(url):
+    return f"{CACHE_DIR}cached_{url.replace('://', '_').replace('/', '_')}.html"
+
+
+async def process_url(url):
+    try:
+        cached_filename = get_cached_filename(url)
+        json_filename = f"{cached_filename}.json"
+        info = read_json_from_file(json_filename)
+
+        if info:
+            return info
+
+        content = read_html_from_file(cached_filename)
+
+        if content is None:
+            print(f"Fetching content from web for {url}...")
+            content = await get_client_rendered_content(url)
+            with open(cached_filename, 'w', encoding='utf-8') as f:
+                f.write(content)
+        else:
+            print(f"Using cached content from file for {url}...")
+
+        info = extract_info(content)
+        transcript = info['transcript']
+        if (transcript):
+            info['transcript'] = parse_transcript(transcript)
+            with open(json_filename, 'w', encoding='utf-8') as f:
+                json.dump(info, f, ensure_ascii=False, indent=4)
+            print(f"Information extracted and saved to {json_filename}")
+        else:
+            print(f"No transcript found for {url}")
+        return info
+
+    except Exception as e:
+        print(f"Error processing URL {url}:")
+        print(traceback.format_exc())
+        print(f"Detailed error: {str(e)}")
+        return None
+
+# This function can be used to process multiple URLs
+
+
+async def process_urls(urls):
+    tasks = [process_url(url) for url in urls]
+    return await asyncio.gather(*tasks)
+
+
+def main():
+
+    app = create_app()
+
+    url_file = "dsp-urls.txt"  # File containing list of URLs
+
+    if not os.path.exists(url_file):
+        print(f"Error: {url_file} not found.")
+        return
+
+    content_hashes, speakers, companies, sentiments, subjects = db_load_metadata_sets()
+
+    with open(url_file, 'r') as f:
+        urls = [line.strip() for line in f if line.strip()]
+
+    for url in urls:
+        # Generate a hash of the url
+        filename_hash = hashlib.md5(url.encode()).hexdigest()
+        # Check if this content has already been added
+        if filename_hash in content_hashes:
+            print(f"{url} already added")
+            continue
+
+        info = asyncio.run(process_url(url))
+        if info is None:
+            continue
+
+        metadata = info['metadata']
+        transcript = info['transcript']
+
+        if transcript is None:
+            continue
+
+        for entry in transcript:
+            metadata.update(entry['metadata'])
+            speakers.add(metadata['speaker'])
+            companies.add(metadata['company'])
+
+            text = entry['text']
+
+            app.add(text, data_type='text', metadata=metadata)
+
+        content_hashes.add(filename_hash)
+        print(f"Added new url: {url}")
+
+    # Save updated hashes
+    save_metadata_sets(content_hashes, speakers,
+                       companies, sentiments, subjects)
+
+    print("Processing complete. Check individual URL outputs for any errors.")
+
+
+if __name__ == "__main__":
+    main()