Upload 6 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Research/video.mp4 filter=lfs diff=lfs merge=lfs -text

Research/checking.ipynb

@@ -0,0 +1,96 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dotenv import load_dotenv\n",
+    "load_dotenv()\n",
+    "import os \n",
+    "\n",
+    "api_key = os.environ.get('GOOGLE_API_KEY')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import google.generativeai as genai\n",
+    "genai.configure(api_key = api_key)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "file = genai.upload_file(\"video.mp4\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.llms.gemini import Gemini\n",
+    "llm = Gemini(model_name = \"models/gemini-1.5-pro\")\n",
+    "response = llm.complete([\"tell me the content of this video\",file])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'This video is a tutorial on linear regression. The narrator explains that linear regression is a statistical technique for modeling the relationship between an output variable and one or more input variables. The narrator explains that this is done by fitting a line through data points. The narrator explains the linear function y = mx + b, where y is the output variable, x is the input variable, m is the slope of the line, and b is the intercept of the line. The narrator explains that the coefficients m and b are what are solved for in linear regression. The narrator explains that the differences between the points and the line are called residuals. The narrator explains that the sum of the squared errors is the loss function. The narrator explains that the coefficients can be solved with a variety of techniques, including matrix decomposition and gradient descent. The narrator explains that to validate a linear regression, a third of the data is put into a test data set, and the remaining two-thirds become the training data set. The training data set is used to fit the regression line, and the test data set is used to validate the regression line. The narrator explains that metrics used to evaluate the linear regression vary from the r-squared, standard error of the estimate, prediction intervals, and statistical significance. The narrator recommends two books, “Essential Math for Data Science” and “Getting Started with SQL.” The narrator also teaches classes on the O’Reilly platform, including machine learning from scratch, probability, and SQL.'"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "response.text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

Research/notebook1.ipynb

@@ -0,0 +1,183 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dotenv import load_dotenv\n",
+    "load_dotenv()\n",
+    "import os \n",
+    "\n",
+    "api_key = os.environ.get('GOOGLE_API_KEY')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import google.generativeai as genai\n",
+    "genai.configure(api_key = api_key)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "Model = genai.GenerativeModel(model_name=\"models/gemini-1.5-pro\")\n",
+    "\n",
+    "file = genai.upload_file(\"video.mp4\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "response = Model.generate_content([\"Tell me the summury of this video\",file])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'This video explains linear regression, a statistical technique used to model the relationship between an output variable and one or more input variables. In simpler terms, this means fitting a line through data points and making predictions using that line. The formula for this is y equals mx + b, where y is the output variable, also called the dependent variable. The x is the input variable, called the independent variable. The m and b variables are the coefficients that are solved for in linear regression. The m variable controls the slope of the line. The b variable controls the intercept of the line. These are also referred to as beta1 and beta0. This equation can also have multiple input variables x1, x2, and x3.\\n\\nThe video also explains how to find the best fit for the regression line using residuals or the differences between the data points and the line. Squaring the residuals, then totaling these squares for a given line will get the sum of squared error (SSE). The beta coefficients that minimize the SSE are the most appropriate for the data.\\n\\nTo validate the regression, machine learning practitioners often divide the data into a training set and a test set. They use the training set to fit the regression line, then use the test set to validate the line. They use the r-squared, standard error, prediction intervals, and statistical significance metrics to evaluate the regression.'"
+      ]
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "response.text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/markdown": [
+       "This video explains linear regression, a statistical technique used to model the relationship between an output variable and one or more input variables. In simpler terms, this means fitting a line through data points and making predictions using that line. The formula for this is y equals mx + b, where y is the output variable, also called the dependent variable. The x is the input variable, called the independent variable. The m and b variables are the coefficients that are solved for in linear regression. The m variable controls the slope of the line. The b variable controls the intercept of the line. These are also referred to as beta1 and beta0. This equation can also have multiple input variables x1, x2, and x3.\n",
+       "\n",
+       "The video also explains how to find the best fit for the regression line using residuals or the differences between the data points and the line. Squaring the residuals, then totaling these squares for a given line will get the sum of squared error (SSE). The beta coefficients that minimize the SSE are the most appropriate for the data.\n",
+       "\n",
+       "To validate the regression, machine learning practitioners often divide the data into a training set and a test set. They use the training set to fit the regression line, then use the test set to validate the line. They use the r-squared, standard error, prediction intervals, and statistical significance metrics to evaluate the regression."
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from IPython.display import Markdown\n",
+    "display(Markdown(response.text))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "response = Model.generate_content([\"what is the equations provided in the video\",file])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'The narrator in this video describes the basic process of using linear regression, a type of statistical modeling that assumes a linear relationship between variables. In the equation, the output or dependent variable y can be expressed as a function of an input or independent variable, such as x, or as several input variables, x1, x2, x3, etc. He shows the two main ways these equations are generally expressed:\\n\\ny=mx+b\\nf(x) = mx + b\\ny = β1x + β0\\ny = β2x2 + β1x1 + β0\\ny = β3x3 + β2x2 + β1x1 + β0\\ny = β4x4 + β3x3 + β2x2 + β1x1 + β0\\ny = β5x5 + β4x4 + β3x3 + β2x2 + β1x1 + β0\\n\\n\\n'"
+      ]
+     },
+     "execution_count": 31,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "response.text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/markdown": [
+       "The narrator in this video describes the basic process of using linear regression, a type of statistical modeling that assumes a linear relationship between variables. In the equation, the output or dependent variable y can be expressed as a function of an input or independent variable, such as x, or as several input variables, x1, x2, x3, etc. He shows the two main ways these equations are generally expressed:\n",
+       "\n",
+       "y=mx+b\n",
+       "f(x) = mx + b\n",
+       "y = β1x + β0\n",
+       "y = β2x2 + β1x1 + β0\n",
+       "y = β3x3 + β2x2 + β1x1 + β0\n",
+       "y = β4x4 + β3x3 + β2x2 + β1x1 + β0\n",
+       "y = β5x5 + β4x4 + β3x3 + β2x2 + β1x1 + β0\n",
+       "\n",
+       "\n"
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "display(Markdown(response.text))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

Research/video.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da41d641e9f6eba6ccffe1ffe52a428fa02118df369716d537e11484904c877e
+size 6121500

backend.py

@@ -0,0 +1,68 @@
+import google.generativeai as genai
+from llama_index.llms.gemini import Gemini
+from llama_index.embeddings.gemini import GeminiEmbedding
+import os
+import tempfile
+from llama_index.core import VectorStoreIndex, SimpleDirectoryReader
+from llama_index.core import Settings
+import time
+from google.api_core.exceptions import GoogleAPIError
+import streamlit as st
+
+genai.configure(api_key=os.environ.get("GOOGLE_API_KEY"))
+
+llm = Gemini(model_name="models/gemini-1.5-pro")
+embeddings = GeminiEmbedding(model_name="models/embedding-001")
+
+def normal_response(query):
+    prompt = """You are a helpful Bot named VisionLang Build by Parthib Karak.
+    Given a question, generate answer based on the Question.
+    Question: {question}
+    """
+    try:
+        response = llm.complete(prompt + query)
+        return response.text
+    except GoogleAPIError as e:
+        return f"Error generating response: {str(e)}"
+
+def uploaded_file_to_response(file, query):
+    file_extension = os.path.splitext(file.name)[-1].lower()
+    try:
+        if file_extension in [".pdf", ".docx", ".txt", ".py", ".js", ".java", ".cpp"]:
+            temp_dir = tempfile.mkdtemp()
+            temp_file_path = os.path.join(temp_dir, file.name)
+            with open(temp_file_path, "wb") as f:
+                f.write(file.read())
+            
+            document = SimpleDirectoryReader(temp_dir)
+            data = document.load_data()
+            Settings.llm = llm
+            Settings.embed_model = embeddings
+            index = VectorStoreIndex.from_documents(data, settings=Settings)
+            query_engine = index.as_query_engine()
+            response = query_engine.query(query)
+            return response
+        elif file_extension in [".mp4", ".avi", ".mov",".mkv"]:
+            temp_dir = tempfile.mkdtemp()
+            temp_file_path = os.path.join(temp_dir, file.name)
+            with open(temp_file_path, "wb") as f:
+                f.write(file.read())
+
+            uploaded_file = genai.upload_file(temp_file_path, mime_type="video/mp4")
+            st.success("video uploaded successfully")
+            time.sleep(2)
+            response = llm.complete([query, uploaded_file])
+            return response.text
+        elif file_extension in [".png", ".jpg", ".jpeg"]:
+            uploaded_file = genai.upload_file(file, mime_type="image/jpeg")
+            time.sleep(2)
+            response = llm.complete([query, uploaded_file])
+            return response.text
+        else:
+            uploaded_file = genai.upload_file(file, mime_type="application/octet-stream")
+            time.sleep(2)
+            response = llm.complete([query, uploaded_file])
+            return response.text
+    except GoogleAPIError as e:
+        return f"Error processing file: {str(e)}"
+

main.py

@@ -0,0 +1,53 @@
+import streamlit as st
+from backend import uploaded_file_to_response, normal_response
+from llama_index.llms.gemini import Gemini
+from llama_index.embeddings.gemini import GeminiEmbedding
+import google.generativeai as genai
+import os
+from dotenv import load_dotenv
+load_dotenv()
+
+genai.configure(api_key=os.environ.get("GOOGLE_API_KEY"))
+
+llm = Gemini(model_name="models/gemini-1.5-pro")
+embeddings = GeminiEmbedding(model_name="models/embedding-001")
+
+if "chat_history" not in st.session_state:
+    st.session_state.chat_history = []
+
+st.markdown("""
+<style>
+    .stApp { background-color: #ffffff; color: black; font-family: 'Arial', sans-serif; }
+    .title { font-size: 36px; font-weight: bold; text-align: center; animation: fadeIn 2s ease-in-out; }
+    @keyframes fadeIn { from { opacity: 0; } to { opacity: 1; } }
+    .chat-container { max-height: 500px; overflow-y: auto; display: flex; flex-direction: column-reverse; padding: 10px; border-radius: 10px; background: rgba(0, 0, 0, 0.05); margin-top: 10px; }
+    .user-message { background: #0078ff; color: white; padding: 10px; border-radius: 10px; margin-bottom: 5px; text-align: left; }
+    .ai-message { background: #f1f1f1; color: black; padding: 10px; border-radius: 10px; margin-bottom: 5px; text-align: left; }
+    .btn-style { background: linear-gradient(45deg, #ff007f, #ff0055); color: white; padding: 8px 16px; border-radius: 6px; font-size: 14px; margin-top: 10px; transition: 0.3s ease-in-out; border: none; cursor: pointer; }
+    .btn-style:hover { background: linear-gradient(45deg, #ff0055, #d4005a); }
+</style>
+""", unsafe_allow_html=True)
+
+st.markdown("<h1 class='title'>🧠 AI Code Companion</h1>", unsafe_allow_html=True)
+st.caption("🚀 Upload files & chat with AI")
+
+uploaded_file = st.file_uploader("Upload a File (Image, Document, Code, Video, or Audio)", type=["png", "jpg", "jpeg", "pdf", "docx", "txt", "py", "js", "java", "cpp", "mp4"], key="file_uploader")
+
+user_input = st.text_input("Type your message here...", key="chat_input", help="Chat with AI", label_visibility="collapsed")
+
+if st.button("Generate Response", key="generate_button", help="Click to get AI response", use_container_width=False):
+    if user_input:
+        with st.spinner("Processing..."):
+            response = normal_response(user_input)
+            if uploaded_file:
+                response = uploaded_file_to_response(uploaded_file, user_input)
+            st.session_state.chat_history.insert(0, (user_input, response))
+
+if st.session_state.chat_history:
+    chat_container = st.container()
+    with chat_container:
+        st.markdown("### Chat History")
+        for user_msg, ai_response in st.session_state.chat_history:
+            st.markdown(f"<div class='user-message'><b>You:</b> {user_msg}</div>", unsafe_allow_html=True)
+            st.markdown(f"<div class='ai-message'><b>AI:</b> {ai_response}</div>", unsafe_allow_html=True)
+            st.markdown("<br>", unsafe_allow_html=True)

requirements.txt

@@ -0,0 +1,4 @@
+google-generativeai
+llama-index-llms-gemini
+llama-index-embeddings-gemini
+python-dotenv