Spaces:

hemangthakur
/

seekr

Paused

App Files Files Community

Hemang Thakur commited on 4 days ago

Commit

44ebcd1

1 Parent(s): 4aefbee

demo is ready

Browse files

Files changed (39) hide show

frontend/package-lock.json +0 -0
frontend/package.json +2 -1
frontend/public/auth-receiver.html +107 -0
frontend/src/Components/AiComponents/ChatComponents/SourcePopup.css +77 -0
frontend/src/Components/AiComponents/ChatComponents/SourcePopup.js +187 -0
frontend/src/Components/AiComponents/ChatComponents/SourceRef.css +21 -0
frontend/src/Components/AiComponents/ChatComponents/Streaming.css +161 -119
frontend/src/Components/AiComponents/ChatComponents/Streaming.js +10 -64
frontend/src/Components/AiComponents/ChatWindow.css +15 -5
frontend/src/Components/AiComponents/ChatWindow.js +106 -3
frontend/src/Components/AiComponents/Dropdowns/AddContentDropdown.css +150 -0
frontend/src/Components/AiComponents/Dropdowns/AddContentDropdown.js +354 -0
frontend/src/Components/AiComponents/Dropdowns/AddFilesDialog.css +191 -0
frontend/src/Components/AiComponents/Dropdowns/AddFilesDialog.js +282 -0
frontend/src/Components/AiComponents/Markdown/CustomMarkdown.js +489 -0
frontend/src/Components/AiComponents/Markdown/TestMarkdown.js +120 -0
frontend/src/Components/AiComponents/Notifications/Notification.css +379 -0
frontend/src/Components/AiComponents/Notifications/Notification.js +242 -0
frontend/src/Components/AiComponents/Notifications/useNotification.js +43 -0
frontend/src/Components/AiComponents/Sidebars/LeftSideBar.js +38 -0
frontend/src/Components/AiComponents/Sidebars/LeftSidebar.css +59 -0
frontend/src/Components/AiComponents/Sidebars/RightSidebar.css +138 -0
frontend/src/Components/AiComponents/Sidebars/RightSidebar.js +142 -0
frontend/src/Components/AiPage.css +66 -6
frontend/src/Components/AiPage.js +611 -32
frontend/src/Components/IntialSetting.css +1 -1
frontend/src/Components/IntialSetting.js +24 -17
frontend/src/Icons/excerpts.png +0 -0
frontend/src/Icons/excerpts.pngZone.Identifier +4 -0
main.py +467 -48
src/crawl/crawler.py +566 -566
src/helpers/helper.py +33 -1
src/integrations/mcp_client.py +506 -0
src/query_processing/query_processor.py +55 -16
src/rag/graph_rag.py +12 -8
src/rag/neo4j_graphrag.py +31 -5
src/reasoning/reasoner.py +226 -19
src/search/search_engine.py +5 -5
src/utils/api_key_manager.py +24 -45

frontend/package-lock.json CHANGED Viewed

The diff for this file is too large to render. See raw diff

frontend/package.json CHANGED Viewed

@@ -3,7 +3,7 @@
   "version": "0.1.0",
   "private": true,
   "dependencies": {
- "@emotion/react": "^11.14.0",
     "@emotion/styled": "^11.14.0",
     "@fortawesome/fontawesome-free": "^6.7.2",
     "@google/generative-ai": "^0.21.0",
@@ -23,6 +23,7 @@
     "rehype-katex": "^6.0.2",
     "rehype-raw": "^6.1.1",
     "rehype-sanitize": "^5.0.1",
     "remark-gfm": "^3.0.1",
     "remark-math": "^5.1.1",
     "styled-components": "^6.1.14",

   "version": "0.1.0",
   "private": true,
   "dependencies": {
+    "@emotion/react": "^11.14.0",
     "@emotion/styled": "^11.14.0",
     "@fortawesome/fontawesome-free": "^6.7.2",
     "@google/generative-ai": "^0.21.0",
     "rehype-katex": "^6.0.2",
     "rehype-raw": "^6.1.1",
     "rehype-sanitize": "^5.0.1",
+    "remark-breaks": "^4.0.0",
     "remark-gfm": "^3.0.1",
     "remark-math": "^5.1.1",
     "styled-components": "^6.1.14",

frontend/public/auth-receiver.html ADDED Viewed

	@@ -0,0 +1,107 @@

+<!DOCTYPE html>
+<html>
+<head>
+  <title>Completing Authentication...</title>
+  <style>
+    body {
+      font-family: Arial, sans-serif;
+      display: flex;
+      justify-content: center;
+      align-items: center;
+      height: 100vh;
+      margin: 0;
+      background-color: #f5f5f5;
+    }
+    .container {
+      text-align: center;
+      padding: 20px;
+      background: white;
+      border-radius: 8px;
+      box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+    }
+    .spinner {
+      border: 3px solid #f3f3f3;
+      border-top: 3px solid #3498db;
+      border-radius: 50%;
+      width: 40px;
+      height: 40px;
+      animation: spin 1s linear infinite;
+      margin: 20px auto;
+    }
+    @keyframes spin {
+      0% { transform: rotate(0deg); }
+      100% { transform: rotate(360deg); }
+    }
+  </style>
+</head>
+<body>
+  <div class="container">
+    <h2>Completing authentication...</h2>
+    <div class="spinner"></div>
+    <p id="status">Please wait...</p>
+  </div>
+  <script>
+    function updateStatus(message) {
+      document.getElementById('status').textContent = message;
+    }
+    try {
+      // Extract token from URL
+      let token = null;
+      // For Google and Microsoft (token in hash)
+      if (window.location.hash) {
+        const hashParams = new URLSearchParams(window.location.hash.substring(1));
+        token = hashParams.get('access_token');
+      }
+      // For Slack (token might be in query params)
+      if (!token && window.location.search) {
+        const queryParams = new URLSearchParams(window.location.search);
+        token = queryParams.get('access_token');
+        // Slack might return a code instead of token
+        const code = queryParams.get('code');
+        if (code && !token) {
+          updateStatus('Slack requires additional setup. Please implement code exchange.');
+          setTimeout(() => {
+            if (window.opener) {
+              window.opener.postMessage({ type: 'auth-failed', error: 'Slack code exchange not implemented' }, '*');
+              window.close();
+            }
+          }, 3000);
+        }
+      }
+      if (token) {
+        updateStatus('Authentication successful! Closing window...');
+        // Send token back to parent window
+        if (window.opener) {
+          window.opener.postMessage({
+            type: 'auth-success',
+            token: token
+          }, window.location.origin);
+          // Close window after a short delay
+          setTimeout(() => window.close(), 1000);
+        } else {
+          updateStatus('Unable to communicate with main window. Please close this window manually.');
+        }
+      } else {
+        updateStatus('Authentication failed. No token received.');
+        setTimeout(() => {
+          if (window.opener) {
+            window.opener.postMessage({ type: 'auth-failed', error: 'No token received' }, '*');
+            window.close();
+          }
+        }, 3000);
+      }
+    } catch (error) {
+      updateStatus('An error occurred: ' + error.message);
+      console.error('Auth error:', error);
+    }
+  </script>
+</body>
+</html>

frontend/src/Components/AiComponents/ChatComponents/SourcePopup.css ADDED Viewed

	@@ -0,0 +1,77 @@

+.source-popup {
+    position: absolute; /* Crucial for positioning */
+    /* transform is set inline based on calculation */
+    transition: opacity 0.2s ease-in-out, transform 0.2s ease-in-out;
+    opacity: 1; /* Start visible, manage via state */
+    pointer-events: auto; /* Allow interaction */
+    width: 300px; /* Or max-width */
+    max-width: 90vw;
+  }
+.source-popup-card {
+    background-color: #333 !important; /* Dark background */
+    color: #eee !important; /* Light text */
+    border: 1px solid #555 !important;
+    border-radius: 8px !important;
+    box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3) !important;
+    padding: 0.5rem !important; /* Reduced padding */
+  }
+.source-popup-card .MuiCardContent-root {
+     padding: 8px !important; /* Further reduce padding inside content */
+     padding-bottom: 8px !important; /* Ensure bottom padding is also reduced */
+  }
+.source-popup-title {
+    font-size: 0.9rem !important; /* Slightly smaller title */
+    font-weight: 600 !important;
+    margin-bottom: 0.3rem !important;
+    line-height: 1.3 !important;
+    color: #eee !important; /* Ensure title color */
+  }
+.source-popup-title a {
+     color: inherit !important; /* Inherit color for link */
+     text-decoration: none !important;
+  }
+.source-popup-title a:hover {
+     text-decoration: underline !important;
+  }
+.source-popup-link-info {
+    display: flex !important;
+    align-items: center !important;
+    font-size: 0.75rem !important; /* Smaller domain text */
+    color: #bbb !important;
+    margin-bottom: 0.4rem !important; /* Space below link info */
+  }
+.source-popup-icon {
+    width: 14px !important; /* Smaller icon */
+    height: 14px !important;
+    margin-right: 0.3rem !important;
+    vertical-align: middle; /* Align icon better */
+    filter: brightness(1.1); /* Slightly brighter icon */
+  }
+.source-popup-domain {
+    vertical-align: middle !important;
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+  }
+.source-popup-description {
+    font-size: 0.8rem !important; /* Smaller description text */
+    color: #ccc !important;
+    line-height: 1.4 !important;
+    /* Limit the number of lines shown */
+    display: -webkit-box;
+    -webkit-line-clamp: 3; /* Show max 3 lines */
+    -webkit-box-orient: vertical;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    margin-top: 0.4rem !important; /* Space above description */
+  }

frontend/src/Components/AiComponents/ChatComponents/SourcePopup.js ADDED Viewed

	@@ -0,0 +1,187 @@

+import React from 'react';
+import Card from '@mui/material/Card';
+import CardContent from '@mui/material/CardContent';
+import Typography from '@mui/material/Typography';
+import Link from '@mui/material/Link';
+import './SourcePopup.css';
+// Helper function to extract a friendly domain name from a URL.
+const getDomainName = (url) => {
+  try {
+    if (!url) return 'Unknown Source';
+    const hostname = new URL(url).hostname;
+    const domain = hostname.startsWith('www.') ? hostname.slice(4) : hostname;
+    const parts = domain.split('.');
+    return parts[0].charAt(0).toUpperCase() + parts[0].slice(1);
+  } catch (err) {
+    console.error("Error parsing URL for domain name:", url, err);
+    return 'Invalid URL';
+  }
+};
+// Helper function for Levenshtein distance calculation
+function levenshtein(a, b) {
+  if (a.length === 0) return b.length;
+  if (b.length === 0) return a.length;
+  const matrix = [];
+  for (let i = 0; i <= b.length; i++) matrix[i] = [i];
+  for (let j = 0; j <= a.length; j++) matrix[0][j] = j;
+  for (let i = 1; i <= b.length; i++) {
+    for (let j = 1; j <= a.length; j++) {
+      if (b.charAt(i - 1) === a.charAt(j - 1)) {
+        matrix[i][j] = matrix[i - 1][j - 1];
+      } else {
+        matrix[i][j] = Math.min(
+          matrix[i - 1][j - 1] + 1,
+          matrix[i][j - 1] + 1,
+          matrix[i - 1][j] + 1
+        );
+      }
+    }
+  }
+  return matrix[b.length][a.length];
+}
+// SourcePopup component to display source information and excerpts
+function SourcePopup({
+  sourceData,
+  excerptsData,
+  position,
+  onMouseEnter,
+  onMouseLeave,
+  statementText
+}) {
+  if (!sourceData || !position) return null;
+  const domain = getDomainName(sourceData.link);
+  let hostname = '';
+  try {
+    hostname = sourceData.link ? new URL(sourceData.link).hostname : '';
+  } catch (err) {
+    hostname = sourceData.link || ''; // Fallback to link if URL parsing fails
+  }
+  let displayExcerpt = null;
+  const sourceIdStr = String(sourceData.id);
+  // Find the relevant excerpt
+  if (excerptsData && Array.isArray(excerptsData) && statementText) {
+    let foundExcerpt = null;
+    let foundByFuzzy = false;
+    const norm = s => s.replace(/\s+/g, ' ').trim();
+    const lower = s => norm(s).toLowerCase();
+    const statementNorm = norm(statementText);
+    const statementLower = lower(statementText);
+    console.log(`[SourcePopup] Searching for excerpt for source ID ${sourceIdStr}: ${statementText}`);
+    // Iterate through the list of statement-to-excerpt mappings
+    for (const entry of excerptsData) {
+      const [thisStatement, sourcesMap] = Object.entries(entry)[0];
+      const thisNorm = norm(thisStatement);
+      const thisLower = lower(thisStatement);
+      console.log(`[SourcePopup] Checking against statement: ${thisStatement}`);
+      // Normalized exact match
+      if (thisNorm === statementNorm && sourcesMap && sourceIdStr in sourcesMap) {
+        foundExcerpt = sourcesMap[sourceIdStr];
+        break;
+      }
+      // Case-insensitive match
+      if (thisLower === statementLower && sourcesMap && sourceIdStr in sourcesMap) {
+        foundExcerpt = sourcesMap[sourceIdStr];
+        break;
+      }
+      // Substring containment
+      if (
+        (statementNorm && thisNorm && statementNorm.includes(thisNorm)) ||
+        (thisNorm && statementNorm && thisNorm.includes(statementNorm))
+      ) {
+        if (sourcesMap && sourceIdStr in sourcesMap) {
+          foundExcerpt = sourcesMap[sourceIdStr];
+          foundByFuzzy = true;
+          break;
+        }
+      }
+      // Levenshtein distance
+      if (
+        levenshtein(statementNorm, thisNorm) <= 5 &&
+        sourcesMap && sourceIdStr in sourcesMap
+      ) {
+        foundExcerpt = sourcesMap[sourceIdStr];
+        foundByFuzzy = true;
+        break;
+      }
+    }
+    // Set displayExcerpt based on what was found
+    if (foundExcerpt && foundExcerpt.toLowerCase() !== 'excerpt not found') {
+      if (foundByFuzzy) {
+        // Fuzzy match found an excerpt
+        console.log("[SourcePopup] Fuzzy match found an excerpt:", foundExcerpt);
+      } else {
+        // Exact match found an excerpt
+        console.log("[SourcePopup] Exact match found an excerpt:", foundExcerpt);
+      }
+      // Exact match found an excerpt
+      displayExcerpt = foundExcerpt;
+    } else if (foundExcerpt) {
+      // Handle case where LLM explicitly said "Excerpt not found"
+      displayExcerpt = "Relevant excerpt could not be automatically extracted.";
+      console.log("[SourcePopup] Excerpt marked as not found or invalid type:", foundExcerpt);
+    } else {
+      // Excerpt for this specific source ID wasn't found in the loaded data
+      displayExcerpt = "Excerpt not found for this citation.";
+      console.log(`[SourcePopup] Excerpt not found for source ID ${sourceIdStr}: ${statementText}`);
+    }
+  }
+  return (
+    <div
+      className="source-popup"
+      style={{
+        position: 'absolute', // Use absolute positioning
+        top: `${position.top}px`,
+        left: `${position.left}px`,
+        transform: 'translate(-50%, -100%)', // Center above the reference
+        zIndex: 1100, // Ensure it's above other content
+      }}
+      onMouseEnter={onMouseEnter} // Keep popup open when mouse enters it
+      onMouseLeave={onMouseLeave} // Hide popup when mouse leaves it
+    >
+      <Card variant="outlined" className="source-popup-card">
+        <CardContent>
+          <Typography variant="subtitle2" component="div" className="source-popup-title" gutterBottom>
+            <Link href={sourceData.link} target="_blank" rel="noopener noreferrer" underline="hover" color="inherit">
+              {sourceData.title || 'Untitled Source'}
+            </Link>
+          </Typography>
+          <Typography variant="body2" className="source-popup-link-info">
+            {hostname && (
+              <img
+                src={`https://www.google.com/s2/favicons?domain=${hostname}&sz=16`}
+                alt=""
+                className="source-popup-icon"
+              />
+            )}
+            <span className="source-popup-domain">{domain}</span>
+          </Typography>
+          {displayExcerpt !== null && (
+             <Typography variant="caption" className="source-popup-excerpt" display="block" sx={{ mt: 1 }}>
+               <Link
+                  href={`${sourceData.link}#:~:text=${encodeURIComponent(displayExcerpt)}`}
+                  target="_blank"
+                  rel="noopener noreferrer"
+                  underline="none"
+                  color="inherit"
+                >
+                  {displayExcerpt}
+                </Link>
+             </Typography>
+          )}
+        </CardContent>
+      </Card>
+    </div>
+  );
+};
+export default SourcePopup;

frontend/src/Components/AiComponents/ChatComponents/SourceRef.css ADDED Viewed

	@@ -0,0 +1,21 @@

+.source-reference {
+    display: inline-block;
+    vertical-align: super;
+    font-size: 0.75em;
+    line-height: 1;
+    margin: 0 0.15em;
+    padding: 0.2em 0.3em;
+    background-color: rgba(135, 131, 120, 0.265);
+    color: #a9a9a9;
+    border-radius: 0.35em;
+    cursor: pointer;
+    transition: background-color 0.2s ease, color 0.2s ease;
+    font-weight: 540;
+    position: relative;
+    top: 0.35em;
+  }
+.source-reference:hover {
+    background-color: rgba(135, 131, 120, 0.463);
+    color: #ffffff;
+}

frontend/src/Components/AiComponents/ChatComponents/Streaming.css CHANGED Viewed

@@ -1,120 +1,162 @@
 .streaming-content {
-    font-family: inherit;
-    line-height: 2rem;
-    white-space: pre-wrap;
-    word-wrap: break-word;
-    margin: 0;
-    padding: 0;
-  }
-  /* Reset margin/padding for all descendants */
-  .streaming-content * {
-    margin: 0;
-    padding: 0;
-  }
-  /* Top-level elements */
-  .streaming-content > * {
-    margin-top: 0.5rem;
-    margin-bottom: 0.5rem;
-  }
-  /* VERY FIRST element in an AI answer */
-  .streaming-content > *:first-child {
-    margin-top: 0 !important;
-  }
-  /* Headings */
-  .streaming-content h1,
-  .streaming-content h2,
-  .streaming-content h3,
-  .streaming-content h4,
-  .streaming-content h5,
-  .streaming-content h6 {
-    margin-top: 1rem;
-    margin-bottom: 0.75rem;
-  }
-  /* If heading is the very first element */
-  .streaming-content > h1:first-child,
-  .streaming-content > h2:first-child,
-  .streaming-content > h3:first-child,
-  .streaming-content > h4:first-child,
-  .streaming-content > h5:first-child,
-  .streaming-content > h6:first-child {
-    margin-top: 0 !important;
-  }
-  /* Paragraphs */
-  .streaming-content p {
-    margin-top: 0.25rem;
-    margin-bottom: 0.25rem;
-  }
-  /* Lists */
-  .streaming-content ul,
-  .streaming-content ol {
-    margin-top: 0.25rem;
-    margin-bottom: 0.25rem;
-    padding-left: 1.25rem;
-    white-space: normal;
-  }
-  .streaming-content li {
-    margin-bottom: 0.25rem;
-  }
-  .streaming-content li ul,
-  .streaming-content li ol {
-    margin-top: 0.15rem;
-    margin-bottom: 0.15rem;
-  }
-  /* Code Blocks */
-  .code-block-container {
-    margin: 0.5rem 0;
-    border-radius: 4px;
-    background-color: #2b2b2b;
-    overflow: hidden;
-  }
-  .code-block-header {
-    background-color: #1e1e1e;
-    color: #ffffff;
-    padding: 0.5rem;
-    font-size: 0.85rem;
-    font-weight: bold;
-  }
-  /* Table Container */
-  .table-container {
-    margin: 0.5rem 0;
-    width: 100%;
-    overflow-x: auto;
-    border: 1px solid #ddd;
-    border-radius: 4px;
-  }
-  .table-container th,
-  .table-container td {
-    border: 1px solid #ddd;
-    padding: 0.5rem;
-  }
-  /* Markdown Links */
-  .markdown-link {
-    color: #1a73e8;
-    text-decoration: none;
-  }
-  .markdown-link:hover {
-    text-decoration: underline;
-  }
-  /* Blockquotes */
-  .markdown-blockquote {
-    border-left: 4px solid #ccc;
-    padding-left: 0.75rem;
-    margin: 0.5rem 0;
-    color: #555;
-    font-style: italic;
-  }

 .streaming-content {
+  font-family: inherit;
+  line-height: 2rem;
+  white-space: pre-wrap;
+  word-wrap: break-word;
+  margin: 0;
+  padding: 0;
+}
+/* Reset margin/padding for all descendants */
+.streaming-content * {
+  margin: 0;
+  padding: 0;
+}
+/* Top-level elements */
+.streaming-content > * {
+  margin-top: 0.5rem;
+  margin-bottom: 0.5rem;
+}
+/* VERY FIRST element in an AI answer */
+.streaming-content > *:first-child {
+  margin-top: 0 !important;
+}
+/* Headings */
+.streaming-content h1,
+.streaming-content h2,
+.streaming-content h3,
+.streaming-content h4,
+.streaming-content h5,
+.streaming-content h6 {
+  margin-top: 1rem;
+  margin-bottom: 0.75rem;
+}
+/* If heading is the very first element */
+.streaming-content > h1:first-child,
+.streaming-content > h2:first-child,
+.streaming-content > h3:first-child,
+.streaming-content > h4:first-child,
+.streaming-content > h5:first-child,
+.streaming-content > h6:first-child {
+  margin-top: 0 !important;
+}
+/* All but the first child in streaming-content */
+.streaming-content p:not(:first-child),
+.streaming-content h1:not(:first-child),
+.streaming-content h2:not(:first-child),
+.streaming-content h3:not(:first-child),
+.streaming-content h4:not(:first-child),
+.streaming-content h5:not(:first-child),
+.streaming-content h6:not(:first-child) {
+  margin-top: -0.5em !important;
+}
+.streaming-content h1:not(:first-child),
+.streaming-content h2:not(:first-child),
+.streaming-content h3:not(:first-child),
+.streaming-content h4:not(:first-child),
+.streaming-content h5:not(:first-child),
+.streaming-content h6:not(:first-child) {
+  margin-bottom: -0.35em !important;
+}
+/* When a list follows a paragraph */
+.streaming-content p + ul,
+.streaming-content p + ol {
+  margin-top: -2rem !important;
+}
+/* When a list follows a header */
+.streaming-content h1 + ul,
+.streaming-content h2 + ul,
+.streaming-content h3 + ul,
+.streaming-content h4 + ul,
+.streaming-content h5 + ul,
+.streaming-content h6 + ul,
+.streaming-content h1 + ol,
+.streaming-content h2 + ol,
+.streaming-content h3 + ol,
+.streaming-content h4 + ol,
+.streaming-content h5 + ol,
+.streaming-content h6 + ol {
+  margin-top: -0.25rem !important;
+}
+/* Paragraphs */
+.streaming-content p {
+  margin-top: 0.25rem;
+  margin-bottom: 0.25rem;
+}
+/* Lists */
+.streaming-content ul,
+.streaming-content ol {
+  margin-top: 0.25rem;
+  margin-bottom: 0.25rem;
+  padding-left: 1.25rem;
+  white-space: normal;
+}
+.streaming-content li {
+  margin-bottom: 0.25rem;
+}
+.streaming-content li ul,
+.streaming-content li ol {
+  margin-top: 0.15rem;
+  margin-bottom: 0.15rem;
+}
+/* Code Blocks */
+.code-block-container {
+  margin: 0.5rem 0;
+  border-radius: 4px;
+  background-color: #2b2b2b;
+  overflow: hidden;
+}
+.code-block-header {
+  background-color: #1e1e1e;
+  color: #ffffff;
+  padding: 0.5rem;
+  font-size: 0.85rem;
+  font-weight: bold;
+}
+/* Table Container */
+.table-container {
+  margin: 0.5rem 0;
+  width: 100%;
+  overflow-x: auto;
+  border: 1px solid #ddd;
+  border-radius: 4px;
+}
+.table-container th,
+.table-container td {
+  border: 1px solid #ddd;
+  padding: 0.5rem;
+}
+/* Markdown Links */
+.markdown-link {
+  color: #1a73e8;
+  text-decoration: none;
+}
+.markdown-link:hover {
+  text-decoration: underline;
+}
+/* Blockquotes */
+.markdown-blockquote {
+  border-left: 4px solid #ccc;
+  padding-left: 0.75rem;
+  margin: 0.5rem 0;
+  color: #555;
+  font-style: italic;
+}

frontend/src/Components/AiComponents/ChatComponents/Streaming.js CHANGED Viewed

@@ -1,80 +1,26 @@
 import React, { useEffect, useRef } from 'react';
-import ReactMarkdown from 'react-markdown';
-import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';
-import { atomDark } from 'react-syntax-highlighter/dist/esm/styles/prism';
-import remarkGfm from 'remark-gfm';
-import rehypeRaw from 'rehype-raw';
 import './Streaming.css';
 // Streaming component for rendering markdown content
-const Streaming = ({ content, isStreaming, onContentRef }) => {
   const contentRef = useRef(null);
   useEffect(() => {
     if (contentRef.current && onContentRef) {
       onContentRef(contentRef.current);
     }
-  }, [content, onContentRef]);
-  const displayContent = isStreaming ? `${content}▌` : (content || '');
   return (
     <div className="streaming-content" ref={contentRef}>
-      <ReactMarkdown
-        remarkPlugins={[remarkGfm]}
-        rehypePlugins={[rehypeRaw]}
-        components={{
-          code({node, inline, className, children, ...props}) {
-            const match = /language-(\w+)/.exec(className || '');
-            return !inline ? (
-              <div className="code-block-container">
-                <div className="code-block-header">
-                  <span>{match ? match[1] : 'code'}</span>
-                </div>
-                <SyntaxHighlighter
-                  style={atomDark}
-                  language={match ? match[1] : 'text'}
-                  PreTag="div"
-                  {...props}
-                >
-                  {String(children).replace(/\n$/, '')}
-                </SyntaxHighlighter>
-              </div>
-            ) : (
-              <code className={className} {...props}>
-                {children}
-              </code>
-            );
-          },
-          table({node, ...props}) {
-            return (
-              <div className="table-container">
-                <table {...props} />
-              </div>
-            );
-          },
-          a({node, children, href, ...props}) {
-            return (
-              <a
-                href={href}
-                target="_blank"
-                rel="noopener noreferrer"
-                className="markdown-link"
-                {...props}
-              >
-                {children}
-              </a>
-            );
-          },
-          blockquote({node, ...props}) {
-            return (
-              <blockquote className="markdown-blockquote" {...props} />
-            );
-          }
-        }}
-      >
-        {displayContent}
-      </ReactMarkdown>
     </div>
   );
 };

 import React, { useEffect, useRef } from 'react';
+import CustomMarkdown from '../Markdown/CustomMarkdown';
 import './Streaming.css';
+import './SourceRef.css';
 // Streaming component for rendering markdown content
+const Streaming = ({ content, isStreaming, onContentRef, showSourcePopup, hideSourcePopup }) => {
   const contentRef = useRef(null);
   useEffect(() => {
     if (contentRef.current && onContentRef) {
       onContentRef(contentRef.current);
     }
+  }, [content, onContentRef]);
   return (
     <div className="streaming-content" ref={contentRef}>
+      <CustomMarkdown
+        content={content}
+        isStreaming={isStreaming}
+        showSourcePopup={showSourcePopup}
+        hideSourcePopup={hideSourcePopup}
+      />
     </div>
   );
 };

frontend/src/Components/AiComponents/ChatWindow.css CHANGED Viewed

@@ -136,7 +136,8 @@
 .post-icons .copy-icon,
 .post-icons .evaluate-icon,
 .post-icons .sources-icon,
-.post-icons .graph-icon {
   cursor: pointer;
   position: relative;
 }
@@ -145,14 +146,16 @@
 .post-icons .copy-icon img,
 .post-icons .evaluate-icon img,
 .post-icons .sources-icon img,
-.post-icons .graph-icon img {
   transition: filter var(--transition-speed);
 }
 .post-icons .copy-icon:hover img,
 .post-icons .evaluate-icon:hover img,
 .post-icons .sources-icon:hover img,
-.post-icons .graph-icon:hover img {
   filter: brightness(0.65);
 }
@@ -179,7 +182,8 @@
 .post-icons .copy-icon:hover .tooltip,
 .post-icons .evaluate-icon:hover .tooltip,
 .post-icons .sources-icon:hover .tooltip,
-.post-icons .graph-icon:hover .tooltip {
   opacity: 1;
   visibility: visible;
   transform: translateX(-50%) translateY(0) scale(1);
@@ -207,7 +211,13 @@
   object-fit: contain;
 }
-/* ChatWindow.css */
 /* Container for the loading state with a dark background */
 .bot-loading {

 .post-icons .copy-icon,
 .post-icons .evaluate-icon,
 .post-icons .sources-icon,
+.post-icons .graph-icon,
+.post-icons .excerpts-icon {
   cursor: pointer;
   position: relative;
 }
 .post-icons .copy-icon img,
 .post-icons .evaluate-icon img,
 .post-icons .sources-icon img,
+.post-icons .graph-icon img,
+.post-icons .excerpts-icon img {
   transition: filter var(--transition-speed);
 }
 .post-icons .copy-icon:hover img,
 .post-icons .evaluate-icon:hover img,
 .post-icons .sources-icon:hover img,
+.post-icons .graph-icon:hover img,
+.post-icons .excerpts-icon:hover img {
   filter: brightness(0.65);
 }
 .post-icons .copy-icon:hover .tooltip,
 .post-icons .evaluate-icon:hover .tooltip,
 .post-icons .sources-icon:hover .tooltip,
+.post-icons .graph-icon:hover .tooltip,
+.post-icons .excerpts-icon:hover .tooltip {
   opacity: 1;
   visibility: visible;
   transform: translateX(-50%) translateY(0) scale(1);
   object-fit: contain;
 }
+/* Increase the size of the excerpts icon */
+.post-icons .excerpts-icon img {
+  width: 20px;
+  height: 26px;
+  margin-top: -3.5px;
+  object-fit: fill;
+}
 /* Container for the loading state with a dark background */
 .bot-loading {

frontend/src/Components/AiComponents/ChatWindow.js CHANGED Viewed

@@ -1,11 +1,12 @@
-import React, { useRef, useState, useEffect } from 'react';
 import Box from '@mui/material/Box';
 import Snackbar from '@mui/material/Snackbar';
 import Slide from '@mui/material/Slide';
 import IconButton from '@mui/material/IconButton';
-import { FaTimes } from 'react-icons/fa';
 import GraphDialog from './ChatComponents/Graph';
 import Streaming from './ChatComponents/Streaming';
 import './ChatWindow.css';
 import bot from '../../Icons/bot.png';
@@ -14,6 +15,7 @@ import evaluate from '../../Icons/evaluate.png';
 import sourcesIcon from '../../Icons/sources.png';
 import graphIcon from '../../Icons/graph.png';
 import user from '../../Icons/user.png';
 // SlideTransition function for both entry and exit transitions.
 function SlideTransition(props) {
@@ -28,6 +30,10 @@ function ChatWindow({
   thinkingTime,
   thoughtLabel,
   sourcesRead,
   actions,
   tasks,
   openRightSidebar,
@@ -35,9 +41,12 @@ function ChatWindow({
   isError,
   errorMessage
 }) {
   const answerRef = useRef(null);
   const [graphDialogOpen, setGraphDialogOpen] = useState(false);
   const [snackbarOpen, setSnackbarOpen] = useState(false);
   // Get the graph action from the actions prop.
   const graphAction = actions && actions.find(a => a.name === "graph");
@@ -101,6 +110,69 @@ function ChatWindow({
     answerRef.current = ref;
   };
   return (
     <>
       { !hasTokens ? (
@@ -186,10 +258,12 @@ function ChatWindow({
                 </div>
                 <div className="message-bubble bot-bubble">
                   <div className="answer">
-                    <Streaming
                       content={combinedAnswer}
                       isStreaming={isStreaming}
                       onContentRef={handleContentRef}
                     />
                   </div>
                 </div>
@@ -218,6 +292,24 @@ function ChatWindow({
                       <span className="tooltip">View Graph</span>
                     </div>
                   )}
                 </div>
               </div>
             </div>
@@ -232,6 +324,17 @@ function ChatWindow({
           )}
         </div>
       )}
       {/* Render error container if there's an error */}
       {isError && (
         <div className="error-block" style={{ marginTop: '1rem' }}>

+import React, { useRef, useState, useCallback, useEffect } from 'react';
 import Box from '@mui/material/Box';
 import Snackbar from '@mui/material/Snackbar';
 import Slide from '@mui/material/Slide';
 import IconButton from '@mui/material/IconButton';
+import { FaTimes, FaSpinner, FaCheckCircle } from 'react-icons/fa';
 import GraphDialog from './ChatComponents/Graph';
 import Streaming from './ChatComponents/Streaming';
+import SourcePopup from './ChatComponents/SourcePopup';
 import './ChatWindow.css';
 import bot from '../../Icons/bot.png';
 import sourcesIcon from '../../Icons/sources.png';
 import graphIcon from '../../Icons/graph.png';
 import user from '../../Icons/user.png';
+import excerpts from '../../Icons/excerpts.png';
 // SlideTransition function for both entry and exit transitions.
 function SlideTransition(props) {
   thinkingTime,
   thoughtLabel,
   sourcesRead,
+  finalSources,
+  excerptsData,
+  isLoadingExcerpts,
+  onFetchExcerpts,
   actions,
   tasks,
   openRightSidebar,
   isError,
   errorMessage
 }) {
+  console.log(`[ChatWindow ${blockId}] Received excerptsData:`, excerptsData);
   const answerRef = useRef(null);
   const [graphDialogOpen, setGraphDialogOpen] = useState(false);
   const [snackbarOpen, setSnackbarOpen] = useState(false);
+  const [hoveredSourceInfo, setHoveredSourceInfo] = useState(null);
+  const popupTimeoutRef = useRef(null);
   // Get the graph action from the actions prop.
   const graphAction = actions && actions.find(a => a.name === "graph");
     answerRef.current = ref;
   };
+  // Handle showing the source popup
+  const showSourcePopup = useCallback((sourceIndex, targetElement, statementText) => {
+    // Clear any existing timeout to prevent flickering
+    if (popupTimeoutRef.current) {
+      clearTimeout(popupTimeoutRef.current);
+      popupTimeoutRef.current = null;
+    }
+    if (!finalSources || !finalSources[sourceIndex] || !targetElement) return;
+    const rect = targetElement.getBoundingClientRect();
+    const scrollY = window.scrollY || window.pageYOffset;
+    const scrollX = window.scrollX || window.pageXOffset;
+    const newHoverInfo = {
+      index: sourceIndex,
+      statementText,
+      position: {
+        top: rect.top + scrollY - 10, // Position above the reference
+        left: rect.left + scrollX + rect.width / 2, // Center horizontally
+      }
+    };
+    setHoveredSourceInfo(newHoverInfo);
+  }, [finalSources]);
+  const hideSourcePopup = useCallback(() => {
+    if (popupTimeoutRef.current) {
+        clearTimeout(popupTimeoutRef.current); // Clear existing timeout if mouse leaves quickly
+    }
+    popupTimeoutRef.current = setTimeout(() => {
+      setHoveredSourceInfo(null);
+      popupTimeoutRef.current = null;
+    }, 15); // Delay allows moving mouse onto popup
+  }, []);
+  // Handle mouse enter on the popup to cancel the hide timeout
+  const cancelHidePopup = useCallback(() => {
+    // Clear the hide timeout if the mouse enters the popup itself
+    if (popupTimeoutRef.current) {
+      clearTimeout(popupTimeoutRef.current);
+      popupTimeoutRef.current = null;
+    }
+  }, []);
+  // Determine button state and appearance for excerpts icon
+  const excerptsLoaded = !!excerptsData;  // True if excerptsData is not null/empty
+  const canFetchExcerpts = finalSources && finalSources.length > 0 &&
+                           !isError && !excerptsLoaded && !isLoadingExcerpts;
+  const buttonDisabled = isLoadingExcerpts || excerptsLoaded; // Disable button if loading or loaded
+  const buttonIcon = isLoadingExcerpts
+                      ? <FaSpinner className="spin" style={{ fontSize: 20 }} />
+                      : excerptsLoaded
+                      ? <FaCheckCircle
+                          style={{
+                            width: 22,
+                            height: 22,
+                            color: 'var(--secondary-color)',
+                            filter: 'brightness(0.75)'
+                          }}
+                        />
+                      : <img src={excerpts} alt="excerpts icon" />;
+  const buttonClassName = `excerpts-icon ${isLoadingExcerpts ? 'loading' : ''} ${excerptsLoaded ? 'loaded' : ''}`;
   return (
     <>
       { !hasTokens ? (
                 </div>
                 <div className="message-bubble bot-bubble">
                   <div className="answer">
+                    <Streaming
                       content={combinedAnswer}
                       isStreaming={isStreaming}
                       onContentRef={handleContentRef}
+                      showSourcePopup={showSourcePopup}
+                      hideSourcePopup={hideSourcePopup}
                     />
                   </div>
                 </div>
                       <span className="tooltip">View Graph</span>
                     </div>
                   )}
+                  {/* Show Excerpts Button - Conditionally Rendered */}
+                  {finalSources && finalSources.length > 0 && !isError && (
+                    <div
+                      className={buttonClassName}
+                      onClick={() => canFetchExcerpts && onFetchExcerpts(blockId)}
+                      style={{
+                        cursor: buttonDisabled ? 'default' : 'pointer',
+                        opacity: excerptsLoaded ? 0.6 : 1
+                      }}
+                    >
+                      {buttonIcon}
+                      <span className="tooltip">
+                        {excerptsLoaded ? 'Excerpts Loaded'
+                          : isLoadingExcerpts ? 'Loading Excerpts…'
+                          : 'Show Excerpts'}
+                      </span>
+                    </div>
+                  )}
                 </div>
               </div>
             </div>
           )}
         </div>
       )}
+      {/* Render Source Popup */}
+      {hoveredSourceInfo && finalSources && finalSources[hoveredSourceInfo.index] && (
+        <SourcePopup
+          sourceData={finalSources[hoveredSourceInfo.index]}
+          excerptsData={excerptsData}
+          position={hoveredSourceInfo.position}
+          onMouseEnter={cancelHidePopup} // Keep popup open if mouse enters it
+          onMouseLeave={hideSourcePopup}
+          statementText={hoveredSourceInfo.statementText}
+        />
+      )}
       {/* Render error container if there's an error */}
       {isError && (
         <div className="error-block" style={{ marginTop: '1rem' }}>

frontend/src/Components/AiComponents/Dropdowns/AddContentDropdown.css ADDED Viewed

	@@ -0,0 +1,150 @@

+.add-content-dropdown {
+  position: absolute;
+  bottom: 100%;
+  left: 0;
+  background-color: #21212f;
+  /* border: 0.01rem solid #444; */
+  border-radius: 0.35rem;
+  box-shadow: 0 0.75rem 0.85rem rgba(0, 0, 0, 0.484);
+  z-index: 1010;
+  width: 13.5rem;
+  padding: 0.3rem 0;
+  margin-bottom: 0.75rem;
+  opacity: 0;
+  visibility: hidden;
+  transform: translateY(10px);
+  transition: opacity 0.2s ease, transform 0.2s ease;
+}
+.add-content-dropdown.open {
+  opacity: 1;
+  visibility: visible;
+  transform: translateY(0);
+}
+.add-content-dropdown ul {
+  list-style: none;
+  margin: 0;
+  padding: 0;
+}
+.add-content-dropdown li {
+  display: flex;
+  align-items: center;
+  padding: 0.75rem 1rem;
+  cursor: pointer;
+  color: #e0e0e0;
+  font-size: 1rem;
+  position: relative;
+  transition: background-color 0.2s ease;
+}
+.add-content-dropdown li:hover {
+  background-color: #15151f;
+  border-radius: 1.35rem;
+}
+.add-content-dropdown li.selected:hover {
+  background-color: #4caf5033;
+  border-radius: 1.35rem;
+}
+/* Active state for items with open sub-menus */
+.add-content-dropdown li.has-submenu.active {
+  background-color: #15151f;
+  border-radius: 1.35rem;
+}
+.add-content-dropdown .dropdown-icon {
+  margin-right: 0.75rem;
+  font-size: 1rem;
+  color: #aaabb9;
+}
+.selected {
+  background-color: #4caf501a;
+}
+.selected:hover {
+  background-color: #4caf5033;
+}
+.menu-item-content {
+  display: flex;
+  align-items: center;
+  width: 100%;
+}
+.add-content-dropdown li.has-submenu {
+  justify-content: space-between;
+  user-select: none; /* Prevent text selection on click */
+}
+.add-content-dropdown .submenu-arrow {
+  font-size: 0.8rem;
+  color: #aaabb9;
+  margin-left: auto;
+  flex-shrink: 0;
+  pointer-events: none; /* Prevent arrow from blocking clicks */
+}
+.dropdown-icon {
+  margin-right: 8px;
+}
+.sub-dropdown {
+  position: absolute;
+  left: 100%;
+  /* Default to opening upwards for chat view */
+  bottom: 0;
+  background-color: #21212f;
+  border-radius: 0.35rem;
+  box-shadow: 0 0.75rem 0.85rem rgba(0, 0, 0, 0.484);
+  z-index: 1020; /* Higher than main dropdown */
+  width: 13.5rem;
+  padding: 0.3rem 0;
+  opacity: 0;
+  visibility: hidden;
+  transform: translateX(10px);
+  transition: opacity 0.2s ease, transform 0.2s ease;
+}
+.sub-dropdown.open {
+  opacity: 1;
+  visibility: visible;
+  transform: translateX(0);
+}
+/* Nested sub-dropdown (third level) */
+.sub-dropdown .sub-dropdown {
+  z-index: 1030; /* Higher than second level */
+}
+.sub-dropdown li.has-submenu {
+  justify-content: space-between;
+}
+/* Initial Chat Window */
+.search-bar .add-content-dropdown {
+  top: 100%;
+  bottom: auto;
+  margin-top: 0.6rem;
+  margin-bottom: 0;
+  box-shadow: 0 -0.75rem 1rem rgba(0, 0, 0, 0.484);
+  transform: translateY(-10px);
+}
+.search-bar .sub-dropdown {
+  top: 0;
+  bottom: auto;
+}
+/* Third level sub-dropdown in search bar - open upward */
+.search-bar .sub-dropdown .sub-dropdown {
+  top: auto;
+  bottom: 0;
+}
+.search-bar .add-content-dropdown.open {
+  transform: translateY(0);
+}

frontend/src/Components/AiComponents/Dropdowns/AddContentDropdown.js ADDED Viewed

	@@ -0,0 +1,354 @@

+import { useEffect, useRef, useState } from 'react';
+import {
+  FaPaperclip,
+  FaCubes,
+  FaGoogle,
+  FaMicrosoft,
+  FaSlack,
+  FaChevronRight,
+  FaFileAlt,
+  FaTable,
+  FaDesktop,
+  FaStickyNote,
+  FaTasks,
+  FaCalendarAlt,
+  FaFolderOpen,
+  FaEnvelope,
+  FaFileWord,
+  FaFileExcel,
+  FaFilePowerpoint,
+  FaClipboardList,
+  FaExchangeAlt,
+  FaCloud
+} from 'react-icons/fa';
+import './AddContentDropdown.css';
+function AddContentDropdown({
+  isOpen,
+  onClose,
+  toggleButtonRef,
+  onAddFilesClick,
+  onServiceClick,
+  selectedServices = { google: [], microsoft: [], slack: false }
+}) {
+  const dropdownRef = useRef(null);
+  const [openSubMenus, setOpenSubMenus] = useState({
+    connectApps: false,
+    googleWorkspace: false,
+    microsoft365: false
+  });
+  // Effect to handle clicks outside the dropdown to close it
+  useEffect(() => {
+    const handleClickOutside = (event) => {
+      // Do not close if the click is on the toggle button itself
+      if (toggleButtonRef && toggleButtonRef.current && toggleButtonRef.current.contains(event.target)) {
+        return;
+      }
+      // Close the dropdown if the click is outside of it
+      if (dropdownRef.current && !dropdownRef.current.contains(event.target)) {
+        onClose();
+        // Reset all sub-menus when closing
+        setOpenSubMenus({
+          connectApps: false,
+          googleWorkspace: false,
+          microsoft365: false
+        });
+      }
+    };
+    if (isOpen) {
+      document.addEventListener('mousedown', handleClickOutside);
+    } else {
+      document.removeEventListener('mousedown', handleClickOutside);
+    }
+    return () => {
+      document.removeEventListener('mousedown', handleClickOutside);
+    };
+  }, [isOpen, onClose, toggleButtonRef]);
+  // Reset sub-menus when dropdown closes
+  useEffect(() => {
+    if (!isOpen) {
+      setOpenSubMenus({
+        connectApps: false,
+        googleWorkspace: false,
+        microsoft365: false
+      });
+    }
+  }, [isOpen]);
+  const handleConnectAppsHover = () => {
+    setOpenSubMenus(prev => ({
+      ...prev,
+      connectApps: true
+    }));
+  };
+  const handleGoogleWorkspaceHover = () => {
+    setOpenSubMenus(prev => ({
+      ...prev,
+      googleWorkspace: true,
+      // Close Microsoft 365 when hovering Google Workspace
+      microsoft365: false
+    }));
+  };
+  const handleMicrosoft365Hover = () => {
+    setOpenSubMenus(prev => ({
+      ...prev,
+      microsoft365: true,
+      // Close Google Workspace when hovering Microsoft 365
+      googleWorkspace: false
+    }));
+  };
+  const handleSlackHover = () => {
+    // Close service sub-menus when hovering Slack
+    setOpenSubMenus(prev => ({
+      ...prev,
+      googleWorkspace: false,
+      microsoft365: false
+    }));
+  };
+  const handleAddFilesHover = () => {
+    // Close Connect Apps menu when hovering Add Files
+    setOpenSubMenus({
+      connectApps: false,
+      googleWorkspace: false,
+      microsoft365: false
+    });
+  };
+  // Simplified handlers - just call onServiceClick
+  const handleGoogleServiceClick = (service) => {
+    if (onServiceClick && typeof onServiceClick === 'function') {
+      onServiceClick('google', service);
+    }
+  };
+  const handleMicrosoftServiceClick = (service) => {
+    if (onServiceClick && typeof onServiceClick === 'function') {
+      onServiceClick('microsoft', service);
+    }
+  };
+  const handleSlackClick = () => {
+    if (onServiceClick && typeof onServiceClick === 'function') {
+      onServiceClick('slack', 'slack');
+    }
+  };
+  // Helper to check if a service is selected
+  const isServiceSelected = (provider, service) => {
+    if (provider === 'slack') {
+      return selectedServices.slack || false;
+    }
+    return selectedServices[provider]?.includes(service) || false;
+  };
+  return (
+    <div className={`add-content-dropdown ${isOpen ? 'open' : ''}`} ref={dropdownRef}>
+      <ul>
+        <li onClick={onAddFilesClick} onMouseEnter={handleAddFilesHover}>
+          <div className="menu-item-content">
+            <FaPaperclip className="dropdown-icon" />
+            <span>Add Files and Links</span>
+          </div>
+        </li>
+        <li className={`has-submenu ${openSubMenus.connectApps ? 'active' : ''}`} onMouseEnter={handleConnectAppsHover}>
+          <div className="menu-item-content">
+            <FaCubes className="dropdown-icon" />
+            <span>Connect Apps</span>
+          </div>
+          <FaChevronRight className="submenu-arrow" />
+          <div className={`sub-dropdown ${openSubMenus.connectApps ? 'open' : ''}`}>
+            <ul>
+              <li className={`has-submenu ${openSubMenus.googleWorkspace ? 'active' : ''}`} onMouseEnter={handleGoogleWorkspaceHover}>
+                <div className="menu-item-content">
+                  <FaGoogle className="dropdown-icon" />
+                  <span>Google Workspace</span>
+                </div>
+                <FaChevronRight className="submenu-arrow" />
+                <div className={`sub-dropdown ${openSubMenus.googleWorkspace ? 'open' : ''}`}>
+                  <ul>
+                    <li
+                      onClick={() => handleGoogleServiceClick('docs')}
+                      className={isServiceSelected('google', 'docs') ? 'selected' : ''}
+                    >
+                      <div className="menu-item-content">
+                        <FaFileAlt className="dropdown-icon" />
+                        <span>Docs</span>
+                      </div>
+                    </li>
+                    <li
+                      onClick={() => handleGoogleServiceClick('sheets')}
+                      className={isServiceSelected('google', 'sheets') ? 'selected' : ''}
+                    >
+                      <div className="menu-item-content">
+                        <FaTable className="dropdown-icon" />
+                        <span>Sheets</span>
+                      </div>
+                    </li>
+                    <li
+                      onClick={() => handleGoogleServiceClick('slides')}
+                      className={isServiceSelected('google', 'slides') ? 'selected' : ''}
+                    >
+                      <div className="menu-item-content">
+                        <FaDesktop className="dropdown-icon" />
+                        <span>Slides</span>
+                      </div>
+                    </li>
+                    <li
+                      onClick={() => handleGoogleServiceClick('keep')}
+                      className={isServiceSelected('google', 'keep') ? 'selected' : ''}
+                    >
+                      <div className="menu-item-content">
+                        <FaStickyNote className="dropdown-icon" />
+                        <span>Keep</span>
+                      </div>
+                    </li>
+                    <li
+                      onClick={() => handleGoogleServiceClick('tasks')}
+                      className={isServiceSelected('google', 'tasks') ? 'selected' : ''}
+                    >
+                      <div className="menu-item-content">
+                        <FaTasks className="dropdown-icon" />
+                        <span>Tasks</span>
+                      </div>
+                    </li>
+                    <li
+                      onClick={() => handleGoogleServiceClick('calendar')}
+                      className={isServiceSelected('google', 'calendar') ? 'selected' : ''}
+                    >
+                      <div className="menu-item-content">
+                        <FaCalendarAlt className="dropdown-icon" />
+                        <span>Calendar</span>
+                      </div>
+                    </li>
+                    <li
+                      onClick={() => handleGoogleServiceClick('drive')}
+                      className={isServiceSelected('google', 'drive') ? 'selected' : ''}
+                    >
+                      <div className="menu-item-content">
+                        <FaFolderOpen className="dropdown-icon" />
+                        <span>Drive</span>
+                      </div>
+                    </li>
+                    <li
+                      onClick={() => handleGoogleServiceClick('gmail')}
+                      className={isServiceSelected('google', 'gmail') ? 'selected' : ''}
+                    >
+                      <div className="menu-item-content">
+                        <FaEnvelope className="dropdown-icon" />
+                        <span>Gmail</span>
+                      </div>
+                    </li>
+                  </ul>
+                </div>
+              </li>
+              <li className={`has-submenu ${openSubMenus.microsoft365 ? 'active' : ''}`} onMouseEnter={handleMicrosoft365Hover}>
+                <div className="menu-item-content">
+                  <FaMicrosoft className="dropdown-icon" />
+                  <span>Microsoft 365</span>
+                </div>
+                <FaChevronRight className="submenu-arrow" />
+                <div className={`sub-dropdown ${openSubMenus.microsoft365 ? 'open' : ''}`}>
+                  <ul>
+                    <li
+                      onClick={() => handleMicrosoftServiceClick('word')}
+                      className={isServiceSelected('microsoft', 'word') ? 'selected' : ''}
+                    >
+                      <div className="menu-item-content">
+                        <FaFileWord className="dropdown-icon" />
+                        <span>Word</span>
+                      </div>
+                    </li>
+                    <li
+                      onClick={() => handleMicrosoftServiceClick('excel')}
+                      className={isServiceSelected('microsoft', 'excel') ? 'selected' : ''}
+                    >
+                      <div className="menu-item-content">
+                        <FaFileExcel className="dropdown-icon" />
+                        <span>Excel</span>
+                      </div>
+                    </li>
+                    <li
+                      onClick={() => handleMicrosoftServiceClick('powerpoint')}
+                      className={isServiceSelected('microsoft', 'powerpoint') ? 'selected' : ''}
+                    >
+                      <div className="menu-item-content">
+                        <FaFilePowerpoint className="dropdown-icon" />
+                        <span>PowerPoint</span>
+                      </div>
+                    </li>
+                    <li
+                      onClick={() => handleMicrosoftServiceClick('onenote')}
+                      className={isServiceSelected('microsoft', 'onenote') ? 'selected' : ''}
+                    >
+                      <div className="menu-item-content">
+                        <FaStickyNote className="dropdown-icon" />
+                        <span>OneNote</span>
+                      </div>
+                    </li>
+                    <li
+                      onClick={() => handleMicrosoftServiceClick('todo')}
+                      className={isServiceSelected('microsoft', 'todo') ? 'selected' : ''}
+                    >
+                      <div className="menu-item-content">
+                        <FaClipboardList className="dropdown-icon" />
+                        <span>To Do</span>
+                      </div>
+                    </li>
+                    <li
+                      onClick={() => handleMicrosoftServiceClick('exchange')}
+                      className={isServiceSelected('microsoft', 'exchange') ? 'selected' : ''}
+                    >
+                      <div className="menu-item-content">
+                        <FaExchangeAlt className="dropdown-icon" />
+                        <span>Exchange</span>
+                      </div>
+                    </li>
+                    <li
+                      onClick={() => handleMicrosoftServiceClick('onedrive')}
+                      className={isServiceSelected('microsoft', 'onedrive') ? 'selected' : ''}
+                    >
+                      <div className="menu-item-content">
+                        <FaCloud className="dropdown-icon" />
+                        <span>OneDrive</span>
+                      </div>
+                    </li>
+                    <li
+                      onClick={() => handleMicrosoftServiceClick('outlook')}
+                      className={isServiceSelected('microsoft', 'outlook') ? 'selected' : ''}
+                    >
+                      <div className="menu-item-content">
+                        <FaEnvelope className="dropdown-icon" />
+                        <span>Outlook</span>
+                      </div>
+                    </li>
+                  </ul>
+                </div>
+              </li>
+              <li
+                onMouseEnter={handleSlackHover}
+                onClick={handleSlackClick}
+                className={isServiceSelected('slack', 'slack') ? 'selected' : ''}
+              >
+                <div className="menu-item-content">
+                  <FaSlack className="dropdown-icon" />
+                  <span>Slack</span>
+                </div>
+              </li>
+            </ul>
+          </div>
+        </li>
+      </ul>
+    </div>
+  );
+}
+export default AddContentDropdown;

frontend/src/Components/AiComponents/Dropdowns/AddFilesDialog.css ADDED Viewed

	@@ -0,0 +1,191 @@

+.add-files-dialog {
+  position: fixed;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100vh;
+  background-color: rgba(0, 0, 0, 0.2);
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  z-index: 1000;
+  overflow: hidden;
+}
+.add-files-dialog-inner {
+  position: relative;
+  border-radius: 12px;
+  padding: 32px;
+  width: 45%;
+  max-width: 100%;
+  background-color: #1e1e1e;
+  max-height: 80vh;
+  overflow-y: auto;
+  padding-top: 4.5rem;
+}
+.add-files-dialog-inner .dialog-title {
+  position: absolute;
+  font-weight: bold;
+  font-size: 1.5rem;
+  top: 16px;
+  left: 16px;
+  color: #e0e0e0;
+}
+.add-files-dialog-inner .close-btn {
+  position: absolute;
+  top: 16px;
+  right: 16px;
+  background: none;
+  color: white;
+  padding: 7px;
+  border-radius: 5px;
+  cursor: pointer;
+  border: none;
+}
+.add-files-dialog-inner .close-btn:hover {
+  background: rgba(255, 255, 255, 0.1);
+  color: white;
+}
+.dialog-content-area {
+  color: #e0e0e0;
+}
+.url-input-container {
+  margin-bottom: 1.5rem;
+}
+.url-input-label {
+  display: block;
+  margin-bottom: 0.5rem;
+  font-size: 0.9rem;
+  font-weight: 500;
+}
+.url-input-textarea {
+  width: 100%;
+  min-height: 80px;
+  background: #1E1E1E;
+  color: #DDD;
+  border: 1px solid #444;
+  padding: 10px;
+  border-radius: 5px;
+  font-size: 16px;
+  resize: vertical;
+  transition: border 0.3s ease, background 0.3s ease;
+}
+.file-drop-zone {
+  margin-top: 1rem;
+  border: 2px dashed #555;
+  border-radius: 8px;
+  padding: 2rem;
+  text-align: center;
+  cursor: pointer;
+  transition: border-color 0.2s ease, background-color 0.2s ease;
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  justify-content: center;
+  color: #aaa;
+}
+.file-drop-zone:hover {
+  border-color: #777;
+  background-color: #2a2a2a;
+}
+.file-drop-zone.dragging {
+  border-color: #26a8dc;
+  background-color: rgba(38, 168, 220, 0.1);
+}
+.file-drop-zone .upload-icon {
+  font-size: 3rem;
+  margin-bottom: 1rem;
+  color: #666;
+}
+.file-drop-zone p {
+  margin: 0;
+  font-size: 1rem;
+}
+.file-list {
+  margin-top: 1.5rem;
+  max-height: 250px;
+  overflow-y: auto;
+  padding-right: 0.5rem; /* Space for scrollbar */
+}
+.file-item {
+  display: flex;
+  align-items: center;
+  background-color: #2a2a2a;
+  padding: 0.75rem;
+  border-radius: 6px;
+  margin-bottom: 0.5rem;
+}
+.file-icon {
+  color: #aaa;
+  font-size: 1.5rem;
+  margin-right: 1rem;
+}
+.file-info {
+  flex-grow: 1;
+  display: flex;
+  flex-direction: column;
+  overflow: hidden;
+}
+.file-name {
+  white-space: nowrap;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  font-size: 0.9rem;
+}
+.file-size {
+  font-size: 0.75rem;
+  color: #888;
+}
+.progress-bar-container {
+  width: 100px;
+  height: 8px;
+  background-color: #444;
+  border-radius: 4px;
+  margin: 0 1rem;
+}
+.progress-bar {
+  height: 100%;
+  background-color: #26a8dc;
+  border-radius: 4px;
+  transition: width 0.3s ease;
+}
+.cancel-file-btn {
+  background: none;
+  border: none;
+  color: #aaa;
+  cursor: pointer;
+  font-size: 1rem;
+  padding: 0.25rem;
+}
+.cancel-file-btn:hover {
+  color: #fff;
+}
+.dialog-actions {
+  margin-top: 1.5rem;
+  display: flex;
+  justify-content: flex-end;
+  gap: 0.75rem;
+}

frontend/src/Components/AiComponents/Dropdowns/AddFilesDialog.js ADDED Viewed

	@@ -0,0 +1,282 @@

+import React, { useState, useRef, useCallback } from 'react';
+import { FaTimes, FaFileUpload, FaFileAlt } from 'react-icons/fa';
+import Button from '@mui/material/Button';
+import './AddFilesDialog.css';
+const MAX_TOTAL_SIZE = 10 * 1024 * 1024; // 10 MB
+const ALLOWED_EXTENSIONS = new Set([
+  // Documents
+  '.pdf', '.doc', '.docx', '.odt', '.txt', '.rtf', '.md',
+  // Spreadsheets
+  '.csv', '.xls', '.xlsx',
+  // Presentations
+  '.ppt', '.pptx',
+  // Code files
+  '.js', '.jsx', '.ts', '.tsx', '.py', '.java', '.c', '.cpp', '.h',
+  '.cs', '.html', '.css', '.scss', '.json', '.xml', '.sql', '.sh',
+  '.rb', '.php', '.go'
+]);
+function AddFilesDialog({ isOpen, onClose, openSnackbar, setSessionContent }) {
+  const [isUploading, setIsUploading] = useState(false);
+  const [isDragging, setIsDragging] = useState(false);
+  const [files, setFiles] = useState([]);
+  const [urlInput, setUrlInput] = useState("");
+  const fileInputRef = useRef(null);
+  // Function to handle files dropped or selected
+  const handleFiles = useCallback((incomingFiles) => {
+    if (incomingFiles && incomingFiles.length > 0) {
+      let currentTotalSize = files.reduce((acc, f) => acc + f.file.size, 0);
+      const validFiles = [];
+      for (const file of Array.from(incomingFiles)) {
+        // 1. Check for duplicates
+        if (files.some(existing => existing.file.name === file.name && existing.file.size === file.size)) {
+          continue; // Skip duplicate file
+        }
+        // 2. Check file type
+        const fileExtension = file.name.slice(file.name.lastIndexOf('.')).toLowerCase();
+        if (!ALLOWED_EXTENSIONS.has(fileExtension)) {
+          openSnackbar(`File type not supported: ${file.name}`, 'error', 5000);
+          continue; // Skip unsupported file type
+        }
+        // 3. Check total size limit
+        if (currentTotalSize + file.size > MAX_TOTAL_SIZE) {
+          openSnackbar('Total file size cannot exceed 10 MB', 'error', 5000);
+          break; // Stop processing further files as limit is reached
+        }
+        currentTotalSize += file.size;
+        validFiles.push({
+          id: window.crypto.randomUUID(),
+          file: file,
+          progress: 0,
+        });
+      }
+      if (validFiles.length > 0) {
+        setFiles(prevFiles => [...prevFiles, ...validFiles]);
+      }
+    }
+  }, [files, openSnackbar]);
+  // Function to handle file removal
+  const handleRemoveFile = useCallback((fileId) => {
+    setFiles(prevFiles => prevFiles.filter(f => f.id !== fileId));
+  }, []);
+  // Ensure that the component does not render if isOpen is false
+  if (!isOpen) {
+    return null;
+  }
+  // Function to format file size in a human-readable format
+  const formatFileSize = (bytes) => {
+    if (bytes === 0) return '0 Bytes';
+    const k = 1024;
+    const sizes = ['Bytes', 'KB', 'MB', 'GB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+  };
+  // Handlers for drag and drop events
+  const handleDragOver = (e) => {
+    e.preventDefault();
+    e.stopPropagation();
+    setIsDragging(true);
+  };
+  // Handler for when the drag leaves the drop zone
+  const handleDragLeave = (e) => {
+    e.preventDefault();
+    e.stopPropagation();
+    setIsDragging(false);
+  };
+  // Handler for when files are dropped into the drop zone
+  const handleDrop = (e) => {
+    e.preventDefault();
+    e.stopPropagation();
+    setIsDragging(false);
+    handleFiles(e.dataTransfer.files);
+  };
+  // Handler for when files are selected via the file input
+  const handleFileSelect = (e) => {
+    handleFiles(e.target.files);
+    // Reset input value to allow selecting the same file again
+    e.target.value = null;
+  };
+  // Handler for clicking the drop zone to open the file dialog
+  const handleBoxClick = () => {
+    fileInputRef.current.click();
+  };
+  // Handler for resetting the file list
+  const handleReset = () => {
+    setFiles([]);
+    setUrlInput("");
+  };
+  // Handler for adding files
+  const handleAdd = () => {
+    setIsUploading(true); // Start upload state, disable buttons
+    // Regex to validate URL format
+    const urlRegex = /^(https?:\/\/)?([\w-]+\.)+[\w-]+(\/[\w-./?%&=]*)?$/;
+    const urls = urlInput.split('\n').map(url => url.trim()).filter(url => url);
+    // 1. Validate URLs before proceeding
+    if (files.length === 0 && urls.length === 0) {
+      openSnackbar("Please add files or URLs before submitting.", "error", 5000);
+      return;
+    }
+    for (const url of urls) {
+      if (!urlRegex.test(url)) {
+        openSnackbar(`Invalid URL format: ${url}`, 'error', 5000);
+        setIsUploading(false); // Reset upload state on validation error
+        return; // Stop the process if an invalid URL is found
+      }
+    }
+    // 2. If all URLs are valid, proceed with logging/uploading
+    const formData = new FormData();
+    if (files.length > 0) {
+      files.forEach(fileWrapper => {
+        formData.append('files', fileWrapper.file, fileWrapper.file.name);
+      });
+    }
+    formData.append('urls', JSON.stringify(urls));
+    const xhr = new XMLHttpRequest();
+    xhr.open('POST', '/add-content', true);
+    // Track upload progress
+    xhr.upload.onprogress = (event) => {
+      if (event.lengthComputable) {
+        const percentage = Math.round((event.loaded / event.total) * 100);
+        setFiles(prevFiles =>
+          prevFiles.map(f => ({ ...f, progress: percentage }))
+        );
+      }
+    };
+    // Handle completion
+    xhr.onload = () => {
+      if (xhr.status === 200) {
+        // --- ARTIFICIAL DELAY FOR LOCAL DEVELOPMENT ---
+        // This timeout ensures the 100% progress bar is visible before the dialog closes.
+        // This can be removed for production.
+        setTimeout(() => {
+          const result = JSON.parse(xhr.responseText);
+          openSnackbar('Content added successfully!', 'success');
+          setSessionContent(prev => ({
+            files: [...prev.files, ...result.files_added],
+            links: [...prev.links, ...result.links_added],
+          }));
+          handleReset();
+          onClose();
+        }, 500); // 0.5-second delay
+      } else {
+        const errorResult = JSON.parse(xhr.responseText);
+        openSnackbar(errorResult.detail || 'Failed to add content.', 'error', 5000);
+        setFiles(prevFiles => prevFiles.map(f => ({ ...f, progress: 0 }))); // Reset progress on error
+        setIsUploading(false); // End upload state
+      }
+    };
+    // Handle network errors
+    xhr.onerror = () => {
+      openSnackbar('An error occurred during the upload. Please check your network.', 'error', 5000);
+      setFiles(prevFiles => prevFiles.map(f => ({ ...f, progress: 0 }))); // Reset progress on error
+    };
+    xhr.send(formData);
+  };
+  return (
+    <div className="add-files-dialog" onClick={isUploading ? null : onClose}>
+      <div className="add-files-dialog-inner" onClick={(e) => e.stopPropagation()}>
+        <label className="dialog-title">Add Files and Links</label>
+        <button className="close-btn" onClick={onClose} disabled={isUploading}>
+          <FaTimes />
+        </button>
+        <div className="dialog-content-area">
+          <div className="url-input-container">
+            <textarea
+              id="url-input"
+              className="url-input-textarea"
+              placeholder="Enter one URL per line"
+              value={urlInput}
+              onChange={(e) => setUrlInput(e.target.value)}
+            />
+          </div>
+          <div
+            className={`file-drop-zone ${isDragging ? 'dragging' : ''}`}
+            onClick={handleBoxClick}
+            onDragOver={handleDragOver}
+            onDragLeave={handleDragLeave}
+            onDrop={handleDrop}
+          >
+            <input
+              type="file"
+              ref={fileInputRef}
+              onChange={handleFileSelect}
+              style={{ display: 'none' }}
+              multiple
+            />
+            <FaFileUpload className="upload-icon" />
+            <p>Drag and drop files here, or click to select files</p>
+          </div>
+          {files.length > 0 && (
+            <div className="file-list">
+              {files.map(fileWrapper => (
+                <div key={fileWrapper.id} className="file-item">
+                  <FaFileAlt className="file-icon" />
+                  <div className="file-info">
+                    <span className="file-name">{fileWrapper.file.name}</span>
+                    <span className="file-size">{formatFileSize(fileWrapper.file.size)}</span>
+                  </div>
+                  {isUploading && (
+                    <div className="progress-bar-container">
+                      <div className="progress-bar" style={{ width: `${fileWrapper.progress}%` }}></div>
+                    </div>
+                  )}
+                  <button className="cancel-file-btn" onClick={() => handleRemoveFile(fileWrapper.id)} disabled={isUploading}>
+                    <FaTimes />
+                  </button>
+                </div>
+              ))}
+            </div>
+          )}
+          <div className="dialog-actions">
+            <Button
+              disabled={isUploading}
+              onClick={handleReset}
+              sx={{ color: "#2196f3" }}
+            >
+              Reset
+            </Button>
+            <Button
+              disabled={isUploading}
+              onClick={handleAdd}
+              variant="contained"
+              color="success"
+            >
+              Add
+            </Button>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+}
+export default AddFilesDialog;

frontend/src/Components/AiComponents/Markdown/CustomMarkdown.js ADDED Viewed

	@@ -0,0 +1,489 @@

+import React, { useEffect, useState, useCallback, useMemo } from 'react';
+import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';
+import { atomDark } from 'react-syntax-highlighter/dist/esm/styles/prism';
+import '../ChatComponents/Streaming.css';
+import '../ChatComponents/SourceRef.css';
+// Complete custom markdown parser and renderer that respects Streaming.css
+const CustomMarkdown = ({ content, isStreaming, showSourcePopup, hideSourcePopup }) => {
+  const [parsedContent, setParsedContent] = useState([]);
+  // Display content with cursor if streaming
+  const displayContent = isStreaming ? `${content}▌` : (content || '');
+  // Normalize citations like [1,2] to [1][2]
+  const normalizeCitations = useCallback((text) => {
+    if (!text) return '';
+    const citationRegex = /\[(\d+(?:,\s*\d+)+)\]/g;
+    return text.replace(citationRegex, (match, capturedNumbers) => {
+      const numbers = capturedNumbers
+        .split(/,\s*/)
+        .map(numStr => numStr.trim())
+        .filter(Boolean);
+      if (numbers.length <= 1) return match;
+      return numbers.map(num => `[${num}]`).join('');
+    });
+  }, []);
+  const normalizedContent = useMemo(() => normalizeCitations(displayContent), [displayContent, normalizeCitations]);
+  // Citation component
+  const Citation = ({ number, showSourcePopup, hideSourcePopup, text }) => {
+    const getSentenceForCitation = () => {
+      const sentenceRegex = /[^.!?\n]+[.!?]+[\])'"`'"]*|[^.!?\n]+$/g;
+      const sentences = text.match(sentenceRegex) || [text];
+      for (const sentence of sentences) {
+        if (sentence.includes(`[${number}]`)) {
+          return sentence.trim();
+        }
+      }
+      return '';
+    };
+    return (
+      <sup
+        className="source-reference"
+        onMouseEnter={(e) => {
+          if (showSourcePopup) {
+            const sentence = getSentenceForCitation();
+            showSourcePopup(number - 1, e.target, sentence);
+          }
+        }}
+        onMouseLeave={hideSourcePopup}
+      >
+        {number}
+      </sup>
+    );
+  };
+  // Parse inline markdown elements
+  const parseInline = useCallback((text) => {
+    if (!text) return null;
+    // Define regex patterns clearly
+    const TWO_SPACES = '  '; // Exactly 2 spaces (not 1, not 3, but 2!)
+    const lineBreakRegex = new RegExp(TWO_SPACES + '\\n', 'g');
+    const elements = [];
+    const patterns = [
+      // Bold + Italic
+      { regex: /\*\*\*(.+?)\*\*\*/g, handler: (m) => <strong key={m.index}><em>{parseInline(m[1])}</em></strong> },
+      { regex: /___(.+?)___/g, handler: (m) => <strong key={m.index}><em>{parseInline(m[1])}</em></strong> },
+      // Bold
+      { regex: /\*\*(.+?)\*\*/g, handler: (m) => <strong key={m.index}>{parseInline(m[1])}</strong> },
+      { regex: /__(.+?)__/g, handler: (m) => <strong key={m.index}>{parseInline(m[1])}</strong> },
+      // Italic
+      { regex: /\*([^*]+)\*/g, handler: (m) => <em key={m.index}>{parseInline(m[1])}</em> },
+      { regex: /_([^_]+)_/g, handler: (m) => <em key={m.index}>{parseInline(m[1])}</em> },
+      // Strikethrough
+      { regex: /~~(.+?)~~/g, handler: (m) => <del key={m.index}>{parseInline(m[1])}</del> },
+      // Inline code (preserve all spaces)
+      { regex: /`([^`]+)`/g, handler: (m) => {
+        // Preserve all whitespace in inline code
+        const codeContent = m[1].replace(/ /g, '\u00A0'); // Replace spaces with non-breaking spaces
+        return <code key={m.index}>{codeContent}</code>;
+      }},
+      // Images
+      { regex: /!\[([^\]]*)\]\(([^)]+)\)/g, handler: (m) => <img key={m.index} src={m[2]} alt={m[1]} style={{ maxWidth: '100%' }} /> },
+      // Links
+      { regex: /\[([^\]]+)\]\(([^)]+)\)/g, handler: (m) => (
+        <a key={m.index} href={m[2]} target="_blank" rel="noopener noreferrer" className="markdown-link">
+          {parseInline(m[1])}
+        </a>
+      )},
+      // Citations
+      { regex: /\[(\d+)\]/g, handler: (m) => (
+        <Citation
+          key={m.index}
+          number={parseInt(m[1], 10)}
+          showSourcePopup={showSourcePopup}
+          hideSourcePopup={hideSourcePopup}
+          text={text}
+        />
+      )},
+      // Line breaks
+      {
+        regex: lineBreakRegex,
+        handler: (m) => <br key={m.index} />
+      },
+    ];
+    // Apply patterns in order
+    let processedText = text;
+    const replacements = [];
+    for (const pattern of patterns) {
+      let match;
+      pattern.regex.lastIndex = 0;
+      while ((match = pattern.regex.exec(text))) {
+        replacements.push({
+          start: match.index,
+          end: match.index + match[0].length,
+          element: pattern.handler(match),
+          priority: patterns.indexOf(pattern)
+        });
+      }
+    }
+    // Sort replacements by position and priority
+    replacements.sort((a, b) => {
+      if (a.start !== b.start) return a.start - b.start;
+      return a.priority - b.priority;
+    });
+    // Build result without overlapping replacements
+    let lastEnd = 0;
+    const used = new Set();
+    for (const replacement of replacements) {
+      // Skip if this overlaps with an already used replacement
+      let overlaps = false;
+      for (const usedRange of used) {
+        if (!(replacement.end <= usedRange.start || replacement.start >= usedRange.end)) {
+          overlaps = true;
+          break;
+        }
+      }
+      if (!overlaps) {
+        if (replacement.start > lastEnd) {
+          // Preserve spaces in text segments
+          const textSegment = processedText.substring(lastEnd, replacement.start);
+          elements.push(textSegment);
+        }
+        elements.push(replacement.element);
+        lastEnd = replacement.end;
+        used.add({ start: replacement.start, end: replacement.end });
+      }
+    }
+    if (lastEnd < processedText.length) {
+      // Preserve spaces in remaining text
+      elements.push(processedText.substring(lastEnd));
+    }
+    return elements.length > 0 ? elements : text;
+  }, [showSourcePopup, hideSourcePopup]);
+  // Parse code blocks separately to handle them properly
+  const extractCodeBlocks = useCallback((text) => {
+    const codeBlocks = [];
+    const placeholder = '___CODE_BLOCK_';
+    let counter = 0;
+    // Replace code blocks with placeholders, preserve exact formatting
+    const textWithoutCode = text.replace(/```(\w*)\n([\s\S]*?)```/g, (match, lang, code) => {
+      const id = `${placeholder}${counter}___`;
+      // Preserve exact code content without any trimming
+      codeBlocks.push({ id, lang: lang || 'text', code: code.replace(/\n$/, '') }); // Only remove final newline
+      counter++;
+      return `\n${id}\n`;
+    });
+    return { textWithoutCode, codeBlocks };
+  }, []);
+  // Parse block-level elements
+  const parseBlocks = useCallback((text, codeBlocks = []) => {
+    if (!text) return [];
+    const blocks = [];
+    const lines = text.split('\n');
+    let i = 0;
+    while (i < lines.length) {
+      const line = lines[i];
+      const trimmedLine = line.trim();
+      // Skip empty lines
+      if (!trimmedLine) {
+        i++;
+        continue;
+      }
+      // Check for code block placeholder
+      const codeBlockMatch = line.match(/___CODE_BLOCK_(\d+)___/);
+      if (codeBlockMatch) {
+        const codeBlock = codeBlocks.find(cb => cb.id === line.trim());
+        if (codeBlock) {
+          blocks.push({
+            type: 'code',
+            lang: codeBlock.lang,
+            content: codeBlock.code
+          });
+          i++;
+          continue;
+        }
+      }
+      // Horizontal rule
+      if (/^[-*_]{3,}$/.test(trimmedLine)) {
+        blocks.push({ type: 'hr' });
+        i++;
+        continue;
+      }
+      // Headers
+      const headerMatch = line.match(/^(#{1,6})\s+(.+)$/);
+      if (headerMatch) {
+        blocks.push({
+          type: 'header',
+          level: headerMatch[1].length,
+          content: headerMatch[2]
+        });
+        i++;
+        continue;
+      }
+      // Blockquotes
+      if (line.startsWith('>')) {
+        const quoteLines = [line.substring(1).trim()];
+        i++;
+        while (i < lines.length && (lines[i].startsWith('>') || lines[i].trim() === '')) {
+          if (lines[i].startsWith('>')) {
+            quoteLines.push(lines[i].substring(1).trim());
+          } else if (lines[i].trim() === '' && i + 1 < lines.length && lines[i + 1].startsWith('>')) {
+            quoteLines.push('');
+          } else {
+            break;
+          }
+          i++;
+        }
+        blocks.push({
+          type: 'blockquote',
+          content: quoteLines.join('\n')
+        });
+        continue;
+      }
+      // Lists (unordered and ordered)
+      const unorderedMatch = line.match(/^([-*+])\s+(.+)$/);
+      const orderedMatch = line.match(/^(\d+)\.\s+(.+)$/);
+      if (unorderedMatch || orderedMatch) {
+        const isOrdered = !!orderedMatch;
+        const items = [];
+        const listIndent = line.search(/\S/);
+        while (i < lines.length) {
+          const currentLine = lines[i];
+          const currentIndent = currentLine.search(/\S/);
+          if (currentIndent === -1) {
+            // Empty line, check if list continues
+            if (i + 1 < lines.length) {
+              const nextIndent = lines[i + 1].search(/\S/);
+              if (nextIndent >= listIndent && (lines[i + 1].match(/^[\s]*[-*+]\s+/) || lines[i + 1].match(/^[\s]*\d+\.\s+/))) {
+                i++;
+                continue;
+              }
+            }
+            break;
+          }
+          const itemMatch = isOrdered
+            ? currentLine.match(/^(\s*)\d+\.\s+(.+)$/)
+            : currentLine.match(/^(\s*)[-*+]\s+(.+)$/);
+          if (itemMatch && currentIndent === listIndent) {
+            items.push({
+              content: itemMatch[2],
+              indent: 0
+            });
+            i++;
+          } else if (currentIndent > listIndent) {
+            // Continuation of previous item or nested list
+            if (items.length > 0) {
+              items[items.length - 1].content += '\n' + currentLine;
+            }
+            i++;
+          } else {
+            break;
+          }
+        }
+        blocks.push({
+          type: isOrdered ? 'ol' : 'ul',
+          items: items.map(item => ({
+            ...item,
+            content: item.content.trim()
+          }))
+        });
+        continue;
+      }
+      // Tables
+      if (i + 1 < lines.length && lines[i + 1].trim().match(/^[-:|]+$/)) {
+        const headerCells = line.split('|').map(cell => cell.trim()).filter(Boolean);
+        const alignmentLine = lines[i + 1];
+        const alignments = alignmentLine.split('|').map(cell => {
+          const trimmed = cell.trim();
+          if (trimmed.startsWith(':') && trimmed.endsWith(':')) return 'center';
+          if (trimmed.endsWith(':')) return 'right';
+          return 'left';
+        }).filter((_, index) => index < headerCells.length);
+        const rows = [];
+        i += 2;
+        while (i < lines.length && lines[i].includes('|')) {
+          const cells = lines[i].split('|').map(cell => cell.trim()).filter(Boolean);
+          if (cells.length > 0) {
+            rows.push(cells);
+          }
+          i++;
+        }
+        blocks.push({
+          type: 'table',
+          headers: headerCells,
+          alignments,
+          rows
+        });
+        continue;
+      }
+      // Paragraph
+      const paragraphLines = [line];
+      i++;
+      while (i < lines.length && lines[i].trim() !== '' &&
+             !lines[i].match(/^#{1,6}\s/) &&
+             !lines[i].match(/^[-*+]\s/) &&
+             !lines[i].match(/^\d+\.\s/) &&
+             !lines[i].startsWith('>') &&
+             !lines[i].match(/^[-*_]{3,}$/) &&
+             !lines[i].match(/___CODE_BLOCK_\d+___/)) {
+        paragraphLines.push(lines[i]);
+        i++;
+      }
+      blocks.push({
+        type: 'paragraph',
+        content: paragraphLines.join('\n')
+      });
+    }
+    return blocks;
+  }, []);
+  // Render a single block
+  const renderBlock = useCallback((block, index) => {
+    switch (block.type) {
+      case 'header':
+        const HeaderTag = `h${block.level}`;
+        return <HeaderTag key={index}>{parseInline(block.content)}</HeaderTag>;
+      case 'paragraph':
+        return <p key={index}>{parseInline(block.content)}</p>;
+      case 'blockquote':
+        const { textWithoutCode, codeBlocks } = extractCodeBlocks(block.content);
+        const quotedBlocks = parseBlocks(textWithoutCode, codeBlocks);
+        return (
+          <blockquote key={index} className="markdown-blockquote">
+            {quotedBlocks.map((b, i) => renderBlock(b, i))}
+          </blockquote>
+        );
+      case 'code':
+        return (
+          <div key={index} className="code-block-container">
+            <div className="code-block-header">
+              <span>{block.lang}</span>
+            </div>
+            <SyntaxHighlighter
+              style={atomDark}
+              language={block.lang}
+              PreTag="div"
+              customStyle={{ margin: 0 }}
+            >
+              {block.content}
+            </SyntaxHighlighter>
+          </div>
+        );
+      case 'ul':
+      case 'ol':
+        const ListTag = block.type === 'ol' ? 'ol' : 'ul';
+        return (
+          <ListTag key={index}>
+            {block.items.map((item, i) => {
+              // Handle nested content properly
+              if (item.content.includes('\n')) {
+                // For multi-line items, parse as nested markdown
+                const { textWithoutCode, codeBlocks } = extractCodeBlocks(item.content);
+                const nestedBlocks = parseBlocks(textWithoutCode, codeBlocks);
+                return (
+                  <li key={i}>
+                    {nestedBlocks.map((b, j) => renderBlock(b, `${i}-${j}`))}
+                  </li>
+                );
+              }
+              // For single-line items, just parse inline
+              return <li key={i}>{parseInline(item.content)}</li>;
+            })}
+          </ListTag>
+        );
+      case 'table':
+        return (
+          <div key={index} className="table-container">
+            <table>
+              <thead>
+                <tr>
+                  {block.headers.map((header, i) => (
+                    <th key={i} style={{ textAlign: block.alignments[i] || 'left' }}>
+                      {parseInline(header)}
+                    </th>
+                  ))}
+                </tr>
+              </thead>
+              <tbody>
+                {block.rows.map((row, rowIndex) => (
+                  <tr key={rowIndex}>
+                    {row.map((cell, cellIndex) => (
+                      <td key={cellIndex} style={{ textAlign: block.alignments[cellIndex] || 'left' }}>
+                        {parseInline(cell)}
+                      </td>
+                    ))}
+                  </tr>
+                ))}
+              </tbody>
+            </table>
+          </div>
+        );
+      case 'hr':
+        return <hr key={index} />;
+      default:
+        return null;
+    }
+  }, [parseInline, extractCodeBlocks, parseBlocks]);
+  // Main parse function
+  const parseMarkdown = useCallback((text) => {
+    if (!text) return [];
+    // Extract code blocks first
+    const { textWithoutCode, codeBlocks } = extractCodeBlocks(text);
+    // Parse blocks
+    const blocks = parseBlocks(textWithoutCode, codeBlocks);
+    // Render blocks
+    return blocks.map((block, index) => renderBlock(block, index));
+  }, [extractCodeBlocks, parseBlocks, renderBlock]);
+  // Parse markdown content whenever it changes
+  useEffect(() => {
+    const parsed = parseMarkdown(normalizedContent);
+    setParsedContent(parsed);
+  }, [normalizedContent, parseMarkdown]);
+  // Return just the parsed content
+  return <>{parsedContent}</>;
+};
+export default CustomMarkdown;

frontend/src/Components/AiComponents/Markdown/TestMarkdown.js ADDED Viewed

	@@ -0,0 +1,120 @@

+import CustomMarkdown from './CustomMarkdown';
+const SpacePreservationTest = () => {
+  const testContent = `# Space Preservation Test
+## Inline Code Spacing
+Here are examples with multiple spaces:
+- One space: \`a b\`
+- Two spaces: \`a  b\`
+- Three spaces: \`a   b\`
+- Four spaces: \`a    b\`
+- Tab character: \`a	b\`
+- Mixed: \`function  (  x,   y   )\`
+## Code Block Indentation
+\`\`\`python
+def example():
+    # 4 spaces indentation
+    if True:
+        # 8 spaces indentation
+        print("Hello")
+    # Empty line above preserved
+    for i in range(5):
+        # Aligned comments
+        print(i)  # End of line comment
+\`\`\`
+## ASCII Art Test
+\`\`\`
+  _____
+ / ___ \\
+| |   | |
+| |___| |
+ \\_____/
+    Spacing matters!
+\`\`\`
+## Table Alignment
+\`\`\`
+Name      Age    City
+----      ---    ----
+Alice     25     NYC
+Bob       30     LA
+Charlie   35     Chicago
+\`\`\`
+## Inline Examples
+The function \`map(  x => x * 2  )\` has spaces around the arrow.
+Configuration: \`{ indent:  4,  tabs:  false }\``;
+  return (
+    <div style={{ maxWidth: '800px', margin: '0 auto', padding: '20px' }}>
+      <h1>Space Preservation Test</h1>
+      <div style={{
+        marginBottom: '20px',
+        padding: '15px',
+        background: '#e3f2fd',
+        borderRadius: '8px'
+      }}>
+        <p><strong>What to check:</strong></p>
+        <ul>
+          <li>Inline code should preserve exact spacing</li>
+          <li>Code blocks should maintain indentation</li>
+          <li>ASCII art should be properly aligned</li>
+          <li>Empty lines in code blocks should be preserved</li>
+        </ul>
+      </div>
+      <div style={{
+        border: '2px solid #333',
+        borderRadius: '8px',
+        background: 'white',
+        padding: '20px'
+      }}>
+        <CustomMarkdown
+          content={testContent}
+          isStreaming={false}
+          showSourcePopup={() => {}}
+          hideSourcePopup={() => {}}
+        />
+      </div>
+      <div style={{
+        marginTop: '20px',
+        padding: '15px',
+        background: '#f5f5f5',
+        borderRadius: '8px',
+        fontFamily: 'monospace',
+        fontSize: '14px'
+      }}>
+        <p><strong>Debug: Raw content preview</strong></p>
+        <pre style={{
+          background: '#333',
+          color: '#fff',
+          padding: '10px',
+          borderRadius: '4px',
+          overflow: 'auto',
+          whiteSpace: 'pre'
+        }}>
+{`'a b'    = one space
+'a  b'   = two spaces
+'a   b'  = three spaces
+'a    b' = four spaces`}
+        </pre>
+      </div>
+    </div>
+  );
+};
+export default SpacePreservationTest;

frontend/src/Components/AiComponents/Notifications/Notification.css ADDED Viewed

	@@ -0,0 +1,379 @@

+.notification-container {
+  position: fixed;
+  z-index: 9999;
+  display: flex;
+  flex-direction: column;
+  gap: var(--spacing, 10px);
+  pointer-events: none;
+}
+.notification-list {
+  display: flex;
+  flex-direction: column;
+  gap: var(--spacing, 10px);
+}
+.notification-list.collapsed {
+  gap: 0;
+}
+.notification-list.collapsed .notification:not(:first-child) {
+  margin-top: -80%;
+  opacity: 0.3;
+  transform: scale(0.95);
+}
+/* Position variations */
+.position-top-left {
+  top: var(--offset-y);
+  left: var(--offset-x);
+  align-items: flex-start;
+}
+.position-top-center {
+  top: var(--offset-y);
+  left: 50%;
+  transform: translateX(-50%);
+  align-items: center;
+}
+.position-top-right {
+  top: var(--offset-y);
+  right: var(--offset-x);
+  align-items: flex-end;
+}
+.position-bottom-left {
+  bottom: var(--offset-y);
+  left: var(--offset-x);
+  align-items: flex-start;
+}
+.position-bottom-center {
+  bottom: var(--offset-y);
+  left: 50%;
+  transform: translateX(-50%);
+  align-items: center;
+}
+.position-bottom-right {
+  bottom: var(--offset-y);
+  right: var(--offset-x);
+  align-items: flex-end;
+}
+.position-center {
+  top: 50%;
+  left: 50%;
+  transform: translate(-50%, -50%);
+  align-items: center;
+}
+/* Stack direction */
+.stack-up {
+  flex-direction: column-reverse;
+}
+.stack-up .notification-list {
+  flex-direction: column-reverse;
+}
+/* Base notification styles */
+.notification {
+  background: white;
+  border-radius: 8px;
+  box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15);
+  min-width: 300px;
+  max-width: 500px;
+  position: relative;
+  overflow: hidden;
+  pointer-events: all;
+  transition: all 0.3s ease;
+}
+.notification-content {
+  padding: 16px;
+  display: flex;
+  gap: 12px;
+  align-items: flex-start;
+}
+.notification-icon {
+  flex-shrink: 0;
+  font-size: 24px;
+  display: flex;
+  align-items: center;
+}
+.notification-body {
+  flex: 1;
+  min-width: 0;
+}
+.notification-title {
+  font-weight: 600;
+  font-size: 16px;
+  margin-bottom: 4px;
+  color: #333;
+  word-wrap: break-word;
+}
+.notification-message {
+  font-size: 14px;
+  color: #666;
+  line-height: 1.5;
+  word-wrap: break-word;
+}
+.notification-actions {
+  display: flex;
+  gap: 8px;
+  margin-top: 12px;
+  flex-wrap: wrap;
+}
+.notification-action {
+  padding: 6px 12px;
+  border: none;
+  border-radius: 4px;
+  font-size: 14px;
+  font-weight: 500;
+  cursor: pointer;
+  transition: all 0.2s;
+  background: #1976d2;
+  color: white;
+}
+.notification-action:hover {
+  background: #1565c0;
+  transform: translateY(-1px);
+}
+.notification-close {
+  background: none;
+  border: none;
+  color: #999;
+  cursor: pointer;
+  font-size: 18px;
+  padding: 4px;
+  transition: color 0.2s;
+  display: flex;
+  align-items: center;
+}
+.notification-close:hover {
+  color: #666;
+}
+.notification-footer {
+  padding: 12px 16px;
+  background: #f5f5f5;
+  border-top: 1px solid #e0e0e0;
+  font-size: 12px;
+  color: #666;
+}
+/* Notification types */
+.notification-success {
+  border-left: 4px solid #4caf50;
+}
+.notification-success .notification-icon {
+  color: #4caf50;
+}
+.notification-error {
+  border-left: 4px solid #f44336;
+}
+.notification-error .notification-icon {
+  color: #f44336;
+}
+.notification-warning {
+  border-left: 4px solid #ff9800;
+}
+.notification-warning .notification-icon {
+  color: #ff9800;
+}
+.notification-info {
+  border-left: 4px solid #2196f3;
+}
+.notification-info .notification-icon {
+  color: #2196f3;
+}
+/* Dark theme */
+.theme-dark .notification {
+  background: #1e1e1e;
+  color: #fff;
+  box-shadow: 0 4px 12px rgba(0, 0, 0, 0.5);
+}
+.theme-dark .notification-title {
+  color: #fff;
+}
+.theme-dark .notification-message {
+  color: #ccc;
+}
+.theme-dark .notification-close {
+  color: #666;
+}
+.theme-dark .notification-close:hover {
+  color: #999;
+}
+.theme-dark .notification-footer {
+  background: #2a2a2a;
+  border-top-color: #444;
+  color: #999;
+}
+/* Animations */
+/* Slide animation */
+.animation-slide {
+  animation: slideIn 0.3s ease-out forwards;
+  animation-delay: var(--animation-delay, 0s);
+}
+@keyframes slideIn {
+  from {
+    transform: translateX(100%);
+    opacity: 0;
+  }
+  to {
+    transform: translateX(0);
+    opacity: 1;
+  }
+}
+.position-top-left .animation-slide,
+.position-bottom-left .animation-slide {
+  animation-name: slideInLeft;
+}
+@keyframes slideInLeft {
+  from {
+    transform: translateX(-100%);
+    opacity: 0;
+  }
+  to {
+    transform: translateX(0);
+    opacity: 1;
+  }
+}
+/* Fade animation */
+.animation-fade {
+  animation: fadeIn 0.3s ease-out forwards;
+  animation-delay: var(--animation-delay, 0s);
+}
+@keyframes fadeIn {
+  from {
+    opacity: 0;
+  }
+  to {
+    opacity: 1;
+  }
+}
+/* Zoom animation */
+.animation-zoom {
+  animation: zoomIn 0.3s ease-out forwards;
+  animation-delay: var(--animation-delay, 0s);
+}
+@keyframes zoomIn {
+  from {
+    transform: scale(0.8);
+    opacity: 0;
+  }
+  to {
+    transform: scale(1);
+    opacity: 1;
+  }
+}
+/* Bounce animation */
+.animation-bounce {
+  animation: bounceIn 0.6s cubic-bezier(0.68, -0.55, 0.265, 1.55) forwards;
+  animation-delay: var(--animation-delay, 0s);
+}
+@keyframes bounceIn {
+  0% {
+    transform: translateY(-100%);
+    opacity: 0;
+  }
+  60% {
+    transform: translateY(10%);
+    opacity: 1;
+  }
+  100% {
+    transform: translateY(0);
+    opacity: 1;
+  }
+}
+/* Progress bar */
+.notification-progress {
+  position: absolute;
+  bottom: 0;
+  left: 0;
+  height: 3px;
+  background: currentColor;
+  opacity: 0.3;
+  animation: progress linear forwards;
+  animation-duration: var(--duration);
+}
+@keyframes progress {
+  from {
+    width: 100%;
+  }
+  to {
+    width: 0%;
+  }
+}
+/* Collapse toggle */
+.notification-collapse-toggle {
+  align-self: center;
+  padding: 8px 16px;
+  background: #1976d2;
+  color: white;
+  border: none;
+  border-radius: 20px;
+  font-size: 14px;
+  cursor: pointer;
+  pointer-events: all;
+  margin-bottom: 8px;
+  transition: all 0.2s;
+}
+.notification-collapse-toggle:hover {
+  background: #1565c0;
+  transform: translateY(-1px);
+}
+/* Responsive */
+@media (max-width: 600px) {
+  .notification {
+    min-width: calc(100vw - 40px);
+    max-width: calc(100vw - 40px);
+  }
+  .position-top-center,
+  .position-bottom-center {
+    transform: none;
+    left: 20px;
+    right: 20px;
+  }
+}

frontend/src/Components/AiComponents/Notifications/Notification.js ADDED Viewed

	@@ -0,0 +1,242 @@

+import React, { useState, useEffect, useRef, useCallback } from 'react';
+import {
+  FaTimes,
+  FaCheckCircle,
+  FaExclamationCircle,
+  FaInfoCircle,
+  FaExclamationTriangle
+} from 'react-icons/fa';
+import './Notification.css';
+const Notification = ({
+  notifications = [],
+  position = 'top-right',
+  animation = 'slide',
+  stackDirection = 'down',
+  maxNotifications = 5,
+  spacing = 10,
+  offset = { x: 20, y: 20 },
+  onDismiss,
+  onAction,
+  autoStackCollapse = false,
+  theme = 'light'
+}) => {
+  const [internalNotifications, setInternalNotifications] = useState([]);
+  const [collapsed, setCollapsed] = useState(false);
+  const timersRef = useRef({});
+  const handleDismiss = useCallback((id) => {
+    if (timersRef.current[id]) {
+      clearTimeout(timersRef.current[id]);
+      delete timersRef.current[id];
+    }
+    onDismiss?.(id);
+  }, [onDismiss]);
+  useEffect(() => {
+    // Update internal notifications
+    const processedNotifications = notifications.slice(
+      stackDirection === 'up' ? -maxNotifications : 0,
+      stackDirection === 'up' ? undefined : maxNotifications
+    );
+    setInternalNotifications(processedNotifications);
+    // Keep track of current timer IDs for this effect
+    const currentTimerIds = [];
+    // Set up auto-dismiss timers
+    processedNotifications.forEach(notification => {
+      if (notification.autoDismiss && notification.duration && !timersRef.current[notification.id]) {
+        const timerId = setTimeout(() => {
+          handleDismiss(notification.id);
+        }, notification.duration);
+        timersRef.current[notification.id] = timerId;
+        currentTimerIds.push(notification.id);
+      }
+    });
+    // Cleanup function
+    return () => {
+      // Use the captured timer IDs and current ref
+      const timers = timersRef.current;
+      // Clear timers for notifications that were removed
+      Object.keys(timers).forEach(id => {
+        if (!processedNotifications.find(n => n.id === id)) {
+          clearTimeout(timers[id]);
+          delete timers[id];
+        }
+      });
+    };
+  }, [notifications, maxNotifications, stackDirection, handleDismiss]);
+  const handleAction = (notificationId, actionId, actionData) => {
+    onAction?.(notificationId, actionId, actionData);
+  };
+  const getIcon = (type, customIcon) => {
+    if (customIcon) return customIcon;
+    switch (type) {
+      case 'success':
+        return <FaCheckCircle />;
+      case 'error':
+        return <FaExclamationCircle />;
+      case 'warning':
+        return <FaExclamationTriangle />;
+      case 'info':
+        return <FaInfoCircle />;
+      default:
+        return null;
+    }
+  };
+  const getPositionClasses = () => {
+    const classes = ['notification-container'];
+    // Position classes
+    switch (position) {
+      case 'top-left':
+        classes.push('position-top-left');
+        break;
+      case 'top-center':
+        classes.push('position-top-center');
+        break;
+      case 'top-right':
+        classes.push('position-top-right');
+        break;
+      case 'bottom-left':
+        classes.push('position-bottom-left');
+        break;
+      case 'bottom-center':
+        classes.push('position-bottom-center');
+        break;
+      case 'bottom-right':
+        classes.push('position-bottom-right');
+        break;
+      case 'center':
+        classes.push('position-center');
+        break;
+      default:
+        classes.push('position-top-right');
+    }
+    // Stack direction
+    if (stackDirection === 'up') {
+      classes.push('stack-up');
+    }
+    // Theme
+    classes.push(`theme-${theme}`);
+    return classes.join(' ');
+  };
+  const getAnimationClass = (index) => {
+    return `animation-${animation} animation-${animation}-${index}`;
+  };
+  const containerStyle = {
+    '--spacing': `${spacing}px`,
+    '--offset-x': `${offset.x}px`,
+    '--offset-y': `${offset.y}px`,
+  };
+  if (internalNotifications.length === 0) return null;
+  return (
+    <div
+      className={getPositionClasses()}
+      style={containerStyle}
+    >
+      {autoStackCollapse && internalNotifications.length > 3 && (
+        <button
+          className="notification-collapse-toggle"
+          onClick={() => setCollapsed(!collapsed)}
+        >
+          {collapsed ? `Show ${internalNotifications.length} notifications` : 'Collapse'}
+        </button>
+      )}
+      <div className={`notification-list ${collapsed ? 'collapsed' : ''}`}>
+        {internalNotifications.map((notification, index) => (
+          <div
+            key={notification.id}
+            className={`notification notification-${notification.type || 'default'} ${getAnimationClass(index)} ${notification.className || ''}`}
+            style={{
+              '--animation-delay': `${index * 0.05}s`,
+              ...notification.style
+            }}
+          >
+            {notification.showProgress && notification.duration && (
+              <div
+                className="notification-progress"
+                style={{
+                  '--duration': `${notification.duration}ms`
+                }}
+              />
+            )}
+            <div className="notification-content">
+              {(notification.icon !== false) && (
+                <div className="notification-icon">
+                  {getIcon(notification.type, notification.icon)}
+                </div>
+              )}
+              <div className="notification-body">
+                {notification.title && (
+                  <div className="notification-title">{notification.title}</div>
+                )}
+                {notification.message && (
+                  <div className="notification-message">
+                    {typeof notification.message === 'string'
+                      ? notification.message
+                      : notification.message
+                    }
+                  </div>
+                )}
+                {notification.actions && notification.actions.length > 0 && (
+                  <div className="notification-actions">
+                    {notification.actions.map((action) => (
+                      <button
+                        key={action.id}
+                        className={`notification-action ${action.className || ''}`}
+                        onClick={() => handleAction(notification.id, action.id, action.data)}
+                        style={action.style}
+                      >
+                        {action.label}
+                      </button>
+                    ))}
+                  </div>
+                )}
+              </div>
+              {notification.dismissible !== false && (
+                <button
+                  className="notification-close"
+                  onClick={() => handleDismiss(notification.id)}
+                  aria-label="Dismiss notification"
+                >
+                  <FaTimes />
+                </button>
+              )}
+            </div>
+            {notification.footer && (
+              <div className="notification-footer">
+                {notification.footer}
+              </div>
+            )}
+          </div>
+        ))}
+      </div>
+    </div>
+  );
+};
+export default Notification;

frontend/src/Components/AiComponents/Notifications/useNotification.js ADDED Viewed

	@@ -0,0 +1,43 @@

+import { useState, useCallback } from 'react';
+export const useNotification = () => {
+  const [notifications, setNotifications] = useState([]);
+  const addNotification = useCallback((notification) => {
+    const id = notification.id || `notification-${Date.now()}-${Math.random()}`;
+    const newNotification = {
+      id,
+      type: 'info',
+      dismissible: true,
+      autoDismiss: false,
+      duration: 5000,
+      showProgress: false,
+      ...notification
+    };
+    setNotifications(prev => [...prev, newNotification]);
+    return id;
+  }, []);
+  const removeNotification = useCallback((id) => {
+    setNotifications(prev => prev.filter(n => n.id !== id));
+  }, []);
+  const clearAll = useCallback(() => {
+    setNotifications([]);
+  }, []);
+  const updateNotification = useCallback((id, updates) => {
+    setNotifications(prev =>
+      prev.map(n => n.id === id ? { ...n, ...updates } : n)
+    );
+  }, []);
+  return {
+    notifications,
+    addNotification,
+    removeNotification,
+    clearAll,
+    updateNotification
+  };
+};

frontend/src/Components/AiComponents/Sidebars/LeftSideBar.js ADDED Viewed

	@@ -0,0 +1,38 @@

+import React, { useState, useEffect } from 'react';
+import { FaBars } from 'react-icons/fa';
+import './LeftSidebar.css';
+function LeftSidebar() {
+  const [isLeftSidebarOpen, setLeftSidebarOpen] = useState(
+    localStorage.getItem("leftSidebarState") === "true"
+  );
+  useEffect(() => {
+    localStorage.setItem("leftSidebarState", isLeftSidebarOpen);
+  }, [isLeftSidebarOpen]);
+  const toggleLeftSidebar = () => {
+    setLeftSidebarOpen(!isLeftSidebarOpen);
+  };
+  return (
+    <>
+      <nav className={`left-side-bar ${isLeftSidebarOpen ? 'open' : 'closed'}`}>
+        ... (left sidebar content)
+      </nav>
+      {!isLeftSidebarOpen && (
+        <button className='toggle-btn left-toggle' onClick={toggleLeftSidebar}>
+          <FaBars />
+        </button>
+      )}
+    </>
+  );
+//   return (
+//     <div className="left-side-bar-placeholder">
+//       {/* Left sidebar is currently disabled. Uncomment the code in LeftSidebar.js to enable it. */}
+//       Left sidebar is disabled.
+//     </div>
+//   );
+}
+export default LeftSidebar;

frontend/src/Components/AiComponents/Sidebars/LeftSidebar.css ADDED Viewed

	@@ -0,0 +1,59 @@

+/* Left Sidebar Specific */
+.left-side-bar {
+    background-color: var(--primary-color);
+    color: var(--text-color);
+    display: flex;
+    flex-direction: column;
+    padding: 1rem;
+    transition: transform var(--transition-speed);
+    z-index: 1000;
+    position: absolute;
+    top: 0;
+    left: 0;
+    height: 100%;
+  }
+  .left-side-bar.closed {
+    transform: translateX(-100%);
+  }
+  /* Toggle Button for Left Sidebar */
+  .toggle-btn.left-toggle {
+    background-color: var(--primary-color);
+    color: var(--text-color);
+    border: none;
+    padding: 0.5rem;
+    border-radius: 4px;
+    cursor: pointer;
+    transition: background-color var(--transition-speed);
+    z-index: 1100;
+    position: fixed;
+    top: 50%;
+    left: 0;
+    transform: translate(-50%, -50%);
+  }
+  /* Responsive Adjustments for Left Sidebar */
+  @media (max-width: 768px) {
+    .left-side-bar {
+      width: 200px;
+    }
+  }
+  @media (max-width: 576px) {
+    .left-side-bar {
+      width: 100%;
+      height: 100%;
+      top: 0;
+      left: 0;
+      transform: translateY(-100%);
+    }
+    .left-side-bar.open {
+      transform: translateY(0);
+    }
+    .toggle-btn.left-toggle {
+      top: auto;
+      bottom: 1rem;
+      left: 1rem;
+    }
+  }

frontend/src/Components/AiComponents/Sidebars/RightSidebar.css ADDED Viewed

	@@ -0,0 +1,138 @@

+:root {
+  /* Dark theme variables */
+  --sidebar-background: #2b2b2b;
+  --text-light: #eee;
+  --border-dark: #333;
+}
+/* Main sidebar container */
+.right-side-bar {
+  display: flex;
+  flex-direction: column;
+  position: fixed;
+  top: 0;
+  right: 0;
+  height: 100%;
+  background-color: var(--sidebar-background);  /* Keep background uniform */
+  color: var(--text-light);
+  box-shadow: -2px 0 8px rgba(0, 0, 0, 0.5);
+  transition: width 0.4s ease;
+  overflow-y: auto;
+  z-index: 1000;
+}
+/* Sidebar resizing */
+.right-side-bar.resizing {
+  transition: none;
+}
+/* When the sidebar is closed */
+.right-side-bar.closed {
+  width: 0;
+  overflow: hidden;
+}
+/* Sidebar header styling */
+.sidebar-header {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  padding: 16px;
+  border-bottom: 3px solid var(--border-dark);
+}
+.sidebar-header h3 {
+  margin: 0;
+  font-size: 1.2rem;
+}
+/* Close button styling */
+.close-btn {
+  background: none;
+  border: none;
+  padding: 6px;
+  color: var(--text-color);
+  font-size: 1.2rem;
+  cursor: pointer;
+  transition: color var(--transition-speed);
+}
+.close-btn:hover {
+  background: rgba(255, 255, 255, 0.1);
+  color: white;
+}
+/* Ensure the sidebar background remains uniform */
+.sidebar-content {
+  padding: 16px;
+  background: transparent;
+  overflow-x: hidden;
+  overflow-y: auto;
+}
+/* Also clear any default marker via the pseudo-element */
+.nav-links.no-bullets li::marker {
+  content: "";
+}
+/* Lay out each task item using flex so that the icon and text align */
+.task-item {
+  display: flex;
+  align-items: flex-start;
+  margin-bottom: 1rem;
+}
+/* Icon span: fixed width and margin for spacing */
+.task-icon {
+  flex-shrink: 0;
+  margin-right: 1rem;
+}
+/* Task list text */
+.task-text {
+  white-space: pre-wrap;
+}
+/* Resizer for sidebar width adjustment */
+.resizer {
+  position: absolute;
+  left: 0;
+  top: 0;
+  width: 5px;
+  height: 100%;
+  cursor: ew-resize;
+}
+/* Toggle button (when sidebar is closed) */
+.toggle-btn.right-toggle {
+  position: fixed;
+  top: 50%;
+  right: 0;
+  transform: translateY(-50%);
+  background-color: var(--dark-surface);
+  color: var(--text-light);
+  border: none;
+  padding: 8px;
+  cursor: pointer;
+  z-index: 1001;
+  box-shadow: -2px 0 4px rgba(0, 0, 0, 0.5);
+}
+.spin {
+  animation: spin 1s linear infinite;
+  color: #328bff;
+}
+.checkmark {
+  color: #03c203;
+}
+.x {
+  color: #d10808;
+}
+/* Keyframes for the spinner animation */
+@keyframes spin {
+  from { transform: rotate(0deg); }
+  to { transform: rotate(360deg); }
+}

frontend/src/Components/AiComponents/Sidebars/RightSidebar.js ADDED Viewed

	@@ -0,0 +1,142 @@

+import React, { useRef } from 'react';
+import { FaTimes, FaCheck, FaSpinner } from 'react-icons/fa';
+import { BsChevronLeft } from 'react-icons/bs';
+import CircularProgress from '@mui/material/CircularProgress';
+import Sources from '../ChatComponents/Sources';
+import Evaluate from '../ChatComponents/Evaluate';
+import './RightSidebar.css';
+function RightSidebar({
+  isOpen,
+  rightSidebarWidth,
+  setRightSidebarWidth,
+  toggleRightSidebar,
+  sidebarContent,
+  tasks = [],
+  tasksLoading,
+  sources = [],
+  sourcesLoading,
+  onTaskClick,
+  onSourceClick,
+  evaluation
+}) {
+  const minWidth = 200;
+  const maxWidth = 450;
+  const sidebarRef = useRef(null);
+  // Called when the user starts resizing the sidebar.
+  const startResize = (e) => {
+    e.preventDefault();
+    sidebarRef.current.classList.add("resizing"); // Add the "resizing" class to the sidebar when resizing
+    document.addEventListener("mousemove", resizeSidebar);
+    document.addEventListener("mouseup", stopResize);
+  };
+  const resizeSidebar = (e) => {
+    let newWidth = window.innerWidth - e.clientX;
+    if (newWidth < minWidth) newWidth = minWidth;
+    if (newWidth > maxWidth) newWidth = maxWidth;
+    setRightSidebarWidth(newWidth);
+  };
+  const stopResize = () => {
+    sidebarRef.current.classList.remove("resizing"); // Remove the "resizing" class from the sidebar when resizing stops
+    document.removeEventListener("mousemove", resizeSidebar);
+    document.removeEventListener("mouseup", stopResize);
+  };
+  // Default handler for source clicks: open the link in a new tab.
+  const handleSourceClick = (source) => {
+    if (source && source.link) {
+      window.open(source.link, '_blank');
+    }
+  };
+  // Helper function to return the proper icon based on task status.
+  const getTaskIcon = (task) => {
+    // If the task is a simple string, default to the completed icon.
+    if (typeof task === 'string') {
+      return <FaCheck />;
+    }
+    // Use the status field to determine which icon to render.
+    switch (task.status) {
+      case 'RUNNING':
+        // FaSpinner is used for running tasks. The CSS class "spin" can be defined to add animation.
+        return <FaSpinner className="spin"/>;
+      case 'DONE':
+        return <FaCheck className="checkmark" />;
+      case 'FAILED':
+        return <FaTimes className="x" />;
+      default:
+        return <FaCheck />;
+    }
+  };
+  return (
+    <>
+      <nav
+        ref={sidebarRef}
+        className={`right-side-bar ${isOpen ? "open" : "closed"}`}
+        style={{ width: isOpen ? rightSidebarWidth : 0 }}
+      >
+        <div className="sidebar-header">
+          <h3>
+              {sidebarContent === "sources"
+                ? "Sources"
+                : sidebarContent === "evaluate"
+                ? "Evaluation"
+                : "Tasks"}
+            </h3>
+          <button className="close-btn" onClick={toggleRightSidebar}>
+            <FaTimes />
+          </button>
+        </div>
+        <div className="sidebar-content">
+          {sidebarContent === "sources" ? ( // If the sidebar content is "sources", show the sources component
+            sourcesLoading ? (
+              <div className="tasks-loading">
+                <CircularProgress size={20} sx={{ color: '#ccc' }} />
+                <span className="loading-tasks-text">Generating sources...</span>
+              </div>
+            ) : (
+              <Sources sources={sources} handleSourceClick={onSourceClick || handleSourceClick} />
+            )
+          )
+          // If the sidebar content is "evaluate", show the evaluation component
+          : sidebarContent === "evaluate" ? (
+            <Evaluate evaluation={evaluation} />
+          ) : (
+            // Otherwise, show tasks
+            tasksLoading ? (
+              <div className="tasks-loading">
+                <CircularProgress size={20} sx={{ color: '#ccc' }} />
+                <span className="loading-tasks-text">Generating tasks...</span>
+              </div>
+            ) : (
+              <ul className="nav-links" style={{ listStyle: 'none', padding: 0 }}>
+                {tasks.map((task, index) => (
+                  <li key={index} className="task-item">
+                    <span className="task-icon">
+                      {getTaskIcon(task)}
+                    </span>
+                    <span className="task-text">
+                      {typeof task === 'string' ? task : task.task}
+                    </span>
+                  </li>
+                ))}
+              </ul>
+            )
+          )}
+        </div>
+        <div className="resizer" onMouseDown={startResize}></div>
+      </nav>
+      {!isOpen && (
+        <button className="toggle-btn right-toggle" onClick={toggleRightSidebar}>
+          <BsChevronLeft />
+        </button>
+      )}
+    </>
+  );
+}
+export default RightSidebar;

frontend/src/Components/AiPage.css CHANGED Viewed

@@ -7,6 +7,7 @@
   --text-color: #e0e0e0;          /* Off-white text */
   --hover-bg: #3a3a3a;           /* Slightly lighter for hover effects */
   --transition-speed: 0.25s;      /* Speed of transitions */
 }
 /* Global font settings */
@@ -58,7 +59,7 @@ html, body {
   position: relative;
   width: 100%;
   border-radius: 0.35rem;
-  background-color: #21212f;
 }
 .search-input-wrapper {
@@ -100,6 +101,7 @@ html, body {
 }
 .settings-btn,
 .send-btn {
   background: transparent;
   border: none;
@@ -108,11 +110,13 @@ html, body {
 }
 .settings-btn svg,
 .send-btn svg {
   font-size: 1.45rem;
 }
 .settings-btn:hover,
 .send-btn:hover {
   color: #888;
 }
@@ -145,7 +149,7 @@ button.send-btn.stop-btn:hover {
   left: 50%;
   transform: translateX(-50%);
   width: 48%;
-  background-color: #21212f;
   border-radius: 0.35rem;
 }
@@ -173,11 +177,17 @@ button.send-btn.stop-btn:hover {
   background-color: transparent;
   color: var(--text-color);
   line-height: 1.4;
-  padding: 0.65rem 3.25rem;
   resize: none;
   white-space: pre-wrap;
 }
 .chat-search-input:focus {
   outline: none;
 }
@@ -196,11 +206,19 @@ button.send-btn.stop-btn:hover {
 }
 /* Re-enable pointer events on the actual buttons so they remain clickable */
-.chat-icon-container button {
   pointer-events: auto;
 }
 .chat-settings-btn,
 .chat-send-btn {
   background: transparent;
   border: none;
@@ -211,10 +229,52 @@ button.send-btn.stop-btn:hover {
 }
 .chat-settings-btn:hover,
 .chat-send-btn:hover {
   color: #888;
 }
 /* Floating sidebar container for chat mode */
 .floating-sidebar {
   position: fixed;
@@ -228,7 +288,7 @@ button.send-btn.stop-btn:hover {
 /* Chat container */
 .chat-container {
   flex-grow: 1;
-  margin-bottom: 9rem;
 }
 /* Responsive Adjustments */
@@ -244,4 +304,4 @@ button.send-btn.stop-btn:hover {
     margin: 0;
     padding: 1rem;
   }
-}

   --text-color: #e0e0e0;          /* Off-white text */
   --hover-bg: #3a3a3a;           /* Slightly lighter for hover effects */
   --transition-speed: 0.25s;      /* Speed of transitions */
+  --search-bar: #21212f;      /* Search bar background */
 }
 /* Global font settings */
   position: relative;
   width: 100%;
   border-radius: 0.35rem;
+  background-color: var(--search-bar);
 }
 .search-input-wrapper {
 }
 .settings-btn,
+.add-btn,
 .send-btn {
   background: transparent;
   border: none;
 }
 .settings-btn svg,
+.add-btn svg,
 .send-btn svg {
   font-size: 1.45rem;
 }
 .settings-btn:hover,
+.add-btn:hover,
 .send-btn:hover {
   color: #888;
 }
   left: 50%;
   transform: translateX(-50%);
   width: 48%;
+  background-color: var(--search-bar);
   border-radius: 0.35rem;
 }
   background-color: transparent;
   color: var(--text-color);
   line-height: 1.4;
+  padding: 0.65rem 3.25rem 0.65rem 5.5rem;
   resize: none;
   white-space: pre-wrap;
 }
+.left-icons {
+  display: flex;
+  align-items: center;
+  gap: 0.15rem;
+}
 .chat-search-input:focus {
   outline: none;
 }
 }
 /* Re-enable pointer events on the actual buttons so they remain clickable */
+.chat-icon-container button,
+.chat-left-icons {
   pointer-events: auto;
 }
+.chat-left-icons {
+  display: flex;
+  align-items: center;
+  gap: 0.15rem;
+}
 .chat-settings-btn,
+.chat-add-btn,
 .chat-send-btn {
   background: transparent;
   border: none;
 }
 .chat-settings-btn:hover,
+.chat-add-btn:hover,
 .chat-send-btn:hover {
   color: #888;
 }
+/* Tooltip Wrapper */
+.tooltip-wrapper {
+  position: relative;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+/* Tooltip styling */
+.tooltip {
+  position: absolute;
+  bottom: 100%;
+  left: 50%;
+  transform: translateX(-50%) translateY(10px) scale(0.9);
+  transform-origin: bottom center;
+  margin-bottom: 0.65rem;
+  padding: 0.3rem 0.6rem;
+  background-color: var(--primary-color);
+  color: var(--text-color);
+  border-radius: 0.25rem;
+  white-space: nowrap;
+  font-size: 0.85rem;
+  opacity: 0;
+  visibility: hidden;
+  transition: transform 0.3s ease, opacity 0.3s ease;
+  z-index: 10;
+}
+/* Show the tooltip on hover */
+.tooltip-wrapper:hover .tooltip {
+  opacity: 1;
+  visibility: visible;
+  transform: translateX(-50%) translateY(0) scale(1);
+}
+/* Hide tooltip when its associated dropdown is open */
+.tooltip-wrapper .tooltip.hidden {
+  opacity: 0;
+  visibility: hidden;
+ }
 /* Floating sidebar container for chat mode */
 .floating-sidebar {
   position: fixed;
 /* Chat container */
 .chat-container {
   flex-grow: 1;
+  margin-bottom: 40rem;
 }
 /* Responsive Adjustments */
     margin: 0;
     padding: 1rem;
   }
+}

frontend/src/Components/AiPage.js CHANGED Viewed

@@ -2,10 +2,14 @@ import React, { useState, useEffect, useRef, useCallback, useMemo } from 'react'
 import { flushSync } from 'react-dom';
 import Snackbar from '@mui/material/Snackbar';
 import Alert from '@mui/material/Alert';
-import { FaCog, FaPaperPlane, FaStop } from 'react-icons/fa';
 import IntialSetting from './IntialSetting';
 import ChatWindow from './AiComponents/ChatWindow';
-import RightSidebar from './AiComponents/ChatComponents/RightSidebar';
 import './AiPage.css';
 function AiPage() {
@@ -24,9 +28,18 @@ function AiPage() {
   const [chatBlocks, setChatBlocks] = useState([]);
   const [selectedChatBlockId, setSelectedChatBlockId] = useState(null);
   const [defaultChatHeight, setDefaultChatHeight] = useState(null);
   const [chatBottomPadding, setChatBottomPadding] = useState("60px");
   // States/refs for streaming
   const [isProcessing, setIsProcessing] = useState(false);
   const [activeBlockId, setActiveBlockId] = useState(null);
@@ -39,21 +52,59 @@ function AiPage() {
     severity: "success",
   });
   // Function to open the snackbar
-  const openSnackbar = (message, severity = "success") => {
-    setSnackbar({ open: true, message, severity });
-  };
   // Function to close the snackbar
   const closeSnackbar = (event, reason) => {
     if (reason === 'clickaway') return;
-    setSnackbar(prev => ({ ...prev, open: false }));
   };
   useEffect(() => {
     localStorage.setItem("rightSidebarState", isRightSidebarOpen);
   }, [isRightSidebarOpen]);
   useEffect(() => {
     document.documentElement.style.setProperty('--right-sidebar-width', rightSidebarWidth + 'px');
   }, [rightSidebarWidth]);
@@ -89,6 +140,40 @@ function AiPage() {
     }
   }, [searchText, defaultChatHeight]);
   const handleOpenRightSidebar = (content, chatBlockId = null) => {
     flushSync(() => {
       if (chatBlockId) {
@@ -109,6 +194,148 @@ function AiPage() {
     );
   }, []);
   // Initiate the SSE
   const initiateSSE = (query, blockId) => {
     const startTime = Date.now();
@@ -138,6 +365,7 @@ function AiPage() {
     });
     eventSource.addEventListener("final_message", (e) => {
       const endTime = Date.now();
       const thinkingTime = ((endTime - startTime) / 1000).toFixed(1);
       // Only update thinkingTime so the streaming flag turns false and the cursor disappears
@@ -147,6 +375,19 @@ function AiPage() {
           : block
       ));
     });
     // Listen for the "complete" event to know when to close the connection.
     eventSource.addEventListener("complete", (e) => {
@@ -337,6 +578,270 @@ function AiPage() {
     if (searchText.trim()) handleSend();
   };
   // Get the chat block whose details should be shown in the sidebar.
   const selectedBlock = chatBlocks.find(block => block.id === selectedChatBlockId);
   const evaluateAction = selectedBlock && selectedBlock.actions
@@ -362,6 +867,18 @@ function AiPage() {
           : 0,
       }}
     >
       {showChatWindow && selectedBlock && (sidebarContent !== "default" || (selectedBlock.tasks && selectedBlock.tasks.length > 0) || (selectedBlock.sources && selectedBlock.sources.length > 0)) && (
         <div className="floating-sidebar">
           <RightSidebar
@@ -397,6 +914,10 @@ function AiPage() {
                   thinkingTime={block.thinkingTime}
                   thoughtLabel={block.thoughtLabel}
                   sourcesRead={block.sourcesRead}
                   actions={block.actions}
                   tasks={block.tasks}
                   openRightSidebar={handleOpenRightSidebar}
@@ -426,19 +947,44 @@ function AiPage() {
                 />
               </div>
               <div className="chat-icon-container">
-                <button
-                  className="chat-settings-btn"
-                  onClick={() => setShowSettingsModal(true)}
-                >
-                  <FaCog />
-                </button>
                 {/* Conditionally render Stop or Send button */}
-                <button
-                  className={`chat-send-btn ${isProcessing ? 'stop-btn' : ''}`}
-                  onClick={isProcessing ? handleStop : handleSendButtonClick}
-                >
-                  {isProcessing ? <FaStop size={12} color="black" /> : <FaPaperPlane />}
-                </button>
               </div>
             </div>
           </>
@@ -458,24 +1004,49 @@ function AiPage() {
                 />
               </div>
               <div className="icon-container">
-                <button
-                  className="settings-btn"
-                  onClick={() => setShowSettingsModal(true)}
-                >
-                  <FaCog />
-                </button>
-                <button
-                  className={`send-btn ${isProcessing ? 'stop-btn' : ''}`}
-                  onClick={isProcessing ? handleStop : handleSendButtonClick}
-                >
-                  {isProcessing ? <FaStop /> : <FaPaperPlane />}
-                </button>
               </div>
             </div>
           </div>
         )}
       </main>
       {showSettingsModal && (
         <IntialSetting
           trigger={true}
@@ -485,9 +1056,17 @@ function AiPage() {
           closeSnackbar={closeSnackbar}
         />
       )}
       <Snackbar
         open={snackbar.open}
-        autoHideDuration={snackbar.severity === 'success' ? 3000 : null}
         onClose={closeSnackbar}
         anchorOrigin={{ vertical: 'top', horizontal: 'center' }}
       >

 import { flushSync } from 'react-dom';
 import Snackbar from '@mui/material/Snackbar';
 import Alert from '@mui/material/Alert';
+import { FaCog, FaPaperPlane, FaStop, FaPlus, FaGoogle, FaMicrosoft, FaSlack } from 'react-icons/fa';
 import IntialSetting from './IntialSetting';
+import AddContentDropdown from './AiComponents/Dropdowns/AddContentDropdown';
+import AddFilesDialog from './AiComponents/Dropdowns/AddFilesDialog';
 import ChatWindow from './AiComponents/ChatWindow';
+import RightSidebar from './AiComponents/Sidebars/RightSidebar';
+import Notification from '../Components/AiComponents/Notifications/Notification';
+import { useNotification } from '../Components/AiComponents/Notifications/useNotification';
 import './AiPage.css';
 function AiPage() {
   const [chatBlocks, setChatBlocks] = useState([]);
   const [selectedChatBlockId, setSelectedChatBlockId] = useState(null);
+  const addBtnRef = useRef(null);
+  const chatAddBtnRef = useRef(null);
+  const [isAddContentOpen, setAddContentOpen] = useState(false);
+  const [isTooltipSuppressed, setIsTooltipSuppressed] = useState(false);
+  const [isAddFilesDialogOpen, setIsAddFilesDialogOpen] = useState(false);
   const [defaultChatHeight, setDefaultChatHeight] = useState(null);
   const [chatBottomPadding, setChatBottomPadding] = useState("60px");
+  const [sessionContent, setSessionContent] = useState({ files: [], links: [] });
   // States/refs for streaming
   const [isProcessing, setIsProcessing] = useState(false);
   const [activeBlockId, setActiveBlockId] = useState(null);
     severity: "success",
   });
+  // State for tracking selected services
+  const [selectedServices, setSelectedServices] = useState({
+    google: [],
+    microsoft: [],
+    slack: false
+  });
+  // Notifications
+  const {
+    notifications,
+    addNotification,
+    removeNotification,
+    updateNotification
+  } = useNotification();
+  // Token management
+  const tokenExpiryTimersRef = useRef({});
+  const notificationIdsRef = useRef({});
   // Function to open the snackbar
+  const openSnackbar = useCallback((message, severity = "success", duration) => {
+    let finalDuration;
+    if (duration !== undefined) {
+      // If a specific duration is provided (e.g., 5000 or null), use it.
+      finalDuration = duration;
+    } else {
+      // Otherwise, use the default logic.
+      finalDuration = severity === 'success' ? 3000 : null; // Success auto-hides, others are persistent by default.
+    }
+    setSnackbar({ open: true, message, severity, duration: finalDuration });
+  }, []);
   // Function to close the snackbar
   const closeSnackbar = (event, reason) => {
     if (reason === 'clickaway') return;
+    setSnackbar(prev => ({ ...prev, open: false, duration: null }));
   };
   useEffect(() => {
     localStorage.setItem("rightSidebarState", isRightSidebarOpen);
   }, [isRightSidebarOpen]);
+  // Add cleanup handler for when the user closes the tab/browser
+  useEffect(() => {
+    const handleCleanup = () => {
+      navigator.sendBeacon('/cleanup');
+    };
+    window.addEventListener('beforeunload', handleCleanup);
+    return () => window.removeEventListener('beforeunload', handleCleanup);
+  }, []);
   useEffect(() => {
     document.documentElement.style.setProperty('--right-sidebar-width', rightSidebarWidth + 'px');
   }, [rightSidebarWidth]);
     }
   }, [searchText, defaultChatHeight]);
+  // Update backend whenever selected services change
+  useEffect(() => {
+    const updateSelectedServices = async () => {
+      try {
+        await fetch('/api/selected-services', {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({
+            services: selectedServices
+          })
+        });
+      } catch (error) {
+        console.error('Failed to update selected services:', error);
+      }
+    };
+    updateSelectedServices();
+  }, [selectedServices]);
+  // Clear all tokens on page load
+  useEffect(() => {
+    // Clear all provider tokens on new tab/page load
+    ['google', 'microsoft', 'slack'].forEach(provider => {
+      sessionStorage.removeItem(`${provider}_token`);
+      sessionStorage.removeItem(`${provider}_token_expiry`);
+    });
+    // Clear any existing timers
+    Object.values(tokenExpiryTimersRef.current).forEach(timer => clearTimeout(timer));
+    tokenExpiryTimersRef.current = {};
+    console.log('Cleared all tokens for new session');
+  }, []);
   const handleOpenRightSidebar = (content, chatBlockId = null) => {
     flushSync(() => {
       if (chatBlockId) {
     );
   }, []);
+  // Function to store token with expiry
+  const storeTokenWithExpiry = (provider, token) => {
+    const expiryTime = Date.now() + (60 * 60 * 1000); // 1 hour from now
+    sessionStorage.setItem(`${provider}_token`, token);
+    sessionStorage.setItem(`${provider}_token_expiry`, expiryTime.toString());
+    // Set up expiry timer
+    setupTokenExpiryTimer(provider, expiryTime);
+  };
+  // Function to check if token is valid
+  const isTokenValid = (provider) => {
+    const token = sessionStorage.getItem(`${provider}_token`);
+    const expiry = sessionStorage.getItem(`${provider}_token_expiry`);
+    if (!token || !expiry) return false;
+    const expiryTime = parseInt(expiry);
+    return Date.now() < expiryTime;
+  };
+  // Function to get valid token
+  const getValidToken = (provider) => {
+    if (isTokenValid(provider)) {
+      return sessionStorage.getItem(`${provider}_token`);
+    }
+    return null;
+  };
+  // Function to get provider icon
+  const getProviderIcon = useCallback((provider) => {
+    switch (provider.toLowerCase()) {
+      case 'google':
+        return <FaGoogle />;
+      case 'microsoft':
+        return <FaMicrosoft />;
+      case 'slack':
+        return <FaSlack />;
+      default:
+        return null;
+    }
+  }, []);
+  // Function to get provider color
+  const getProviderColor = useCallback((provider) => {
+    switch (provider.toLowerCase()) {
+      case 'google':
+        return '#4285F4';
+      case 'microsoft':
+        return '#00A4EF';
+      case 'slack':
+        return '#4A154B';
+      default:
+        return '#666';
+    }
+  }, []);
+  // Function to set up timer for token expiry notification
+  const setupTokenExpiryTimer = useCallback((provider, expiryTime) => {
+    // Clear existing timer if any
+    if (tokenExpiryTimersRef.current[provider]) {
+      clearTimeout(tokenExpiryTimersRef.current[provider]);
+    }
+    // Remove any existing notification for this provider
+    if (notificationIdsRef.current[provider]) {
+      removeNotification(notificationIdsRef.current[provider]);
+      delete notificationIdsRef.current[provider];
+    }
+    const timeUntilExpiry = expiryTime - Date.now();
+    if (timeUntilExpiry > 0) {
+      tokenExpiryTimersRef.current[provider] = setTimeout(() => {
+        const providerName = provider.charAt(0).toUpperCase() + provider.slice(1);
+        const providerColor = getProviderColor(provider);
+        // Add notification
+        const notificationId = addNotification({
+          type: 'warning',
+          title: `${providerName} Authentication Expired`,
+          message: `Your ${providerName} authentication has expired. Please reconnect to continue using ${providerName} services.`,
+          icon: getProviderIcon(provider),
+          dismissible: true,
+          autoDismiss: false,
+          actions: [
+            {
+              id: 'reconnect',
+              label: `Reconnect ${providerName}`,
+              style: {
+                background: providerColor,
+                color: 'white',
+                border: 'none'
+              },
+              data: { provider }
+            }
+          ],
+          style: {
+            borderLeftColor: providerColor
+          }
+        });
+        // Store notification ID
+        notificationIdsRef.current[provider] = notificationId;
+        // Clear token data
+        sessionStorage.removeItem(`${provider}_token`);
+        sessionStorage.removeItem(`${provider}_token_expiry`);
+        // Update selected services to reflect disconnection
+        if (provider === 'slack') {
+          setSelectedServices(prev => ({ ...prev, slack: false }));
+        } else {
+          setSelectedServices(prev => ({ ...prev, [provider]: [] }));
+        }
+      }, timeUntilExpiry);
+    }
+  }, [addNotification, getProviderColor, getProviderIcon, removeNotification, setSelectedServices]);
+  // Check existing tokens on component mount and set up timers
+  useEffect(() => {
+    ['google', 'microsoft', 'slack'].forEach(provider => {
+      const expiry = sessionStorage.getItem(`${provider}_token_expiry`);
+      if (expiry) {
+        const expiryTime = parseInt(expiry);
+        if (Date.now() < expiryTime) {
+          setupTokenExpiryTimer(provider, expiryTime);
+        } else {
+          // Token already expired, clear it
+          sessionStorage.removeItem(`${provider}_token`);
+          sessionStorage.removeItem(`${provider}_token_expiry`);
+        }
+      }
+    });
+    // Cleanup timers on unmount
+    return () => {
+      Object.values(tokenExpiryTimersRef.current).forEach(timer => clearTimeout(timer));
+    };
+  }, [setupTokenExpiryTimer]);
   // Initiate the SSE
   const initiateSSE = (query, blockId) => {
     const startTime = Date.now();
     });
     eventSource.addEventListener("final_message", (e) => {
+      console.log("[SSE final message]", e.data);
       const endTime = Date.now();
       const thinkingTime = ((endTime - startTime) / 1000).toFixed(1);
       // Only update thinkingTime so the streaming flag turns false and the cursor disappears
           : block
       ));
     });
+    // Listen for the "final_sources" event to update sources in AI answer of this chat block.
+    eventSource.addEventListener("final_sources", (e) => {
+      try {
+        const sources = JSON.parse(e.data);
+        console.log("Final sources received:", sources);
+        setChatBlocks(prev => prev.map(block =>
+          block.id === blockId ? { ...block, finalSources: sources } : block
+        ));
+      } catch (err) {
+        console.error("Error parsing final_sources event:", err);
+      }
+    });
     // Listen for the "complete" event to know when to close the connection.
     eventSource.addEventListener("complete", (e) => {
     if (searchText.trim()) handleSend();
   };
+  // Toggle the Add Content dropdown
+  const handleToggleAddContent = (event) => {
+    event.stopPropagation(); // Prevents the click from closing the menu immediately
+    // If we are about to close the dropdown, suppress the tooltip.
+    if (isAddContentOpen) {
+      setIsTooltipSuppressed(true);
+    }
+     setAddContentOpen(prev => !prev);
+  };
+  // Handle mouse enter on the Add Content button to suppress tooltip
+  const handleMouseLeaveAddBtn = () => {
+    setIsTooltipSuppressed(false);
+  };
+  // Close the Add Content dropdown
+  const closeAddContentDropdown = () => {
+    setAddContentOpen(false);
+  };
+  // Open the Add Files dialog
+  const handleOpenAddFilesDialog = () => {
+    setAddContentOpen(false); // Close the dropdown when opening the dialog
+    setIsAddFilesDialogOpen(true);
+  };
+  // Fetch excerpts for a specific block
+  const handleFetchExcerpts = useCallback(async (blockId) => {
+    let blockIndex = -1;
+    let currentBlock = null;
+    // Find the block to check its current state
+    setChatBlocks(prev => {
+        blockIndex = prev.findIndex(b => b.id === blockId);
+        if (blockIndex !== -1) {
+            currentBlock = prev[blockIndex];
+        }
+        // No state change here, just reading the state
+        return prev;
+    });
+    // Prevent fetching if already loaded or currently loading
+    if (blockIndex === -1 || !currentBlock || currentBlock.excerptsData || currentBlock.isLoadingExcerpts) return;
+    // Set loading state for the specific block
+    setChatBlocks(prev => prev.map(b =>
+      b.id === blockId ? { ...b, isLoadingExcerpts: true } : b
+    ));
+    try {
+      // Call the backend endpoint to get excerpts
+      const response = await fetch('/action/excerpts', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ blockId: blockId })
+      });
+      if (!response.ok) {
+        const errorData = await response.json();
+        throw new Error(errorData.detail || `HTTP error! status: ${response.status}`);
+      }
+      const data = await response.json();
+      console.log("Fetched excerpts data from backend:", data.result);
+      // Update the specific block with the fetched excerptsData
+      setChatBlocks(prev => prev.map(b =>
+        b.id === blockId
+          ? {
+              ...b,
+              excerptsData: data.result, // Store the fetched data
+              isLoadingExcerpts: false,    // Turn off loading
+            }
+          : b
+      ));
+      openSnackbar("Excerpts loaded successfully!", "success");
+    } catch (error) {
+      console.error("Error requesting excerpts:", error);
+      // Reset loading state on error
+      setChatBlocks(prev => prev.map(b =>
+        b.id === blockId ? { ...b, isLoadingExcerpts: false } : b
+      ));
+      openSnackbar(`Failed to load excerpts`, "error");
+    }
+  }, [openSnackbar]);
+  // Function to handle notification actions
+  const handleNotificationAction = (notificationId, actionId, actionData) => {
+    console.log('Notification action triggered:', { notificationId, actionId, actionData });
+    // Handle both 'reconnect' and 'connect' actions
+    if ((actionId === 'reconnect' || actionId === 'connect') && actionData?.provider) {
+      // Remove the notification
+      removeNotification(notificationId);
+      // Clean up stored notification ID if it exists
+      if (notificationIdsRef.current[actionData.provider] === notificationId) {
+        delete notificationIdsRef.current[actionData.provider];
+      }
+      // Trigger authentication
+      initiateOAuth(actionData.provider);
+    }
+  };
+  // Function to initiate OAuth
+  const initiateOAuth = (provider) => {
+    const authUrls = {
+      google: `https://accounts.google.com/o/oauth2/v2/auth?` +
+        `client_id=${process.env.REACT_APP_GOOGLE_CLIENT_ID}&` +
+        `response_type=token&` +
+        `scope=email profile https://www.googleapis.com/auth/drive.readonly https://www.googleapis.com/auth/gmail.readonly https://www.googleapis.com/auth/calendar.readonly https://www.googleapis.com/auth/tasks.readonly&` +
+        `redirect_uri=${window.location.origin}/auth-receiver.html&` +
+        `prompt=select_account`,
+      microsoft: `https://login.microsoftonline.com/common/oauth2/v2.0/authorize?` +
+        `client_id=${process.env.REACT_APP_MICROSOFT_CLIENT_ID}&` +
+        `response_type=token&` +
+        `scope=openid profile email Files.Read.All Mail.Read Calendars.Read Tasks.Read Notes.Read&` +
+        `redirect_uri=${window.location.origin}/auth-receiver.html&` +
+        `response_mode=fragment&` +
+        `prompt=select_account`,
+      slack: `https://slack.com/oauth/v2/authorize?` +
+        `client_id=${process.env.REACT_APP_SLACK_CLIENT_ID}&` +
+        `scope=channels:read,channels:history,files:read,groups:read,im:read,mpim:read,search:read,users:read&` +
+        `redirect_uri=${window.location.origin}/auth-receiver.html`
+    };
+    const authWindow = window.open(
+      authUrls[provider],
+      'Connect Account',
+      'width=600,height=700,left=200,top=100'
+    );
+    // Show connecting notification
+    const connectingNotificationId = addNotification({
+      type: 'info',
+      title: `Connecting to ${provider.charAt(0).toUpperCase() + provider.slice(1)}`,
+      message: 'Please complete the authentication in the popup window...',
+      icon: getProviderIcon(provider),
+      dismissible: false,
+      autoDismiss: false
+    });
+    // Set up message listener
+    const messageHandler = async (event) => {
+      if (event.origin !== window.location.origin) return;
+      if (event.data.type === 'auth-success') {
+        const { token } = event.data;
+        // Remove connecting notification
+        removeNotification(connectingNotificationId);
+        // Store token with expiry
+        storeTokenWithExpiry(provider, token);
+        // Send token to backend
+        try {
+          const response = await fetch('/api/session-token', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+              provider,
+              token
+            })
+          });
+          if (response.ok) {
+            // Show success notification
+            addNotification({
+              type: 'success',
+              title: 'Connected Successfully',
+              message: `Successfully connected to ${provider.charAt(0).toUpperCase() + provider.slice(1)}!`,
+              icon: getProviderIcon(provider),
+              autoDismiss: true,
+              duration: 3000,
+              showProgress: true
+            });
+          }
+        } catch (error) {
+          console.error(`Failed to connect to ${provider}:`, error);
+          addNotification({
+            type: 'error',
+            title: 'Connection Failed',
+            message: `Failed to connect to ${provider.charAt(0).toUpperCase() + provider.slice(1)}. Please try again.`,
+            autoDismiss: true,
+            duration: 5000
+          });
+        }
+        window.removeEventListener('message', messageHandler);
+      } else if (event.data.type === 'auth-failed') {
+        // Remove connecting notification
+        removeNotification(connectingNotificationId);
+        // Show error notification
+        addNotification({
+          type: 'error',
+          title: 'Authentication Failed',
+          message: `Failed to authenticate with ${provider.charAt(0).toUpperCase() + provider.slice(1)}. Please try again.`,
+          autoDismiss: true,
+          duration: 5000
+        });
+        window.removeEventListener('message', messageHandler);
+      }
+    };
+    window.addEventListener('message', messageHandler);
+    // Handle if user closes the popup without authenticating
+    const checkInterval = setInterval(() => {
+      if (authWindow.closed) {
+        clearInterval(checkInterval);
+        removeNotification(connectingNotificationId);
+        window.removeEventListener('message', messageHandler);
+      }
+    }, 1000);
+  };
+  // Handle service selection from dropdown
+  const handleServiceClick = useCallback((provider, service) => {
+    // Toggle selection
+    if (provider === 'slack') {
+      setSelectedServices(prev => ({ ...prev, slack: !prev.slack }));
+    } else {
+      setSelectedServices(prev => ({
+        ...prev,
+        [provider]: prev[provider].includes(service)
+          ? prev[provider].filter(s => s !== service)
+          : [...prev[provider], service]
+      }));
+    }
+    // Check if token is valid
+    if (!isTokenValid(provider)) {
+      // Show notification prompting to authenticate
+      const notificationId = addNotification({
+        type: 'info',
+        title: 'Authentication Required',
+        message: `Please connect your ${provider.charAt(0).toUpperCase() + provider.slice(1)} account to use this service.`,
+        icon: getProviderIcon(provider),
+        actions: [
+          {
+            id: 'connect',
+            label: `Connect ${provider.charAt(0).toUpperCase() + provider.slice(1)}`,
+            style: {
+              background: getProviderColor(provider),
+              color: 'white',
+              border: 'none'
+            },
+            data: { provider }
+          }
+        ],
+        autoDismiss: true,
+        duration: 5000,
+        showProgress: true
+      });
+    }
+  }, [addNotification, getProviderIcon, getProviderColor]);
   // Get the chat block whose details should be shown in the sidebar.
   const selectedBlock = chatBlocks.find(block => block.id === selectedChatBlockId);
   const evaluateAction = selectedBlock && selectedBlock.actions
           : 0,
       }}
     >
+      <Notification
+        notifications={notifications}
+        position="top-right"
+        animation="slide"
+        stackDirection="down"
+        maxNotifications={5}
+        spacing={12}
+        offset={{ x: 20, y: 20 }}
+        onDismiss={removeNotification}
+        onAction={handleNotificationAction}
+        theme="light"
+      />
       {showChatWindow && selectedBlock && (sidebarContent !== "default" || (selectedBlock.tasks && selectedBlock.tasks.length > 0) || (selectedBlock.sources && selectedBlock.sources.length > 0)) && (
         <div className="floating-sidebar">
           <RightSidebar
                   thinkingTime={block.thinkingTime}
                   thoughtLabel={block.thoughtLabel}
                   sourcesRead={block.sourcesRead}
+                  finalSources={block.finalSources}
+                  excerptsData={block.excerptsData}
+                  isLoadingExcerpts={block.isLoadingExcerpts}
+                  onFetchExcerpts={handleFetchExcerpts}
                   actions={block.actions}
                   tasks={block.tasks}
                   openRightSidebar={handleOpenRightSidebar}
                 />
               </div>
               <div className="chat-icon-container">
+                <div className="chat-left-icons">
+                  <div className="tooltip-wrapper">
+                    <button
+                      className="chat-settings-btn"
+                      onClick={() => setShowSettingsModal(true)}
+                    >
+                      <FaCog />
+                    </button>
+                    <span className="tooltip">Settings</span>
+                  </div>
+                  <div
+                    className="tooltip-wrapper"
+                    onMouseLeave={handleMouseLeaveAddBtn}
+                  >
+                    <button className="chat-add-btn" onClick={handleToggleAddContent} ref={chatAddBtnRef}>
+                      <FaPlus />
+                    </button>
+                    <span className={`tooltip ${isAddContentOpen || isTooltipSuppressed ? 'hidden' : ''}`}>Add Content</span>
+                    <AddContentDropdown
+                      isOpen={isAddContentOpen}
+                      onClose={closeAddContentDropdown}
+                      toggleButtonRef={chatAddBtnRef}
+                      onAddFilesClick={handleOpenAddFilesDialog}
+                      onServiceClick={handleServiceClick}
+                      selectedServices={selectedServices}
+                    />
+                  </div>
+                </div>
                 {/* Conditionally render Stop or Send button */}
+                <div className="tooltip-wrapper">
+                  <button
+                    className={`chat-send-btn ${isProcessing ? 'stop-btn' : ''}`}
+                    onClick={isProcessing ? handleStop : handleSendButtonClick}
+                  >
+                    {isProcessing ? <FaStop size={12} color="black" /> : <FaPaperPlane />}
+                  </button>
+                  <span className="tooltip">{isProcessing ? 'Stop' : 'Send'}</span>
+                </div>
               </div>
             </div>
           </>
                 />
               </div>
               <div className="icon-container">
+                <div className="left-icons">
+                  <div className="tooltip-wrapper">
+                    <button
+                      className="settings-btn"
+                      onClick={() => setShowSettingsModal(true)}
+                    >
+                      <FaCog />
+                    </button>
+                    <span className="tooltip">Settings</span>
+                  </div>
+                  <div
+                    className="tooltip-wrapper"
+                    onMouseLeave={handleMouseLeaveAddBtn}
+                  >
+                    <button className="add-btn" onClick={handleToggleAddContent} ref={addBtnRef}>
+                      <FaPlus />
+                    </button>
+                    <span className={`tooltip ${isAddContentOpen || isTooltipSuppressed ? 'hidden' : ''}`}>Add Content</span>
+                    <AddContentDropdown
+                      isOpen={isAddContentOpen}
+                      onClose={closeAddContentDropdown}
+                      toggleButtonRef={addBtnRef}
+                      onAddFilesClick={handleOpenAddFilesDialog}
+                      onServiceClick={handleServiceClick}
+                      selectedServices={selectedServices}
+                    />
+                  </div>
+                </div>
+                <div className="tooltip-wrapper">
+                  <button
+                    className={`send-btn ${isProcessing ? 'stop-btn' : ''}`}
+                    onClick={isProcessing ? handleStop : handleSendButtonClick}
+                  >
+                    {isProcessing ? <FaStop /> : <FaPaperPlane />}
+                  </button>
+                  <span className="tooltip">{isProcessing ? 'Stop' : 'Send'}</span>
+                </div>
               </div>
             </div>
           </div>
         )}
       </main>
       {showSettingsModal && (
         <IntialSetting
           trigger={true}
           closeSnackbar={closeSnackbar}
         />
       )}
+      {isAddFilesDialogOpen && (
+        <AddFilesDialog
+          isOpen={isAddFilesDialogOpen}
+          onClose={() => setIsAddFilesDialogOpen(false)}
+          openSnackbar={openSnackbar}
+          setSessionContent={setSessionContent}
+        />
+      )}
       <Snackbar
         open={snackbar.open}
+        autoHideDuration={snackbar.duration}
         onClose={closeSnackbar}
         anchorOrigin={{ vertical: 'top', horizontal: 'center' }}
       >

frontend/src/Components/IntialSetting.css CHANGED Viewed

@@ -171,4 +171,4 @@ input, select, textarea {
       width: 90%;
       max-height: 75vh; /* Adjust height for smaller screens */
   }
-}

       width: 90%;
       max-height: 75vh; /* Adjust height for smaller screens */
   }
+}

frontend/src/Components/IntialSetting.js CHANGED Viewed

@@ -22,30 +22,37 @@ function IntialSetting(props) {
   // Model options for different providers
   const modelOptions = {
     OpenAI: {
-      "GPT-4 Turbo": "gpt-4-turbo",
-      "GPT-4o": "gpt-4o",
-      "GPT-4o Latest": "gpt-4o-2024-11-20",
-      "GPT-4o Mini": "gpt-4o-mini",
-      "ChatGPT": "chatgpt-4o-latest",
     },
     Anthropic: {
-      "Claude 3.5 Sonnet": "claude-3-5-sonnet-20241022",
-      "Claude 3.5 Haiku": "claude-3-5-haiku-20241022",
-      "Claude 3 Opus": "claude-3-opus-20240229",
-      "Claude 3 Sonnet": "claude-3-sonnet-20240229",
-      "Claude 3 Haiku": "claude-3-haiku-20240307",
     },
     Google: {
-      "Gemini 1.5 Pro": "gemini-1.5-pro",
-      "Gemini 1.5 Flash": "gemini-1.5-flash",
-      "Gemini 2.0 Flash Lite": "gemini-2.0-flash-lite-preview-02-05",
-      "Gemini 2.0 Flash Experimental": "gemini-2.0-flash-exp",
       "Gemini 2.0 Flash": "gemini-2.0-flash",
-      "Gemini 2.0 Pro Experimental": "gemini-2.0-pro-exp-02-05",
     },
     XAI: {
-      "Grok-2": "grok-2-latest",
-      "Grok Beta": "grok-beta",
     },
   };

   // Model options for different providers
   const modelOptions = {
     OpenAI: {
+      "GPT 4o": "gpt-4o",
+      "GPT 4o Latest": "gpt-4o-2024-11-20",
+      "GPT 4o Mini": "gpt-4o-mini",
+      "GPT 4.1": "gpt-4.1",
+      "GPT 4.1 Mini": "gpt-4.1-mini",
+      "GPT 4.1 Nano": "gpt-4.1-nano",
+      "ChatGPT": "chatgpt-4o-latest"
     },
     Anthropic: {
+      "Claude 4 Opus": "claude-opus-4-20250514",
+      "Claude Sonnet 4": "claude-sonnet-4-20250514",
+      "Claude Sonnet 3.7": "claude-3-7-sonnet-20250219",
+      "Claude Sonnet 3.5": "claude-3-5-sonnet-20241022",
+      "Claude Haiku 3.5": "claude-3-5-haiku-20241022",
+      "Claude Opus 3": "claude-3-opus-20240229",
+      "Claude Sonnet 3": "claude-3-sonnet-20240229",
+      "Claude Haiku 3": "claude-3-haiku-20240307"
     },
     Google: {
+      "Gemini 2.0 Flash Lite": "gemini-2.0-flash-lite",
       "Gemini 2.0 Flash": "gemini-2.0-flash",
+      "Gemini 2.5 Flash Lite": "gemini-2.5-flash-lite-preview-06-17",
+      "Gemini 2.5 Flash": "gemini-2.5-flash-preview-04-17",
+      "Gemini 2.5 Pro": "gemini-2.5-pro"
     },
     XAI: {
+      "Grok 2": "grok-2",
+      "Grok 3 Mini": "grok-3-mini-latest",
+      "Grok 3 Mini (Fast)": "grok-3-mini-fast-latest",
+      "Grok 3": "grok-3-latest",
+      "Grok 3 (Fast)": "grok-3-fast-latest"
     },
   };

frontend/src/Icons/excerpts.png ADDED Viewed

frontend/src/Icons/excerpts.pngZone.Identifier ADDED Viewed

	@@ -0,0 +1,4 @@

+[ZoneTransfer]
+ZoneId=3
+ReferrerUrl=https://www.flaticon.com/search?word=quotes
+HostUrl=about:internet

main.py CHANGED Viewed

@@ -1,12 +1,14 @@
 import os
 import re
-import asyncio
 import json
 import time
 import logging
-from typing import Any, Dict
 from fastapi.staticfiles import StaticFiles
-from fastapi import FastAPI, Request, HTTPException
 from fastapi.responses import StreamingResponse, JSONResponse, FileResponse
 from fastapi.middleware.cors import CORSMiddleware
 from dotenv import load_dotenv
@@ -14,17 +16,22 @@ from tenacity import RetryError
 from openai import RateLimitError
 from anthropic import RateLimitError as AnthropicRateLimitError
 from google.api_core.exceptions import ResourceExhausted
 logger = logging.getLogger()
 logger.setLevel(logging.INFO)
 CONTEXT_LENGTH = 128000
 BUFFER = 10000
 MAX_TOKENS_ALLOWED = CONTEXT_LENGTH - BUFFER
-# Path to the .env file
-ENV_FILE_PATH = os.getenv("WRITABLE_DIR", "/tmp") + "/.env"
 # Per-session state
 SESSION_STORE: Dict[str, Dict[str, Any]] = {}
@@ -45,9 +52,22 @@ def stop_on_error():
         state["process_task"].cancel()
         del state["process_task"]
 # Initialize the components
-def initialize_components():
-    load_dotenv(ENV_FILE_PATH, override=True)
     from src.search.search_engine import SearchEngine
     from src.query_processing.query_processor import QueryProcessor
@@ -58,40 +78,206 @@ def initialize_components():
     from src.crawl.crawler import CustomCrawler
     from src.utils.api_key_manager import APIKeyManager
     from src.query_processing.late_chunking.late_chunker import LateChunker
     manager = APIKeyManager()
     manager._reinit()
-    SESSION_STORE['search_engine'] = SearchEngine()
-    SESSION_STORE['query_processor'] = QueryProcessor()
-    SESSION_STORE['crawler'] = CustomCrawler(max_concurrent_requests=1000)
-    # SESSION_STORE['graph_rag'] = Neo4jGraphRAG(num_workers=os.cpu_count() * 2)
-    SESSION_STORE['graph_rag'] = GraphRAG(num_workers=os.cpu_count() * 2)
-    SESSION_STORE['evaluator'] = Evaluator()
-    SESSION_STORE['reasoner'] = Reasoner()
-    SESSION_STORE['model'] = manager.get_llm()
-    SESSION_STORE['late_chunker'] = LateChunker()
-    SESSION_STORE["initialized"] = True
-    SESSION_STORE["session_id"] = None
 async def process_query(user_query: str, sse_queue: asyncio.Queue):
     state = SESSION_STORE
     try:
         category = await state["query_processor"].classify_query(user_query)
         cat_lower = category.lower().strip()
-        if state["session_id"] is None:
-            state["session_id"] = await state["crawler"].create_session()
         user_query = re.sub(r'category:.*', '', user_query, flags=re.IGNORECASE).strip()
         if cat_lower == "basic":
             response = ""
             chunk_counter = 1
-            async for chunk in state["reasoner"].reason(user_query):
-                await sse_queue.put(("token", json.dumps({"chunk": chunk, "index": chunk_counter})))
-                response += chunk
-                chunk_counter += 1
             await sse_queue.put(("final_message", response))
             SESSION_STORE["chat_history"].append({"query": user_query, "response": response})
@@ -121,13 +307,16 @@ async def process_query(user_query: str, sse_queue: asyncio.Queue):
                 max_attempts=1
             )
-            contents = ""
             if search_contents:
                 for k, content in enumerate(search_contents, 1):
                     if isinstance(content, Exception):
                         print(f"Error fetching content: {content}")
                     elif content:
-                        contents += f"Document {k}:\n{content}\n\n"
             if len(contents.strip()) > 0:
                 await sse_queue.put(("step", "Generating Response..."))
@@ -140,13 +329,27 @@ async def process_query(user_query: str, sse_queue: asyncio.Queue):
                 response = ""
                 chunk_counter = 1
-                async for chunk in state["reasoner"].reason(user_query, contents):
                     await sse_queue.put(("token", json.dumps({"chunk": chunk, "index": chunk_counter})))
                     response += chunk
                     chunk_counter += 1
                 await sse_queue.put(("final_message", response))
                 SESSION_STORE["chat_history"].append({"query": user_query, "response": response})
                 await sse_queue.put(("action", {
                     "name": "sources",
@@ -189,7 +392,11 @@ async def process_query(user_query: str, sse_queue: asyncio.Queue):
                     )
                     current_search_results.extend(filtered_urls)
-                    urls = [r.get('link', 'No URL') for r in filtered_urls]
                     search_contents = await state['crawler'].fetch_page_contents(
                         urls,
                         sub_query,
@@ -198,13 +405,13 @@ async def process_query(user_query: str, sse_queue: asyncio.Queue):
                     )
                     current_search_contents.extend(search_contents)
-                    contents = ""
                     if search_contents:
                         for k, c in enumerate(search_contents, 1):
                             if isinstance(c, Exception):
                                 logger.info(f"Error fetching content: {c}")
                             elif c:
-                                contents += f"Document {k}:\n{c}\n\n"
                     if len(contents.strip()) > 0:
                         await sse_queue.put(("task", (sub_query, "DONE")))
@@ -225,7 +432,11 @@ async def process_query(user_query: str, sse_queue: asyncio.Queue):
             results = await asyncio.gather(*tasks)
             end = time.time()
-            contents = "\n\n".join(r for r in results if r.strip())
             unique_results = []
             seen = set()
@@ -255,7 +466,7 @@ async def process_query(user_query: str, sse_queue: asyncio.Queue):
                 response = ""
                 chunk_counter = 1
                 is_first_chunk = True
-                async for chunk in state['reasoner'].reason(user_query, contents):
                     if is_first_chunk:
                         await sse_queue.put(("step", f"Thought and searched for {int(end - start)} seconds"))
                         is_first_chunk = False
@@ -264,8 +475,21 @@ async def process_query(user_query: str, sse_queue: asyncio.Queue):
                     response += chunk
                     chunk_counter += 1
                 await sse_queue.put(("final_message", response))
                 SESSION_STORE["chat_history"].append({"query": user_query, "response": response})
                 await sse_queue.put(("action", {
                     "name": "sources",
@@ -327,7 +551,7 @@ async def process_query(user_query: str, sse_queue: asyncio.Queue):
                                 if isinstance(c, Exception):
                                     logger.info(f"Error fetching content: {c}")
                                 elif c:
-                                    contents += f"Document {k}:\n{c}\n\n"
                         return contents
@@ -361,7 +585,11 @@ async def process_query(user_query: str, sse_queue: asyncio.Queue):
             results = await asyncio.gather(*tasks)
             end = time.time()
             previous_contents = []
             for result in results:
                 if result:
                     for content in result:
@@ -397,7 +625,7 @@ async def process_query(user_query: str, sse_queue: asyncio.Queue):
                 response = ""
                 chunk_counter = 1
                 is_first_chunk = True
-                async for chunk in state['reasoner'].reason(user_query, contents):
                     if is_first_chunk:
                         await sse_queue.put(("step", f"Thought and searched for {int(end - start)} seconds"))
                         is_first_chunk = False
@@ -406,8 +634,21 @@ async def process_query(user_query: str, sse_queue: asyncio.Queue):
                     response += chunk
                     chunk_counter += 1
                 await sse_queue.put(("final_message", response))
                 SESSION_STORE["chat_history"].append({"query": user_query, "response": response})
                 await sse_queue.put(("action", {
                     "name": "sources",
@@ -508,6 +749,11 @@ async def process_query(user_query: str, sse_queue: asyncio.Queue):
             answer = state['graph_rag'].query_graph(user_query)
             if answer:
                 token_count = state['model'].get_num_tokens(answer)
                 if token_count > MAX_TOKENS_ALLOWED:
                     answer = await state['late_chunker'].chunker(
@@ -523,7 +769,7 @@ async def process_query(user_query: str, sse_queue: asyncio.Queue):
                 response = ""
                 chunk_counter = 1
                 is_first_chunk = True
-                async for chunk in state['reasoner'].reason(user_query, answer):
                     if is_first_chunk:
                         await sse_queue.put(("step", f"Thought and searched for {int(end - start)} seconds"))
                         is_first_chunk = False
@@ -532,8 +778,21 @@ async def process_query(user_query: str, sse_queue: asyncio.Queue):
                     response += chunk
                     chunk_counter += 1
                 await sse_queue.put(("final_message", response))
                 SESSION_STORE["chat_history"].append({"query": user_query, "response": response})
                 await sse_queue.put(("action", {
                     "name": "sources",
@@ -557,6 +816,7 @@ async def process_query(user_query: str, sse_queue: asyncio.Queue):
     except Exception as e:
         await sse_queue.put(("error", str(e)))
         stop()
 # Create a FastAPI app
@@ -616,11 +876,12 @@ def action_sources(payload: Dict[str, Any]) -> Dict[str, Any]:
         return {"result": sources}
     except Exception as e:
         return JSONResponse(content={"error": str(e)}, status_code=500)
 # Define the route for graph action to display the graph
 @app.post("/action/graph")
 def action_graph() -> Dict[str, Any]:
     state = SESSION_STORE
     try:
         html_str = state['graph_rag'].display_graph()
@@ -644,7 +905,56 @@ async def action_evaluate(payload: Dict[str, Any]) -> Dict[str, Any]:
         return {"result": result}
     except Exception as e:
         return JSONResponse(content={"error": str(e)}, status_code=500)
 @app.post("/settings")
 async def update_settings(data: Dict[str, Any]):
     from src.helpers.helper import (
@@ -658,9 +968,6 @@ async def update_settings(data: Dict[str, Any]):
     multiple_api_keys = data.get("Model_API_Keys", "").strip()
     brave_api_key = data.get("Brave_Search_API_Key", "").strip()
     proxy_list = data.get("Proxy_List", "").strip()
-    # neo4j_url = data.get("Neo4j_URL", "").strip()
-    # neo4j_username = data.get("Neo4j_Username", "").strip()
-    # neo4j_password = data.get("Neo4j_Password", "").strip()
     model_temperature = str(data.get("Model_Temperature", 0.0))
     model_top_p = str(data.get("Model_Top_P", 1.0))
@@ -674,9 +981,6 @@ async def update_settings(data: Dict[str, Any]):
         env_updates.update(px)
     env_updates["BRAVE_API_KEY"] = brave_api_key
-    # env_updates["NEO4J_URI"] = neo4j_url
-    # env_updates["NEO4J_USER"] = neo4j_username
-    # env_updates["NEO4J_PASSWORD"] = neo4j_password
     env_updates["MODEL_PROVIDER"] = prov_lower
     env_updates["MODEL_NAME"] = model_name
     env_updates["MODEL_TEMPERATURE"] = model_temperature
@@ -684,21 +988,136 @@ async def update_settings(data: Dict[str, Any]):
     update_env_vars(env_updates)
     load_dotenv(override=True)
-    initialize_components()
     return {"success": True}
 @app.on_event("startup")
 def init_chat():
     if not SESSION_STORE:
         print("Initializing chat...")
         SESSION_STORE["settings_saved"] = False
         SESSION_STORE["session_id"] = None
         SESSION_STORE["chat_history"] = []
         print("Chat initialized!")
         return {"sucess": True}
     else:
         print("Chat already initialized!")

 import os
 import re
 import json
 import time
+import shutil
+import asyncio
 import logging
+import traceback
+from typing import List, Dict, Any, Optional
 from fastapi.staticfiles import StaticFiles
+from fastapi import FastAPI, Request, HTTPException, UploadFile, File, Form
 from fastapi.responses import StreamingResponse, JSONResponse, FileResponse
 from fastapi.middleware.cors import CORSMiddleware
 from dotenv import load_dotenv
 from openai import RateLimitError
 from anthropic import RateLimitError as AnthropicRateLimitError
 from google.api_core.exceptions import ResourceExhausted
+from src.helpers.helper import get_folder_size, clear_folder
 logger = logging.getLogger()
 logger.setLevel(logging.INFO)
+# Path to the .env file
+ENV_FILE_PATH = os.getenv("WRITABLE_DIR", "/tmp") + "/.env"
+# Define the upload directory and maximum folder size
+UPLOAD_DIRECTORY = os.getenv("WRITABLE_DIR", "/tmp") + "/uploads"
+MAX_FOLDER_SIZE = 10 * 1024 * 1024  # 10 MB in bytes
 CONTEXT_LENGTH = 128000
 BUFFER = 10000
 MAX_TOKENS_ALLOWED = CONTEXT_LENGTH - BUFFER
 # Per-session state
 SESSION_STORE: Dict[str, Dict[str, Any]] = {}
         state["process_task"].cancel()
         del state["process_task"]
+# Get OAuth tokens for MCP tools
+def get_oauth_token(provider: str) -> Optional[str]:
+    if "oauth_tokens" in SESSION_STORE and provider in SESSION_STORE["oauth_tokens"]:
+        token_data = SESSION_STORE["oauth_tokens"][provider]
+        # Check if token is expired (1 hour)
+        if time.time() - token_data["timestamp"] < 3600:
+            return token_data["token"]
+        else:
+            # Token expired, remove it
+            del SESSION_STORE["oauth_tokens"][provider]
+            logger.info(f"{provider} token expired and removed")
+    return None
 # Initialize the components
+async def initialize_components():
+    load_dotenv(override=True)
     from src.search.search_engine import SearchEngine
     from src.query_processing.query_processor import QueryProcessor
     from src.crawl.crawler import CustomCrawler
     from src.utils.api_key_manager import APIKeyManager
     from src.query_processing.late_chunking.late_chunker import LateChunker
+    from src.integrations.mcp_client import MCPClient
+    state = SESSION_STORE
     manager = APIKeyManager()
     manager._reinit()
+    state['search_engine'] = SearchEngine()
+    state['query_processor'] = QueryProcessor()
+    state['crawler'] = CustomCrawler(max_concurrent_requests=1000)
+    # state['graph_rag'] = Neo4jGraphRAG(num_workers=os.cpu_count() * 2)
+    state['graph_rag'] = GraphRAG(num_workers=os.cpu_count() * 2)
+    state['evaluator'] = Evaluator()
+    state['reasoner'] = Reasoner()
+    state['model'] = manager.get_llm()
+    state['late_chunker'] = LateChunker()
+    state["mcp_client"] = MCPClient()
+    state["initialized"] = True
+    state["session_id"] = await state["crawler"].create_session()
+# Main function to process user queries
 async def process_query(user_query: str, sse_queue: asyncio.Queue):
     state = SESSION_STORE
     try:
+        # --- Categorize the query ---
         category = await state["query_processor"].classify_query(user_query)
         cat_lower = category.lower().strip()
         user_query = re.sub(r'category:.*', '', user_query, flags=re.IGNORECASE).strip()
+        # --- Read and extract user-provided files and links ---
+        # Initialize caches if not present
+        if "user_files_cache" not in state:
+            state["user_files_cache"] = {}
+        if "user_links_cache" not in state:
+            state["user_links_cache"] = {}
+        # Extract user-provided context
+        user_context = ""
+        user_links = state.get("user_provided_links", [])
+        # Read new uploaded files
+        if state["session_id"]:
+            session_upload_path = os.path.join(UPLOAD_DIRECTORY, state["session_id"])
+            if os.path.exists(session_upload_path):
+                for filename in os.listdir(session_upload_path):
+                    file_path = os.path.join(session_upload_path, filename)
+                    if os.path.isfile(file_path):
+                        # Check if file is already in cache
+                        if filename not in state["user_files_cache"]:
+                            try:
+                                await sse_queue.put(("step", "Reading User-Provided Files..."))
+                                with open(file_path, 'r', encoding='utf-8') as f:
+                                    file_content = f.read()
+                                    state["user_files_cache"][filename] = file_content
+                            except Exception as e:
+                                logger.error(f"Error reading file {filename}: {e}")
+                                # Try reading as binary and decode
+                                try:
+                                    with open(file_path, 'rb') as f:
+                                        file_content = f.read().decode('utf-8', errors='ignore')
+                                        state["user_files_cache"][filename] = file_content
+                                except Exception as e2:
+                                    logger.error(f"Error reading file {filename} as binary: {e2}")
+                                    state["user_files_cache"][filename] = ""  # Cache empty to avoid retrying
+        # Add all cached file contents
+        for filename, content in state["user_files_cache"].items():
+            if content:
+                user_context += f"\n[USER PROVIDED FILE: {filename} START]\n{content}\n[USER PROVIDED FILE: {filename} END]\n\n"
+        # Crawl new user-provided links
+        if user_links:
+            await sse_queue.put(("step", "Crawling User-Provided Links..."))
+            new_links = [link for link in user_links if link not in state["user_links_cache"]]
+            if new_links:
+                # Only crawl new links
+                link_contents = await state['crawler'].fetch_page_contents(
+                    new_links,
+                    user_query,
+                    state["session_id"],
+                    max_attempts=1
+                )
+                # Cache the new contents
+                for link, content in zip(new_links, link_contents):
+                    if not isinstance(content, Exception) and content:
+                        state["user_links_cache"][link] = content
+                    else:
+                        state["user_links_cache"][link] = ""  # Cache empty to avoid retrying
+        # Add all cached link contents
+        for link, content in state["user_links_cache"].items():
+            if content:
+                idx = user_links.index(link) + 1 if link in user_links else 0
+                user_context += f"\n[USER PROVIDED LINK {idx} START]\n{content}\n[USER PROVIDED LINK {idx} END]\n\n"
+        # --- Fetch apps data from MCP service ---
+        app_context = ""
+        selected_services = state.get("selected_services", {})
+        # Check if any services are selected
+        has_google = selected_services.get("google", [])
+        has_microsoft = selected_services.get("microsoft", [])
+        has_slack = selected_services.get("slack", False)
+        if has_google or has_microsoft or has_slack:
+            await sse_queue.put(("step", "Fetching Data From Connected Apps..."))
+            # Fetch from each provider in parallel
+            tasks = []
+            # Google services
+            if has_google and len(has_google) > 0:
+                google_token = get_oauth_token("google")
+                tasks.append(
+                    state['mcp_client'].fetch_app_data(
+                        provider="google",
+                        services=has_google,
+                        query=user_query,
+                        user_id=state["session_id"],
+                        access_token=google_token
+                    )
+                )
+            # Microsoft services
+            if has_microsoft and len(has_microsoft) > 0:
+                microsoft_token = get_oauth_token("microsoft")
+                tasks.append(
+                    state['mcp_client'].fetch_app_data(
+                        provider="microsoft",
+                        services=has_microsoft,
+                        query=user_query,
+                        user_id=state["session_id"],
+                        access_token=microsoft_token
+                    )
+                )
+            # Slack
+            if has_slack:
+                slack_token = get_oauth_token("slack")
+                tasks.append(
+                    state['mcp_client'].fetch_app_data(
+                        provider="slack",
+                        services=["messages"],  # Slack doesn't have sub-services
+                        query=user_query,
+                        user_id=state["session_id"],
+                        access_token=slack_token
+                    )
+                )
+            # Execute all requests in parallel
+            if tasks:
+                results = await asyncio.gather(*tasks, return_exceptions=True)
+                # Process results
+                for i, result in enumerate(results):
+                    if isinstance(result, Exception):
+                        logger.error(f"Error fetching app data: {result}")
+                    elif isinstance(result, dict):
+                        # Determine which provider this result is from
+                        if i == 0 and has_google:
+                            provider = "google"
+                        elif (i == 1 and has_microsoft) or (i == 0 and not has_google and has_microsoft):
+                            provider = "microsoft"
+                        else:
+                            provider = "slack"
+                        # Format the data
+                        formatted_context = state['mcp_client'].format_as_context(provider, result)
+                        if formatted_context:
+                            app_context += formatted_context
+                # Log how much app data we got
+                if app_context:
+                    logger.info(f"Retrieved app data: {len(app_context)} characters")
+        # Prepend app context to user context
+        if app_context:
+            user_context = app_context + "\n\n" + user_context
+        # Upgrade basic to advanced if user has provided links
+        if cat_lower == "basic" and user_links:
+            cat_lower = "advanced"
+        # --- Process the query based on the category ---
         if cat_lower == "basic":
             response = ""
             chunk_counter = 1
+            if user_context: # Include user context if available
+                await sse_queue.put(("step", "Generating Response..."))
+                async for chunk in state["reasoner"].answer(user_query, user_context, query_type="basic"):
+                    await sse_queue.put(("token", json.dumps({"chunk": chunk, "index": chunk_counter})))
+                    response += chunk
+                    chunk_counter += 1
+            else: # No user context provided
+                async for chunk in state["reasoner"].answer(user_query):
+                    await sse_queue.put(("token", json.dumps({"chunk": chunk, "index": chunk_counter})))
+                    response += chunk
+                    chunk_counter += 1
             await sse_queue.put(("final_message", response))
             SESSION_STORE["chat_history"].append({"query": user_query, "response": response})
                 max_attempts=1
             )
+            # Start with user-provided context
+            contents = user_context
+            # Add crawled contents
             if search_contents:
                 for k, content in enumerate(search_contents, 1):
                     if isinstance(content, Exception):
                         print(f"Error fetching content: {content}")
                     elif content:
+                        contents += f"[SOURCE {k} START]\n{content}\n[SOURCE {k} END]\n\n"
             if len(contents.strip()) > 0:
                 await sse_queue.put(("step", "Generating Response..."))
                 response = ""
                 chunk_counter = 1
+                async for chunk in state["reasoner"].answer(user_query, contents):
                     await sse_queue.put(("token", json.dumps({"chunk": chunk, "index": chunk_counter})))
                     response += chunk
                     chunk_counter += 1
+                sources_for_answer = []
+                for idx, result in enumerate(search_results, 1):
+                    if search_contents[idx-1]:  # Only include if content was successfully fetched
+                        sources_for_answer.append({
+                            "id": idx,
+                            "title": result.get('title', 'No Title'),
+                            "link": result.get('link', 'No URL')
+                        }
+                    )
                 await sse_queue.put(("final_message", response))
+                await sse_queue.put(("final_sources", json.dumps(sources_for_answer)))
                 SESSION_STORE["chat_history"].append({"query": user_query, "response": response})
+                SESSION_STORE["answer"] = response
+                SESSION_STORE["source_contents"] = contents
                 await sse_queue.put(("action", {
                     "name": "sources",
                     )
                     current_search_results.extend(filtered_urls)
+                    # Combine search results with user-provided links
+                    all_search_results = search_results + \
+                    [{"link": url, "title": f"User provided: {url}", "snippet": ""} for url in user_links]
+                    urls = [r.get('link', 'No URL') for r in all_search_results]
                     search_contents = await state['crawler'].fetch_page_contents(
                         urls,
                         sub_query,
                     )
                     current_search_contents.extend(search_contents)
+                    contents = user_context
                     if search_contents:
                         for k, c in enumerate(search_contents, 1):
                             if isinstance(c, Exception):
                                 logger.info(f"Error fetching content: {c}")
                             elif c:
+                                contents += f"[SOURCE {k} START]\n{c}\n[SOURCE {k} END]\n\n"
                     if len(contents.strip()) > 0:
                         await sse_queue.put(("task", (sub_query, "DONE")))
             results = await asyncio.gather(*tasks)
             end = time.time()
+            # Start with user-provided context
+            contents = user_context
+            # Add searched contents
+            contents += "\n\n".join(r for r in results if r.strip())
             unique_results = []
             seen = set()
                 response = ""
                 chunk_counter = 1
                 is_first_chunk = True
+                async for chunk in state['reasoner'].answer(user_query, contents):
                     if is_first_chunk:
                         await sse_queue.put(("step", f"Thought and searched for {int(end - start)} seconds"))
                         is_first_chunk = False
                     response += chunk
                     chunk_counter += 1
+                sources_for_answer = []
+                for idx, (result, content) in enumerate(zip(current_search_results, current_search_contents), 1):
+                    if content:  # Only include if content was successfully fetched
+                        sources_for_answer.append({
+                            "id": idx,
+                            "title": result.get('title', 'No Title'),
+                            "link": result.get('link', 'No URL')
+                        })
                 await sse_queue.put(("final_message", response))
+                await sse_queue.put(("final_sources", json.dumps(sources_for_answer)))
                 SESSION_STORE["chat_history"].append({"query": user_query, "response": response})
+                SESSION_STORE["answer"] = response
+                SESSION_STORE["source_contents"] = contents
                 await sse_queue.put(("action", {
                     "name": "sources",
                                 if isinstance(c, Exception):
                                     logger.info(f"Error fetching content: {c}")
                                 elif c:
+                                    contents += f"[SOURCE {k} START]\n{c}\n[SOURCE {k} END]\n\n"
                         return contents
             results = await asyncio.gather(*tasks)
             end = time.time()
+            # Start with user-provided context
             previous_contents = []
+            if user_context:
+                previous_contents.append(user_context)
             for result in results:
                 if result:
                     for content in result:
                 response = ""
                 chunk_counter = 1
                 is_first_chunk = True
+                async for chunk in state['reasoner'].answer(user_query, contents):
                     if is_first_chunk:
                         await sse_queue.put(("step", f"Thought and searched for {int(end - start)} seconds"))
                         is_first_chunk = False
                     response += chunk
                     chunk_counter += 1
+                sources_for_answer = []
+                for idx, (result, content) in enumerate(zip(current_search_results, current_search_contents), 1):
+                    if content:  # Only include if content was successfully fetched
+                        sources_for_answer.append({
+                            "id": idx,
+                            "title": result.get('title', 'No Title'),
+                            "link": result.get('link', 'No URL')
+                        })
                 await sse_queue.put(("final_message", response))
+                await sse_queue.put(("final_sources", json.dumps(sources_for_answer)))
                 SESSION_STORE["chat_history"].append({"query": user_query, "response": response})
+                SESSION_STORE["answer"] = response
+                SESSION_STORE["source_contents"] = contents
                 await sse_queue.put(("action", {
                     "name": "sources",
             answer = state['graph_rag'].query_graph(user_query)
             if answer:
+                # Start with user-provided context
+                previous_contents = []
+                if user_context:
+                    previous_contents.append(user_context)
                 token_count = state['model'].get_num_tokens(answer)
                 if token_count > MAX_TOKENS_ALLOWED:
                     answer = await state['late_chunker'].chunker(
                 response = ""
                 chunk_counter = 1
                 is_first_chunk = True
+                async for chunk in state['reasoner'].answer(user_query, answer):
                     if is_first_chunk:
                         await sse_queue.put(("step", f"Thought and searched for {int(end - start)} seconds"))
                         is_first_chunk = False
                     response += chunk
                     chunk_counter += 1
+                sources_for_answer = []
+                for idx, (result, content) in enumerate(zip(current_search_results, current_search_contents), 1):
+                    if content:  # Only include if content was successfully fetched
+                        sources_for_answer.append({
+                            "id": idx,
+                            "title": result.get('title', 'No Title'),
+                            "link": result.get('link', 'No URL')
+                        })
                 await sse_queue.put(("final_message", response))
+                await sse_queue.put(("final_sources", json.dumps(sources_for_answer)))
                 SESSION_STORE["chat_history"].append({"query": user_query, "response": response})
+                SESSION_STORE["answer"] = response
+                SESSION_STORE["source_contents"] = contents
                 await sse_queue.put(("action", {
                     "name": "sources",
     except Exception as e:
         await sse_queue.put(("error", str(e)))
+        traceback.print_exc()
         stop()
 # Create a FastAPI app
         return {"result": sources}
     except Exception as e:
         return JSONResponse(content={"error": str(e)}, status_code=500)
 # Define the route for graph action to display the graph
 @app.post("/action/graph")
 def action_graph() -> Dict[str, Any]:
     state = SESSION_STORE
     try:
         html_str = state['graph_rag'].display_graph()
         return {"result": result}
     except Exception as e:
         return JSONResponse(content={"error": str(e)}, status_code=500)
+# Define the route for excerpts action to display excerpts from the sources
+@app.post("/action/excerpts")
+async def action_excerpts() -> Dict[str, Any]:
+    def validate_excerpts_format(excerpts):
+        if not isinstance(excerpts, list):
+            return False
+        for item in excerpts:
+            if not isinstance(item, dict):
+                return False
+            for statement, sources in item.items():
+                if not isinstance(statement, str) or not isinstance(sources, dict):
+                    return False
+                for src_num, excerpt in sources.items():
+                    if not (isinstance(src_num, int) or isinstance(src_num, str)):
+                        return False
+                    if not isinstance(excerpt, str):
+                        return False
+        return True
+    try:
+        state = SESSION_STORE
+        response = state["answer"]
+        contents = state["source_contents"]
+        if not response or not contents:
+            raise ValueError("Required data for excerpts not found")
+        excerpts_list = await state["reasoner"].get_excerpts(response, contents)
+        cleaned_excerpts = re.sub(
+            r'```[\w\s]*\n?|```|~~~[\w\s]*\n?|~~~', '', excerpts_list, flags=re.MULTILINE | re.DOTALL
+        ).strip()
+        try:
+            excerpts = eval(cleaned_excerpts)
+        except Exception:
+            print(f"Error parsing excerpts:\n{cleaned_excerpts}")
+            raise ValueError("Excerpts could not be parsed as a Python list.")
+        if not validate_excerpts_format(excerpts):
+            print(f"Excerpts format validation failed:\n{excerpts}")
+            raise ValueError("Excerpts are not in the required format.")
+        print(f"Excerpts:\n{excerpts}")
+        return {"result": excerpts}
+    except Exception as e:
+        print(f"Error in action_excerpts: {e}")
+        return JSONResponse(content={"error": str(e)}, status_code=500)
+# Define the route for settings to set or update the environment variables
 @app.post("/settings")
 async def update_settings(data: Dict[str, Any]):
     from src.helpers.helper import (
     multiple_api_keys = data.get("Model_API_Keys", "").strip()
     brave_api_key = data.get("Brave_Search_API_Key", "").strip()
     proxy_list = data.get("Proxy_List", "").strip()
     model_temperature = str(data.get("Model_Temperature", 0.0))
     model_top_p = str(data.get("Model_Top_P", 1.0))
         env_updates.update(px)
     env_updates["BRAVE_API_KEY"] = brave_api_key
     env_updates["MODEL_PROVIDER"] = prov_lower
     env_updates["MODEL_NAME"] = model_name
     env_updates["MODEL_TEMPERATURE"] = model_temperature
     update_env_vars(env_updates)
     load_dotenv(override=True)
+    await initialize_components()
     return {"success": True}
+# Define the route for adding/uploading content for a specific session
+@app.post("/add-content")
+async def add_content(files: Optional[List[UploadFile]] = File(None), urls: str = Form(...)):
+    state = SESSION_STORE
+    session_id = state.get("session_id")
+    if not session_id:
+        raise HTTPException(status_code=400, detail="Session ID is not set. Please start a session first.")
+    session_upload_path = os.path.join(UPLOAD_DIRECTORY, session_id)
+    os.makedirs(session_upload_path, exist_ok=True)
+    saved_filenames = []
+    if files:
+        total_new_files_size = sum(file.size for file in files)
+        current_folder_size = get_folder_size(session_upload_path)
+        # Check if the total size exceeds the maximum allowed folder size
+        if current_folder_size + total_new_files_size > MAX_FOLDER_SIZE:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Cannot add files as total storage would exceed 10 MB. Current size: {current_folder_size / (1024 * 1024):.2f} MB"
+            )
+        for file in files:
+            file_path = os.path.join(session_upload_path, file.filename)
+            try:
+                with open(file_path, "wb") as buffer:
+                    shutil.copyfileobj(file.file, buffer)
+                saved_filenames.append(file.filename)
+            finally:
+                file.file.close()
+    try:
+        parsed_urls = json.loads(urls)
+        print(f"Received links: {parsed_urls}")
+    except json.JSONDecodeError:
+        raise HTTPException(status_code=400, detail="Invalid URL format.")
+    # Store user-provided links in session
+    if parsed_urls:
+        SESSION_STORE["user_provided_links"] = parsed_urls
+    return {
+        "message": "Content added successfully",
+        "files_added": saved_filenames,
+        "links_added": parsed_urls
+    }
+# Define the route to update the selected services for searching
+@app.post("/api/selected-services")
+async def update_selected_services(data: Dict[str, Any]):
+    state = SESSION_STORE
+    selected_services = data.get("services", {})
+    state["selected_services"] = selected_services
+    logger.info(f"Updated selected services: {selected_services}")
+    return {"success": True, "services": selected_services}
+# Define the route to receive OAuth tokens from the frontend
+@app.post("/api/session-token")
+async def receive_session_token(data: Dict[str, Any]):
+    provider = data.get("provider")  # 'google', 'microsoft', 'slack'
+    token = data.get("token")
+    if not provider or not token:
+        raise HTTPException(status_code=400, detail="Provider and token are required")
+    SESSION_STORE["oauth_tokens"][provider] = {
+        "token": token,
+        "timestamp": time.time()
+    }
+    logger.info(f"Stored token {token} for provider {provider}")
+    return {"success": True, "message": f"{provider} token stored successfully"}
+# Define the route for cleaning up a session if the session ID matches
+@app.post("/cleanup")
+async def cleanup_session():
+    state = SESSION_STORE
+    session_id = state.get("session_id")
+    if not session_id:
+        raise HTTPException(status_code=400, detail="Session ID is not set. Please start a session first.")
+    session_upload_path = os.path.join(UPLOAD_DIRECTORY, session_id)
+    if session_id:
+        # Clear the session upload directory
+        clear_folder(session_upload_path)
+        # Clear user-provided links and caches
+        SESSION_STORE["user_provided_links"] = []
+        SESSION_STORE["user_files_cache"] = {}
+        SESSION_STORE["user_links_cache"] = {}
+        SESSION_STORE["selected_services"] = {}
+        SESSION_STORE["oauth_tokens"] = {}
+        return {"message": "Cleanup successful."}
+    return {"message": "No session ID provided, cleanup skipped."}
 @app.on_event("startup")
 def init_chat():
     if not SESSION_STORE:
         print("Initializing chat...")
+        # Create the upload directory if it doesn't exist
+        print("Creating upload directory...")
+        os.makedirs(UPLOAD_DIRECTORY, exist_ok=True)
+        # Initialize the session store
         SESSION_STORE["settings_saved"] = False
         SESSION_STORE["session_id"] = None
+        SESSION_STORE["answer"] = None
+        SESSION_STORE["source_contents"] = None
         SESSION_STORE["chat_history"] = []
+        SESSION_STORE["user_provided_links"] = []
+        SESSION_STORE["user_files_cache"] = {}
+        SESSION_STORE["user_links_cache"] = {}
+        SESSION_STORE["selected_services"] = {}
+        SESSION_STORE["oauth_tokens"] = {}
         print("Chat initialized!")
         return {"sucess": True}
     else:
         print("Chat already initialized!")

src/crawl/crawler.py CHANGED Viewed

@@ -1,17 +1,17 @@
-# from crawl4ai import AsyncWebCrawler
-# from urllib.parse import urlparse
 import aiohttp
 import asyncio
-# from asyncio.exceptions import TimeoutError as async_timeout
 from fast_async import make_async
 from bs4 import BeautifulSoup, NavigableString
-# import secrets
-# from datetime import datetime
-# import random
 import os
 import re
 import uuid
-from typing import List, Dict, Optional #, Tuple
 from io import BytesIO
 import PyPDF2
 from fake_useragent import FakeUserAgent
@@ -20,597 +20,597 @@ from transformers import AutoTokenizer, AutoConfig
 import torch
 import time
-# class Crawler:
-#     def __init__(self, user_dir=None, rate_limit=1, headless=True, verbose=False):
-#         self.session_pool = {}  # Track active sessions
-#         self.verbose = verbose
-#         self.rate_limit = rate_limit
-#         self.user_dir = user_dir
-#         self.headless = headless
-#         self.crawler = AsyncWebCrawler(
-#             context_options={"userDataDir": self.user_dir},
-#             headless=self.headless,
-#             verbose=self.verbose
-#         )
-#         # Browser context management
-#         self._browser_contexts = {}
-#         self._context_locks = {}
-#     async def get_browser_context(self, session_id):
-#         """Get or create a browser context with proper locking"""
-#         if session_id not in self._context_locks:
-#             self._context_locks[session_id] = asyncio.Lock()
-#         async with self._context_locks[session_id]:
-#             if session_id not in self._browser_contexts:
-#                 context = await self.crawler.new_context()
-#                 self._browser_contexts[session_id] = context
-#             return self._browser_contexts[session_id]
-#     async def cleanup_browser_context(self, session_id):
-#         """Safely cleanup browser context"""
-#         if session_id in self._context_locks:
-#             async with self._context_locks[session_id]:
-#                 if session_id in self._browser_contexts:
-#                     try:
-#                         await asyncio.shield(
-#                             self._browser_contexts[session_id].close()
-#                         )
-#                     except Exception as e:
-#                         print(f"Error cleaning up browser context: {e}")
-#                     finally:
-#                         del self._browser_contexts[session_id]
-#     def create_session(self):
-#         """Create a new session with secure ID"""
-#         session_id = secrets.token_urlsafe(32)  # Secure session ID
-#         self.session_pool[session_id] = {
-#             'created_at': datetime.now(),
-#             'last_used': datetime.now(),
-#             'requests_count': 0
-#         }
-#         return session_id
-#     def rotate_session(self, session_id):
-#         """Implement session rotation logic"""
-#         if self.session_pool[session_id]['requests_count'] > 100:
-#             self.cleanup_session(session_id)
-#             return self.create_session()
-#         return session_id
-#     def is_dynamic_page(self, html_content: str) -> Tuple[bool, Optional[str]]:
-#         """Analyzes HTML content to determine if a webpage is dynamically loaded"""
-#         def _check_structural_indicators(soup: BeautifulSoup) -> Dict[str, int]:
-#             """Check structural indicators of dynamic content loading."""
-#             scores = {
-#                 'empty_containers': 0,
-#                 'repeated_structures': 0,
-#                 'api_endpoints': 0,
-#                 'state_management': 0
-#             }
-#             # 1. Check for empty content containers
-#             main_containers = soup.find_all(['main', 'div', 'section'],
-#                                         class_=lambda x: x and any(term in str(x).lower()
-#                                         for term in ['content', 'main', 'feed', 'list', 'container']))
-#             for container in main_containers:
-#                 # Check if container is empty or has minimal content
-#                 if len(container.find_all()) < 3:
-#                     scores['empty_containers'] += 1
-#                 # Check for repeated similar structures (common in dynamic lists)
-#                 children = container.find_all(recursive=False)
-#                 if children:
-#                     first_child_class = children[0].get('class', [])
-#                     similar_siblings = [c for c in children[1:]
-#                                     if c.get('class', []) == first_child_class]
-#                     if len(similar_siblings) > 0:
-#                         scores['repeated_structures'] += 1
-#             # 2. Check for API endpoints in scripts
-#             scripts = soup.find_all('script', {'src': True})
-#             api_patterns = ['/api/', '/graphql', '/rest/', '/v1/', '/v2/']
-#             for script in scripts:
-#                 if any(pattern in script['src'] for pattern in api_patterns):
-#                     scores['api_endpoints'] += 1
-#             # 3. Look for state management setup
-#             state_patterns = [
-#                 r'window\.__INITIAL_STATE__',
-#                 r'window\.__PRELOADED_STATE__',
-#                 r'__REDUX_STATE__',
-#                 r'__NUXT__',
-#                 r'__NEXT_DATA__',
-#                 r'window\.__data'
-#             ]
-#             inline_scripts = soup.find_all('script')
-#             for script in inline_scripts:
-#                 if script.string:
-#                     for pattern in state_patterns:
-#                         if re.search(pattern, script.string):
-#                             scores['state_management'] += 1
-#             return scores
-#         def _check_modern_framework_indicators(soup: BeautifulSoup) -> Dict[str, int]:
-#             """Check for indicators of modern web frameworks and dynamic loading patterns."""
-#             scores = {
-#                 'framework_roots': 0,
-#                 'hydration': 0,
-#                 'routing': 0
-#             }
-#             # 1. Framework-specific root elements
-#             framework_roots = {
-#                 'react': ['react-root', 'react-app', 'root', '__next'],
-#                 'angular': ['ng-version', 'ng-app'],
-#                 'vue': ['v-app', '#app', 'nuxt-app'],
-#                 'modern': ['app-root', 'application', 'spa-root']
-#             }
-#             for framework, identifiers in framework_roots.items():
-#                 for id_value in identifiers:
-#                     if (soup.find(attrs={'id': re.compile(id_value, re.I)}) or
-#                         soup.find(attrs={'class': re.compile(id_value, re.I)}) or
-#                         soup.find(attrs={'data-': re.compile(id_value, re.I)})):
-#                         scores['framework_roots'] += 1
-#             # 2. Check for hydration indicators
-#             hydration_patterns = [
-#                 r'hydrate',
-#                 r'createRoot',
-#                 r'reactive',
-#                 r'observable'
-#             ]
-#             scripts = soup.find_all('script')
-#             for script in scripts:
-#                 if script.string:
-#                     for pattern in hydration_patterns:
-#                         if re.search(pattern, script.string):
-#                             scores['hydration'] += 1
-#             # 3. Check for dynamic routing setup
-#             router_patterns = [
-#                 'router-view',
-#                 'router-link',
-#                 'route-link',
-#                 'history.push',
-#                 'navigation'
-#             ]
-#             for pattern in router_patterns:
-#                 if soup.find(class_=re.compile(pattern, re.I)) or \
-#                 soup.find(id=re.compile(pattern, re.I)):
-#                     scores['routing'] += 1
-#             return scores
-#         def _check_dynamic_loading_patterns(soup: BeautifulSoup) -> Dict[str, int]:
-#             """Check for various dynamic content loading patterns."""
-#             scores = {
-#                 'infinite_scroll': 0,
-#                 'load_more_buttons': 0,
-#                 'pagination': 0,
-#                 'lazy_loading': 0,
-#                 'loading_indicators': 0
-#             }
-#             # 1. Check for infinite scroll indicators
-#             scroll_indicators = [
-#                 'infinite-scroll',
-#                 'data-infinite',
-#                 'data-virtualized',
-#                 'virtual-scroll',
-#                 'scroll-container',
-#                 'scroll-viewport'
-#             ]
-#             for indicator in scroll_indicators:
-#                 elements = soup.find_all(
-#                     lambda tag: any(indicator.lower() in str(v).lower()
-#                                 for v in tag.attrs.values())
-#                 )
-#                 if elements:
-#                     scores['infinite_scroll'] += len(elements)
-#             # 2. Check for load more buttons
-#             button_patterns = [
-#                 r'load[_-]?more',
-#                 r'show[_-]?more',
-#                 r'view[_-]?more',
-#                 r'see[_-]?more',
-#                 r'more[_-]?posts',
-#                 r'more[_-]?results'
-#             ]
-#             for pattern in button_patterns:
-#                 elements = soup.find_all(
-#                     ['button', 'a', 'div', 'span'],
-#                     text=re.compile(pattern, re.I)
-#                 )
-#                 if elements:
-#                     scores['load_more_buttons'] += len(elements)
-#             # 3. Check for pagination
-#             pagination_patterns = [
-#                 'pagination',
-#                 'page-numbers',
-#                 'page-nav',
-#                 'page-links'
-#             ]
-#             for pattern in pagination_patterns:
-#                 elements = soup.find_all(class_=re.compile(pattern, re.I))
-#                 if elements:
-#                     scores['pagination'] += len(elements)
-#             # 4. Check for lazy loading
-#             lazy_patterns = ['lazy', 'data-src', 'data-lazy']
-#             for pattern in lazy_patterns:
-#                 elements = soup.find_all(
-#                     lambda tag: any(pattern.lower() in str(v).lower()
-#                                 for v in tag.attrs.values())
-#                 )
-#                 if elements:
-#                     scores['lazy_loading'] += len(elements)
-#             # 5. Check for loading indicators
-#             loading_patterns = [
-#                 'loading',
-#                 'spinner',
-#                 'skeleton',
-#                 'placeholder',
-#                 'shimmer'
-#             ]
-#             for pattern in loading_patterns:
-#                 elements = soup.find_all(class_=re.compile(pattern, re.I))
-#                 if elements:
-#                     scores['loading_indicators'] += len(elements)
-#             return scores
-#         def _evaluate_dynamic_indicators(
-#             structural: Dict[str, int],
-#             framework: Dict[str, int],
-#             loading: Dict[str, int]
-#         ) -> Tuple[bool, Optional[str]]:
-#             """Evaluate dynamic indicators and return JavaScript instructions."""
-#             methods = []
-#             js_snippets = []
-#             # Infinite Scroll
-#             if loading['infinite_scroll'] > 0:
-#                 methods.append("scroll")
-#                 js_snippets.append(
-#                     """
-# window.scrollTo(0, document.body.scrollHeight);
-# await new Promise(resolve => setTimeout(resolve, 1000));
-#                     """.strip().replace('\n', '')
-#                 )
-#             # Load More Buttons
-#             if loading['load_more_buttons'] > 0:
-#                 methods.append("button")
-#                 js_snippets.append(
-#                     """
-# const button = Array.from(document.querySelectorAll('button, a, div, span')).find(
-#     el => /load[_-]?more|show[_-]?more/i.test(el.textContent)
-# );
-# if (button) {
-#     button.click();
-#     await new Promise(resolve => setTimeout(resolve, 1000));
-# } else {
-#     console.warn("No 'Load More' button found.");
-# }
-#                     """.strip().replace('\n', '')
-#                 )
-#             # Paginated Interfaces
-#             if loading.get('pagination', 0) > 0:
-#                 methods.append("pagination")
-#                 js_snippets.append(
-#                     """
-#   const nextPage = document.querySelector('a[rel="next"], .pagination-next, .page-next');
-# if (nextPage) {
-#     nextPage.click();
-#     await new Promise(resolve => setTimeout(resolve, 1000));
-# } else {
-#     console.warn("No pagination link found.");
-# }
-#                     """.strip().replace('\n', '')
-#                 )
-#             # Lazy Loading
-#             if loading.get('lazy_loading', 0) > 0:
-#                 methods.append("lazy")
-#                 js_snippets.append(
-#                     """
-# if (window.__INITIAL_STATE__ || window.__REDUX_STATE__ || window.__NUXT__ || window.__NEXT_DATA__) {
-#     console.log('Framework state detected. Consider monitoring network requests for further actions.');
-# }
-#                     """.strip().replace('\n', '')
-#                 )
-#             # Framework and State Management Indicators
-#             if framework['framework_roots'] > 0 or structural['state_management'] > 0:
-#                 methods.append("stateful")
-#                 js_snippets.append(
-#                     """
-# if (window.__INITIAL_STATE__ || window.__REDUX_STATE__ || window.__NUXT__ || window.__NEXT_DATA__) {
-#     console.log('Detected stateful framework data loading.');
-#                     }
-#                     """.strip().replace('\n', '')
-#                 )
-#             # API-Driven Content
-#             if structural['api_endpoints'] > 0:
-#                 methods.append("api")
-#                 js_snippets.append(
-#                     """
-# console.log('API requests detected. Use browser devtools to inspect network activity for specific endpoints.');
-#                     """.strip().replace('\n', '')
-#                 )
-#             # Aggregate and finalize
-#             if methods:
-#                 js_code = "\n".join(js_snippets)
-#                 return True, js_code
-#             return False, None
-#         # Main execution
-#         soup = BeautifulSoup(html_content, 'html.parser')
-#         # Run all checks
-#         structural_scores = _check_structural_indicators(soup)
-#         framework_scores = _check_modern_framework_indicators(soup)
-#         loading_scores = _check_dynamic_loading_patterns(soup)
-#         # Evaluate results
-#         return _evaluate_dynamic_indicators(structural_scores, framework_scores, loading_scores)
-#     async def crawl(
-#             self,
-#             url,
-#             depth=2,
-#             max_pages=5,
-#             session_id=None,
-#             human_simulation=True,
-#             rotate_user_agent=True,
-#             rotate_proxy=True,
-#             return_html=False
-#         ):
-#         if not session_id:
-#             session_id = self.create_session()
-#         session_id = self.rotate_session(session_id)
-#         # List of rotating user agents
-#         user_agents = [
-#             'Chrome/115.0.0.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
-#             'Chrome/115.0.0.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
-#             'Chrome/115.0.0.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
-#             'Chrome/115.0.0.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
-#             'Chrome/115.0.0.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36'
-#         ]
-#         # List of rotating proxies
-#         proxies = [
-#             "http://50.62.183.123:80",
-#             "http://104.129.60.84:6516",
-#             "http://156.228.118.163:3128",
-#             "http://142.111.104.97:6107",
-#             "http://156.228.99.99:3128"
-#         ]
-#         try:
-#             async with self.crawler as crawler:
-#                 # Rotate user agent and optimize headers for each attempt
-#                 headers = {
-#                     "User-Agent": random.choice(user_agents) if rotate_user_agent else user_agents[0],
-#                     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-#                     "Accept-Language": "en-US,en;q=0.5",
-#                     "Accept-Encoding": "gzip, deflate",
-#                     "Connection": "keep-alive",
-#                     "Upgrade-Insecure-Requests": "1",
-#                     "Sec-Fetch-Dest": "document",
-#                     "Sec-Fetch-Mode": "navigate",
-#                     "Sec-Fetch-Site": "none",
-#                     "Sec-Fetch-User": "?1",
-#                     "Cache-Control": "max-age=0"
-#                 }
-#                 # Update crawler headers for rotation
-#                 crawler.crawler_strategy.headers = headers
-#                 if rotate_proxy:
-#                     # Update crawler proxy for rotation
-#                     crawler.crawler_strategy.proxy = random.choice(proxies)
-#                 result_1 = await crawler.arun(
-#                     session_id=session_id,
-#                     url=url,
-#                     magic=True if human_simulation else False,
-#                     simulate_user=True if human_simulation else False,
-#                     override_navigator=True if human_simulation else False,
-#                     depth=depth,
-#                     max_pages=max_pages,
-#                     bypass_cache=True,
-#                     remove_overlay_elements=True,
-#                     delay_before_retrieve_html=1.0,
-#                     verbose=self.verbose
-#                 )
-#             # Update session metrics
-#             self.session_pool[session_id]['requests_count'] += 1
-#             self.session_pool[session_id]['last_used'] = datetime.now()
-#             if result_1.success:
-#                 if hasattr(result_1, 'html'):
-#                     success, js_code = self.is_dynamic_page(result_1.html)
-#                     if success:
-#                         async with crawler as crawler:
-#                             # Update crawler headers for rotation
-#                             crawler.crawler_strategy.headers = headers
-#                             if rotate_proxy:
-#                                 # Update crawler proxy for rotation
-#                                 crawler.crawler_strategy.proxy = random.choice(proxies)
-#                             print(f"Executing JS code: {js_code}")
-#                             result_2 = await crawler.arun(
-#                                 session_id=session_id,
-#                                 url=url,
-#                                 magic=True if human_simulation else False,
-#                                 simulate_user=True if human_simulation else False,
-#                                 override_navigator=True if human_simulation else False,
-#                                 depth=depth,
-#                                 max_pages=max_pages,
-#                                 js_code=js_code,
-#                                 bypass_cache=True,
-#                                 remove_overlay_elements=True,
-#                                 delay_before_retrieve_html=1.0,
-#                                 verbose=self.verbose
-#                             )
-#                         if result_2.success:
-#                             result = result_2
-#                         else:
-#                             result = result_1
-#                         # Update session metrics
-#                         self.session_pool[session_id]['requests_count'] += 1
-#                         self.session_pool[session_id]['last_used'] = datetime.now()
-#                     else:
-#                         result = result_1
-#                     if return_html and hasattr(result, 'html'):
-#                         return result.html
-#                     elif hasattr(result, 'fit_markdown'):
-#                         return result.fit_markdown
-#                     elif hasattr(result, 'markdown'):
-#                         return self.extract_content(result.markdown)
-#         except Exception as e:
-#             print(f"Error crawling {url}: {str(e)}")
-#         return None
-#     async def crawl_with_retry(
-#             self,
-#             url,
-#             depth=2,
-#             max_pages=5,
-#             max_retries=3,
-#             backoff_factor=1,
-#             session_id=None,
-#             human_simulation=True,
-#             rotate_user_agent=True,
-#             rotate_proxy=True,
-#             return_html=False,
-#             timeout=10.0
-#         ):
-#         """Crawl with retry logic and anti-blocking measures"""
-#         async def attempt_crawl(attempt):
-#             try:
-#                 async with async_timeout.timeout(timeout):
-#                     context = await self.get_browser_context(session_id)
-#                     return await self.crawl(
-#                         context,
-#                         url,
-#                         depth,
-#                         max_pages,
-#                         session_id,
-#                         human_simulation,
-#                         rotate_user_agent,
-#                         rotate_proxy,
-#                         return_html
-#                     )
-#             except asyncio.TimeoutError:
-#                 print(f"Timeout on attempt {attempt} for {url}")
-#                 raise
-#             except Exception as e:
-#                 print(f"Error on attempt {attempt} for {url}: {e}")
-#                 raise
-#         if not self.is_valid_url(url) and not self.is_html_url(url):
-#             print(f"Invalid URL: {url}")
-#             return f"No web results found for query: {url}"
-#         for attempt in range(max_retries):
-#             try:
-#                 if attempt > 0:
-#                     # Add delay between retries with exponential backoff
-#                     delay = backoff_factor * (2 ** (attempt - 1))
-#                     await asyncio.sleep(delay)
-#                 return await attempt_crawl(attempt + 1)
-#             except Exception as e:
-#                 if attempt == max_retries - 1:
-#                     print(f"Max retries ({max_retries}) reached for {url}")
-#                     return f"Failed to crawl after {max_retries} attempts: {url}"
-#                 continue
-#         return f"No content found after {max_retries} attempts for: {url}"
-#     def extract_content(self, html_content):
-#         soup = BeautifulSoup(html_content, 'html.parser')
-#         for script in soup(["script", "style"]):
-#             script.decompose()
-#         text = soup.get_text()
-#         lines = (line.strip() for line in text.splitlines())
-#         chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
-#         text = '\n'.join(chunk for chunk in chunks if chunk)
-#         return text
-#     def cleanup_session(self, session_id):
-#         """Clean up a session"""
-#         print(f"Cleaning up session {session_id}")
-#         if session_id in self.session_pool:
-#             self.crawler.crawler_strategy.kill_session(session_id)
-#             del self.session_pool[session_id]
-#     def cleanup_expired_sessions(self):
-#         """Regular cleanup of expired sessions using proper time calculation"""
-#         try:
-#             current_time = datetime.now()
-#             expired_sessions = []
-#             for sid, data in self.session_pool.items():
-#                 # Calculate time difference in seconds
-#                 time_diff = (current_time - data['last_used']).total_seconds()
-#                 # Check if more than 1 hour (3600 seconds)
-#                 if time_diff > 3600:
-#                     expired_sessions.append(sid)
-#             # Cleanup expired sessions
-#             for session_id in expired_sessions:
-#                 self.cleanup_session(session_id)
-#         except Exception as e:
-#             if self.verbose:
-#                 print(f"Error during session cleanup: {str(e)}")
-#     @staticmethod
-#     def is_valid_url(url):
-#         try:
-#             result = urlparse(url)
-#             return all([result.scheme, result.netloc])
-#         except ValueError:
-#             return False
-#     @staticmethod
-#     def is_html_url(url):
-#         return url.endswith(".html") or url.endswith(".htm")
 class CustomCrawler:
     def __init__(

+from crawl4ai import AsyncWebCrawler
+from urllib.parse import urlparse
 import aiohttp
 import asyncio
+from asyncio.exceptions import TimeoutError as async_timeout
 from fast_async import make_async
 from bs4 import BeautifulSoup, NavigableString
+import secrets
+from datetime import datetime
+import random
 import os
 import re
 import uuid
+from typing import List, Dict, Tuple, Optional
 from io import BytesIO
 import PyPDF2
 from fake_useragent import FakeUserAgent
 import torch
 import time
+class Crawler:
+    def __init__(self, user_dir=None, rate_limit=1, headless=True, verbose=False):
+        self.session_pool = {}  # Track active sessions
+        self.verbose = verbose
+        self.rate_limit = rate_limit
+        self.user_dir = user_dir
+        self.headless = headless
+        self.crawler = AsyncWebCrawler(
+            context_options={"userDataDir": self.user_dir},
+            headless=self.headless,
+            verbose=self.verbose
+        )
+        # Browser context management
+        self._browser_contexts = {}
+        self._context_locks = {}
+    async def get_browser_context(self, session_id):
+        """Get or create a browser context with proper locking"""
+        if session_id not in self._context_locks:
+            self._context_locks[session_id] = asyncio.Lock()
+        async with self._context_locks[session_id]:
+            if session_id not in self._browser_contexts:
+                context = await self.crawler.new_context()
+                self._browser_contexts[session_id] = context
+            return self._browser_contexts[session_id]
+    async def cleanup_browser_context(self, session_id):
+        """Safely cleanup browser context"""
+        if session_id in self._context_locks:
+            async with self._context_locks[session_id]:
+                if session_id in self._browser_contexts:
+                    try:
+                        await asyncio.shield(
+                            self._browser_contexts[session_id].close()
+                        )
+                    except Exception as e:
+                        print(f"Error cleaning up browser context: {e}")
+                    finally:
+                        del self._browser_contexts[session_id]
+    def create_session(self):
+        """Create a new session with secure ID"""
+        session_id = secrets.token_urlsafe(32)  # Secure session ID
+        self.session_pool[session_id] = {
+            'created_at': datetime.now(),
+            'last_used': datetime.now(),
+            'requests_count': 0
+        }
+        return session_id
+    def rotate_session(self, session_id):
+        """Implement session rotation logic"""
+        if self.session_pool[session_id]['requests_count'] > 100:
+            self.cleanup_session(session_id)
+            return self.create_session()
+        return session_id
+    def is_dynamic_page(self, html_content: str) -> Tuple[bool, Optional[str]]:
+        """Analyzes HTML content to determine if a webpage is dynamically loaded"""
+        def _check_structural_indicators(soup: BeautifulSoup) -> Dict[str, int]:
+            """Check structural indicators of dynamic content loading."""
+            scores = {
+                'empty_containers': 0,
+                'repeated_structures': 0,
+                'api_endpoints': 0,
+                'state_management': 0
+            }
+            # 1. Check for empty content containers
+            main_containers = soup.find_all(['main', 'div', 'section'],
+                                        class_=lambda x: x and any(term in str(x).lower()
+                                        for term in ['content', 'main', 'feed', 'list', 'container']))
+            for container in main_containers:
+                # Check if container is empty or has minimal content
+                if len(container.find_all()) < 3:
+                    scores['empty_containers'] += 1
+                # Check for repeated similar structures (common in dynamic lists)
+                children = container.find_all(recursive=False)
+                if children:
+                    first_child_class = children[0].get('class', [])
+                    similar_siblings = [c for c in children[1:]
+                                    if c.get('class', []) == first_child_class]
+                    if len(similar_siblings) > 0:
+                        scores['repeated_structures'] += 1
+            # 2. Check for API endpoints in scripts
+            scripts = soup.find_all('script', {'src': True})
+            api_patterns = ['/api/', '/graphql', '/rest/', '/v1/', '/v2/']
+            for script in scripts:
+                if any(pattern in script['src'] for pattern in api_patterns):
+                    scores['api_endpoints'] += 1
+            # 3. Look for state management setup
+            state_patterns = [
+                r'window\.__INITIAL_STATE__',
+                r'window\.__PRELOADED_STATE__',
+                r'__REDUX_STATE__',
+                r'__NUXT__',
+                r'__NEXT_DATA__',
+                r'window\.__data'
+            ]
+            inline_scripts = soup.find_all('script')
+            for script in inline_scripts:
+                if script.string:
+                    for pattern in state_patterns:
+                        if re.search(pattern, script.string):
+                            scores['state_management'] += 1
+            return scores
+        def _check_modern_framework_indicators(soup: BeautifulSoup) -> Dict[str, int]:
+            """Check for indicators of modern web frameworks and dynamic loading patterns."""
+            scores = {
+                'framework_roots': 0,
+                'hydration': 0,
+                'routing': 0
+            }
+            # 1. Framework-specific root elements
+            framework_roots = {
+                'react': ['react-root', 'react-app', 'root', '__next'],
+                'angular': ['ng-version', 'ng-app'],
+                'vue': ['v-app', '#app', 'nuxt-app'],
+                'modern': ['app-root', 'application', 'spa-root']
+            }
+            for framework, identifiers in framework_roots.items():
+                for id_value in identifiers:
+                    if (soup.find(attrs={'id': re.compile(id_value, re.I)}) or
+                        soup.find(attrs={'class': re.compile(id_value, re.I)}) or
+                        soup.find(attrs={'data-': re.compile(id_value, re.I)})):
+                        scores['framework_roots'] += 1
+            # 2. Check for hydration indicators
+            hydration_patterns = [
+                r'hydrate',
+                r'createRoot',
+                r'reactive',
+                r'observable'
+            ]
+            scripts = soup.find_all('script')
+            for script in scripts:
+                if script.string:
+                    for pattern in hydration_patterns:
+                        if re.search(pattern, script.string):
+                            scores['hydration'] += 1
+            # 3. Check for dynamic routing setup
+            router_patterns = [
+                'router-view',
+                'router-link',
+                'route-link',
+                'history.push',
+                'navigation'
+            ]
+            for pattern in router_patterns:
+                if soup.find(class_=re.compile(pattern, re.I)) or \
+                soup.find(id=re.compile(pattern, re.I)):
+                    scores['routing'] += 1
+            return scores
+        def _check_dynamic_loading_patterns(soup: BeautifulSoup) -> Dict[str, int]:
+            """Check for various dynamic content loading patterns."""
+            scores = {
+                'infinite_scroll': 0,
+                'load_more_buttons': 0,
+                'pagination': 0,
+                'lazy_loading': 0,
+                'loading_indicators': 0
+            }
+            # 1. Check for infinite scroll indicators
+            scroll_indicators = [
+                'infinite-scroll',
+                'data-infinite',
+                'data-virtualized',
+                'virtual-scroll',
+                'scroll-container',
+                'scroll-viewport'
+            ]
+            for indicator in scroll_indicators:
+                elements = soup.find_all(
+                    lambda tag: any(indicator.lower() in str(v).lower()
+                                for v in tag.attrs.values())
+                )
+                if elements:
+                    scores['infinite_scroll'] += len(elements)
+            # 2. Check for load more buttons
+            button_patterns = [
+                r'load[_-]?more',
+                r'show[_-]?more',
+                r'view[_-]?more',
+                r'see[_-]?more',
+                r'more[_-]?posts',
+                r'more[_-]?results'
+            ]
+            for pattern in button_patterns:
+                elements = soup.find_all(
+                    ['button', 'a', 'div', 'span'],
+                    text=re.compile(pattern, re.I)
+                )
+                if elements:
+                    scores['load_more_buttons'] += len(elements)
+            # 3. Check for pagination
+            pagination_patterns = [
+                'pagination',
+                'page-numbers',
+                'page-nav',
+                'page-links'
+            ]
+            for pattern in pagination_patterns:
+                elements = soup.find_all(class_=re.compile(pattern, re.I))
+                if elements:
+                    scores['pagination'] += len(elements)
+            # 4. Check for lazy loading
+            lazy_patterns = ['lazy', 'data-src', 'data-lazy']
+            for pattern in lazy_patterns:
+                elements = soup.find_all(
+                    lambda tag: any(pattern.lower() in str(v).lower()
+                                for v in tag.attrs.values())
+                )
+                if elements:
+                    scores['lazy_loading'] += len(elements)
+            # 5. Check for loading indicators
+            loading_patterns = [
+                'loading',
+                'spinner',
+                'skeleton',
+                'placeholder',
+                'shimmer'
+            ]
+            for pattern in loading_patterns:
+                elements = soup.find_all(class_=re.compile(pattern, re.I))
+                if elements:
+                    scores['loading_indicators'] += len(elements)
+            return scores
+        def _evaluate_dynamic_indicators(
+            structural: Dict[str, int],
+            framework: Dict[str, int],
+            loading: Dict[str, int]
+        ) -> Tuple[bool, Optional[str]]:
+            """Evaluate dynamic indicators and return JavaScript instructions."""
+            methods = []
+            js_snippets = []
+            # Infinite Scroll
+            if loading['infinite_scroll'] > 0:
+                methods.append("scroll")
+                js_snippets.append(
+                    """
+window.scrollTo(0, document.body.scrollHeight);
+await new Promise(resolve => setTimeout(resolve, 1000));
+                    """.strip().replace('\n', '')
+                )
+            # Load More Buttons
+            if loading['load_more_buttons'] > 0:
+                methods.append("button")
+                js_snippets.append(
+                    """
+const button = Array.from(document.querySelectorAll('button, a, div, span')).find(
+    el => /load[_-]?more|show[_-]?more/i.test(el.textContent)
+);
+if (button) {
+    button.click();
+    await new Promise(resolve => setTimeout(resolve, 1000));
+} else {
+    console.warn("No 'Load More' button found.");
+}
+                    """.strip().replace('\n', '')
+                )
+            # Paginated Interfaces
+            if loading.get('pagination', 0) > 0:
+                methods.append("pagination")
+                js_snippets.append(
+                    """
+  const nextPage = document.querySelector('a[rel="next"], .pagination-next, .page-next');
+if (nextPage) {
+    nextPage.click();
+    await new Promise(resolve => setTimeout(resolve, 1000));
+} else {
+    console.warn("No pagination link found.");
+}
+                    """.strip().replace('\n', '')
+                )
+            # Lazy Loading
+            if loading.get('lazy_loading', 0) > 0:
+                methods.append("lazy")
+                js_snippets.append(
+                    """
+if (window.__INITIAL_STATE__ || window.__REDUX_STATE__ || window.__NUXT__ || window.__NEXT_DATA__) {
+    console.log('Framework state detected. Consider monitoring network requests for further actions.');
+}
+                    """.strip().replace('\n', '')
+                )
+            # Framework and State Management Indicators
+            if framework['framework_roots'] > 0 or structural['state_management'] > 0:
+                methods.append("stateful")
+                js_snippets.append(
+                    """
+if (window.__INITIAL_STATE__ || window.__REDUX_STATE__ || window.__NUXT__ || window.__NEXT_DATA__) {
+    console.log('Detected stateful framework data loading.');
+                    }
+                    """.strip().replace('\n', '')
+                )
+            # API-Driven Content
+            if structural['api_endpoints'] > 0:
+                methods.append("api")
+                js_snippets.append(
+                    """
+console.log('API requests detected. Use browser devtools to inspect network activity for specific endpoints.');
+                    """.strip().replace('\n', '')
+                )
+            # Aggregate and finalize
+            if methods:
+                js_code = "\n".join(js_snippets)
+                return True, js_code
+            return False, None
+        # Main execution
+        soup = BeautifulSoup(html_content, 'html.parser')
+        # Run all checks
+        structural_scores = _check_structural_indicators(soup)
+        framework_scores = _check_modern_framework_indicators(soup)
+        loading_scores = _check_dynamic_loading_patterns(soup)
+        # Evaluate results
+        return _evaluate_dynamic_indicators(structural_scores, framework_scores, loading_scores)
+    async def crawl(
+            self,
+            url,
+            depth=2,
+            max_pages=5,
+            session_id=None,
+            human_simulation=True,
+            rotate_user_agent=True,
+            rotate_proxy=True,
+            return_html=False
+        ):
+        if not session_id:
+            session_id = self.create_session()
+        session_id = self.rotate_session(session_id)
+        # List of rotating user agents
+        user_agents = [
+            'Chrome/115.0.0.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
+            'Chrome/115.0.0.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
+            'Chrome/115.0.0.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
+            'Chrome/115.0.0.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
+            'Chrome/115.0.0.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36'
+        ]
+        # List of rotating proxies
+        proxies = [
+            "http://50.62.183.123:80",
+            "http://104.129.60.84:6516",
+            "http://156.228.118.163:3128",
+            "http://142.111.104.97:6107",
+            "http://156.228.99.99:3128"
+        ]
+        try:
+            async with self.crawler as crawler:
+                # Rotate user agent and optimize headers for each attempt
+                headers = {
+                    "User-Agent": random.choice(user_agents) if rotate_user_agent else user_agents[0],
+                    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+                    "Accept-Language": "en-US,en;q=0.5",
+                    "Accept-Encoding": "gzip, deflate",
+                    "Connection": "keep-alive",
+                    "Upgrade-Insecure-Requests": "1",
+                    "Sec-Fetch-Dest": "document",
+                    "Sec-Fetch-Mode": "navigate",
+                    "Sec-Fetch-Site": "none",
+                    "Sec-Fetch-User": "?1",
+                    "Cache-Control": "max-age=0"
+                }
+                # Update crawler headers for rotation
+                crawler.crawler_strategy.headers = headers
+                if rotate_proxy:
+                    # Update crawler proxy for rotation
+                    crawler.crawler_strategy.proxy = random.choice(proxies)
+                result_1 = await crawler.arun(
+                    session_id=session_id,
+                    url=url,
+                    magic=True if human_simulation else False,
+                    simulate_user=True if human_simulation else False,
+                    override_navigator=True if human_simulation else False,
+                    depth=depth,
+                    max_pages=max_pages,
+                    bypass_cache=True,
+                    remove_overlay_elements=True,
+                    delay_before_retrieve_html=1.0,
+                    verbose=self.verbose
+                )
+            # Update session metrics
+            self.session_pool[session_id]['requests_count'] += 1
+            self.session_pool[session_id]['last_used'] = datetime.now()
+            if result_1.success:
+                if hasattr(result_1, 'html'):
+                    success, js_code = self.is_dynamic_page(result_1.html)
+                    if success:
+                        async with crawler as crawler:
+                            # Update crawler headers for rotation
+                            crawler.crawler_strategy.headers = headers
+                            if rotate_proxy:
+                                # Update crawler proxy for rotation
+                                crawler.crawler_strategy.proxy = random.choice(proxies)
+                            print(f"Executing JS code: {js_code}")
+                            result_2 = await crawler.arun(
+                                session_id=session_id,
+                                url=url,
+                                magic=True if human_simulation else False,
+                                simulate_user=True if human_simulation else False,
+                                override_navigator=True if human_simulation else False,
+                                depth=depth,
+                                max_pages=max_pages,
+                                js_code=js_code,
+                                bypass_cache=True,
+                                remove_overlay_elements=True,
+                                delay_before_retrieve_html=1.0,
+                                verbose=self.verbose
+                            )
+                        if result_2.success:
+                            result = result_2
+                        else:
+                            result = result_1
+                        # Update session metrics
+                        self.session_pool[session_id]['requests_count'] += 1
+                        self.session_pool[session_id]['last_used'] = datetime.now()
+                    else:
+                        result = result_1
+                    if return_html and hasattr(result, 'html'):
+                        return result.html
+                    elif hasattr(result, 'fit_markdown'):
+                        return result.fit_markdown
+                    elif hasattr(result, 'markdown'):
+                        return self.extract_content(result.markdown)
+        except Exception as e:
+            print(f"Error crawling {url}: {str(e)}")
+        return None
+    async def crawl_with_retry(
+            self,
+            url,
+            depth=2,
+            max_pages=5,
+            max_retries=3,
+            backoff_factor=1,
+            session_id=None,
+            human_simulation=True,
+            rotate_user_agent=True,
+            rotate_proxy=True,
+            return_html=False,
+            timeout=10.0
+        ):
+        """Crawl with retry logic and anti-blocking measures"""
+        async def attempt_crawl(attempt):
+            try:
+                async with async_timeout.timeout(timeout):
+                    context = await self.get_browser_context(session_id)
+                    return await self.crawl(
+                        context,
+                        url,
+                        depth,
+                        max_pages,
+                        session_id,
+                        human_simulation,
+                        rotate_user_agent,
+                        rotate_proxy,
+                        return_html
+                    )
+            except asyncio.TimeoutError:
+                print(f"Timeout on attempt {attempt} for {url}")
+                raise
+            except Exception as e:
+                print(f"Error on attempt {attempt} for {url}: {e}")
+                raise
+        if not self.is_valid_url(url) and not self.is_html_url(url):
+            print(f"Invalid URL: {url}")
+            return f"No web results found for query: {url}"
+        for attempt in range(max_retries):
+            try:
+                if attempt > 0:
+                    # Add delay between retries with exponential backoff
+                    delay = backoff_factor * (2 ** (attempt - 1))
+                    await asyncio.sleep(delay)
+                return await attempt_crawl(attempt + 1)
+            except Exception as e:
+                if attempt == max_retries - 1:
+                    print(f"Max retries ({max_retries}) reached for {url}")
+                    return f"Failed to crawl after {max_retries} attempts: {url}"
+                continue
+        return f"No content found after {max_retries} attempts for: {url}"
+    def extract_content(self, html_content):
+        soup = BeautifulSoup(html_content, 'html.parser')
+        for script in soup(["script", "style"]):
+            script.decompose()
+        text = soup.get_text()
+        lines = (line.strip() for line in text.splitlines())
+        chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
+        text = '\n'.join(chunk for chunk in chunks if chunk)
+        return text
+    def cleanup_session(self, session_id):
+        """Clean up a session"""
+        print(f"Cleaning up session {session_id}")
+        if session_id in self.session_pool:
+            self.crawler.crawler_strategy.kill_session(session_id)
+            del self.session_pool[session_id]
+    def cleanup_expired_sessions(self):
+        """Regular cleanup of expired sessions using proper time calculation"""
+        try:
+            current_time = datetime.now()
+            expired_sessions = []
+            for sid, data in self.session_pool.items():
+                # Calculate time difference in seconds
+                time_diff = (current_time - data['last_used']).total_seconds()
+                # Check if more than 1 hour (3600 seconds)
+                if time_diff > 3600:
+                    expired_sessions.append(sid)
+            # Cleanup expired sessions
+            for session_id in expired_sessions:
+                self.cleanup_session(session_id)
+        except Exception as e:
+            if self.verbose:
+                print(f"Error during session cleanup: {str(e)}")
+    @staticmethod
+    def is_valid_url(url):
+        try:
+            result = urlparse(url)
+            return all([result.scheme, result.netloc])
+        except ValueError:
+            return False
+    @staticmethod
+    def is_html_url(url):
+        return url.endswith(".html") or url.endswith(".htm")
 class CustomCrawler:
     def __init__(

src/helpers/helper.py CHANGED Viewed

@@ -1,11 +1,13 @@
 import os
 import re
 import gc
 import torch
 import transformers
 from langchain.text_splitter import RecursiveCharacterTextSplitter, TokenTextSplitter
 ENV_FILE_PATH = os.path.join(os.getenv("WRITABLE_DIR", "/tmp"), ".env")
 def remove_markdown(text: str) -> str:
         # Remove code block format type and the code block itself
@@ -165,6 +167,17 @@ def read_env():
 # Function to update .env file
 def update_env_vars(new_values: dict):
     # Overwrite .env file with new values
     with open(ENV_FILE_PATH, "w", encoding="utf-8") as f:
         for var, val in new_values.items():
@@ -198,4 +211,23 @@ def prepare_proxy_list_updates(proxy_list: str) -> list:
     for i, proxy in enumerate(lines, start=1):
         proxies[f"PROXY_{i}"] = proxy
-    return proxies

 import os
 import re
 import gc
+import shutil
 import torch
 import transformers
 from langchain.text_splitter import RecursiveCharacterTextSplitter, TokenTextSplitter
 ENV_FILE_PATH = os.path.join(os.getenv("WRITABLE_DIR", "/tmp"), ".env")
+WEBHOOK_PATH = os.path.join(os.getenv("WRITABLE_DIR", "/tmp"), ".webhook_secret")
 def remove_markdown(text: str) -> str:
         # Remove code block format type and the code block itself
 # Function to update .env file
 def update_env_vars(new_values: dict):
+    # Function to load webhook URL securely
+    def load_webhook_url_securely() -> str:
+        if os.path.exists(WEBHOOK_PATH):
+            with open(WEBHOOK_PATH, "r", encoding="utf-8") as f:
+                return f.read().strip()
+        raise FileNotFoundError(f"Webhook secret file not found at {WEBHOOK_PATH}")
+    # Load webhook URL
+    webhook_url = load_webhook_url_securely()
+    new_values["PIPEDREAM_WEBHOOK_URL"] = webhook_url
     # Overwrite .env file with new values
     with open(ENV_FILE_PATH, "w", encoding="utf-8") as f:
         for var, val in new_values.items():
     for i, proxy in enumerate(lines, start=1):
         proxies[f"PROXY_{i}"] = proxy
+    return proxies
+# Get the size of a folder
+def get_folder_size(folder_path: str) -> int:
+    total_size = 0
+    if not os.path.exists(folder_path):
+        return 0
+    for entry in os.scandir(folder_path):
+        if entry.is_file():
+            total_size += entry.stat().st_size
+    return total_size
+# Clear a given folder
+def clear_folder(folder_path: str):
+    if os.path.exists(folder_path):
+        try:
+            shutil.rmtree(folder_path)
+            print(f"Successfully cleared upload directory: {folder_path}")
+        except Exception as e:
+            print(f'Failed to delete {folder_path}. Reason: {e}')

src/integrations/mcp_client.py ADDED Viewed

	@@ -0,0 +1,506 @@

+import os
+import json
+import asyncio
+import aiohttp
+import logging
+import requests
+from typing import List, Dict, Any
+from datetime import datetime, timezone
+logger = logging.getLogger(__name__)
+# Client for interacting with the MCP service
+class MCPClient:
+    def __init__(self):
+        self.webhook_url = os.getenv("PIPEDREAM_WEBHOOK_URL")
+        if not self.webhook_url:
+            logger.warning("PIPEDREAM_WEBHOOK_URL not set in environment variables")
+        # Set timeout for requests
+        self.timeout = aiohttp.ClientTimeout(total=60)
+    # Fetch app data from MCP service
+    async def fetch_app_data(
+        self,
+        provider: str,
+        services: List[str],
+        query: str,
+        user_id: str,
+        access_token: str
+    ) -> Dict[str, Any]:
+        if not self.webhook_url:
+            logger.error("Pipedream webhook URL not configured")
+            return {"error": "Pipedream integration not configured"}
+        # Add debugging
+        print(f"=== MCP fetch_app_data called ===")
+        print(f"Provider: {provider}")
+        print(f"Services: {services}")
+        print(f"Query: {query}")
+        print(f"User ID: {user_id}")
+        print(f"Access token exists: {bool(access_token)}")
+        print(f"Access token length: {len(access_token) if access_token else 0}")
+        print("==================================")
+        # Check if token is None
+        if not access_token:
+            logger.error(f"No access token for {provider}! Cannot proceed.")
+            return {"error": f"No authentication token for {provider}"}
+        payload = {
+            "provider": provider,
+            "services": services,
+            "query": query,
+            "user_id": user_id,
+            "token": access_token,
+            "timestamp": datetime.now(timezone.utc).isoformat()
+        }
+        # Manually set headers
+        headers = {'Content-Type': 'application/json'}
+        print(f"Fetching {provider} data for services: {services}")
+        print(f"Payload to send: {json.dumps({**payload, 'token': 'REDACTED' if payload['token'] else None}, indent=2)}")
+        response = requests.post(self.webhook_url, json={})
+        print("Status Code:", response.status_code)
+        print("Response text:", response.text)
+        try:
+            print("Parsed JSON:", response.json())
+        except Exception as e:
+            print("JSON parse error:", e)
+        # try:
+        #     async with aiohttp.ClientSession(timeout=self.timeout) as session:
+        #         payload_str = json.dumps(payload)
+        #         async with session.post(self.webhook_url, json={'data': payload_str}, headers=headers) as response:
+        #             print(f"Response status: {response.status}")
+        #             print(f"Response headers: {dict(response.headers)}")
+        #             if response.status == 200:
+        #                 # Handle potential empty or null responses from Pipedream
+        #                 try:
+        #                     data = await response.json()
+        #                     print(f"Received data: {json.dumps(data, indent=2)}")
+        #                     if data is None:
+        #                         logger.warning("Pipedream returned a null response.")
+        #                         return {"error": "Received no data from the provider."}
+        #                 except aiohttp.ContentTypeError:
+        #                     logger.error("Pipedream returned a non-JSON or empty response.")
+        #                     return {"error": "Invalid response from the provider."}
+        #                 # Check if any service within the data returned an auth error
+        #                 auth_error = False
+        #                 for service_key in data:
+        #                     if isinstance(data[service_key], dict) and data[service_key].get('error'):
+        #                         error_details = data[service_key].get('details', '')
+        #                         if '401' in str(error_details) or 'authError' in str(error_details) or 'UNAUTHENTICATED' in str(error_details):
+        #                             auth_error = True
+        #                             break
+        #                 if auth_error:
+        #                     logger.error(f"Authentication failed for {provider}")
+        #                     return {"error": f"Authentication failed for {provider}. Please reconnect your account."}
+        #                 logger.info(f"Successfully fetched {provider} data")
+        #                 return data
+        #             else:
+        #                 error_text = await response.text()
+        #                 logger.error(f"Pipedream request failed: {response.status} - {error_text}")
+        #                 return {"error": f"Failed to fetch data: {response.status} - {error_text}"}
+        # except asyncio.TimeoutError:
+        #     logger.error("Pipedream request timed out")
+        #     return {"error": "Request timed out. Please try again."}
+        # except aiohttp.ClientError as e:
+        #     logger.error(f"Network error calling Pipedream: {str(e)}")
+        #     return {"error": "Network error. Please check your connection."}
+        # except Exception as e:
+        #     # Catching TypeError if data is None from response.json()
+        #     logger.error(f"Unexpected error calling Pipedream: {str(e)}")
+        #     return {"error": "An unexpected error occurred"}
+    # Format the raw app data into a context string for LLM
+    def format_as_context(self, provider: str, data: Dict[str, Any]) -> str:
+        if not data or "error" in data:
+            return ""
+        context = f"\n[{provider.upper()} APP DATA]\n"
+        context += "=" * 50 + "\n"
+        # Format based on provider
+        if provider == "google":
+            context += self._format_google_data(data)
+        elif provider == "microsoft":
+            context += self._format_microsoft_data(data)
+        elif provider == "slack":
+            context += self._format_slack_data(data)
+        else:
+            context += f"Unknown provider: {provider}\n"
+        context += "=" * 50 + "\n"
+        return context
+    # Helper methods to format data for Google apps
+    def _format_google_data(self, data: Dict[str, Any]) -> str:
+        formatted = ""
+        # Google Drive
+        if "drive" in data and isinstance(data["drive"], dict) and "files" in data["drive"]:
+            formatted += "\n📁 GOOGLE DRIVE FILES:\n"
+            formatted += "-" * 30 + "\n"
+            files = data["drive"]["files"]
+            if not files:
+                formatted += "No files found matching the query.\n"
+            else:
+                for i, file in enumerate(files[:10], 1):  # Limit to 10 files
+                    formatted += f"\n{i}. File: {file.get('name', 'Unknown')}\n"
+                    formatted += f"   Type: {file.get('mimeType', 'Unknown')}\n"
+                    formatted += f"   Modified: {file.get('modifiedTime', 'Unknown')}\n"
+                    if file.get('webViewLink'):
+                        formatted += f"   Link: {file['webViewLink']}\n"
+                    if file.get('content'):
+                        content_preview = file['content'][:500]
+                        if len(file['content']) > 500:
+                            content_preview += "..."
+                        formatted += f"   Content Preview:\n   {content_preview}\n"
+                    formatted += "\n"
+        # Gmail
+        if "gmail" in data and isinstance(data["gmail"], dict) and "messages" in data["gmail"]:
+            formatted += "\n📧 GMAIL MESSAGES:\n"
+            formatted += "-" * 30 + "\n"
+            messages = data["gmail"]["messages"]
+            if not messages:
+                formatted += "No messages found matching the query.\n"
+            else:
+                for i, msg in enumerate(messages[:10], 1):
+                    formatted += f"\n{i}. From: {msg.get('from', 'Unknown')}\n"
+                    formatted += f"   Subject: {msg.get('subject', 'No subject')}\n"
+                    body_preview = msg.get('body', '')[:300]
+                    if msg.get('body', '') and len(msg['body']) > 300:
+                        body_preview += "..."
+                    formatted += f"   Preview: {body_preview}\n"
+        # Google Calendar
+        if "calendar" in data and isinstance(data["calendar"], dict) and "events" in data["calendar"]:
+            formatted += "\n📅 GOOGLE CALENDAR EVENTS:\n"
+            formatted += "-" * 30 + "\n"
+            events = data["calendar"]["events"]
+            if not events:
+                formatted += "No calendar events found matching the query.\n"
+            else:
+                for i, event in enumerate(events[:10], 1):
+                    formatted += f"\n{i}. Event: {event.get('summary', 'No title')}\n"
+                    formatted += f"   Time: {event.get('start', 'Unknown')}\n"
+                    if event.get('location'):
+                        formatted += f"   Location: {event['location']}\n"
+                    if event.get('description'):
+                        desc_preview = event['description'][:200]
+                        if len(event['description']) > 200:
+                            desc_preview += "..."
+                        formatted += f"   Description: {desc_preview}\n"
+        # Google Docs
+        if "docs" in data and isinstance(data["docs"], dict) and "docs" in data["docs"]:
+            formatted += "\n📄 GOOGLE DOCS:\n"
+            formatted += "-" * 30 + "\n"
+            docs = data["docs"]["docs"]
+            if not docs:
+                formatted += "No documents found matching the query.\n"
+            else:
+                for i, doc in enumerate(docs[:5], 1):
+                    formatted += f"\n{i}. Document: {doc.get('name', 'Unknown')}\n"
+                    formatted += f"   Modified: {doc.get('modifiedTime', 'Unknown')}\n"
+                    if doc.get('content'):
+                        content_preview = doc['content'][:500]
+                        if len(doc['content']) > 500:
+                            content_preview += "..."
+                        formatted += f"   Content Preview:\n   {content_preview}\n"
+        # Google Sheets
+        if "sheets" in data and isinstance(data["sheets"], dict) and "sheets" in data["sheets"]:
+            formatted += "\n📊 GOOGLE SHEETS:\n"
+            formatted += "-" * 30 + "\n"
+            sheets = data["sheets"]["sheets"]
+            if not sheets:
+                formatted += "No spreadsheets found matching the query.\n"
+            else:
+                for i, sheet in enumerate(sheets[:5], 1):
+                    formatted += f"\n{i}. Spreadsheet: {sheet.get('name', 'Unknown')}\n"
+                    formatted += f"   Modified: {sheet.get('modifiedTime', 'Unknown')}\n"
+                    if sheet.get('content'):
+                        content_preview = sheet['content'][:300]
+                        if len(sheet['content']) > 300:
+                            content_preview += "..."
+                        formatted += f"   Data Preview:\n   {content_preview}\n"
+        # Google Tasks
+        if "tasks" in data and isinstance(data["tasks"], dict) and "tasks" in data["tasks"]:
+            formatted += "\n✅ GOOGLE TASKS:\n"
+            formatted += "-" * 30 + "\n"
+            tasks = data["tasks"]["tasks"]
+            if not tasks:
+                formatted += "No tasks found matching the query.\n"
+            else:
+                for i, task in enumerate(tasks[:10], 1):
+                    formatted += f"\n{i}. Task: {task.get('title', 'No title')}\n"
+                    formatted += f"   List: {task.get('listTitle', 'Unknown')}\n"
+                    formatted += f"   Status: {task.get('status', 'Unknown')}\n"
+                    if task.get('notes'):
+                        formatted += f"   Notes: {task['notes'][:200]}...\n"
+                    if task.get('due'):
+                        formatted += f"   Due: {task['due']}\n"
+        # Add other Google services as needed
+        return formatted
+    # Helper methods to format data for Microsoft apps
+    def _format_microsoft_data(self, data: Dict[str, Any]) -> str:
+        formatted = ""
+        # Word
+        if "word" in data and isinstance(data["word"], dict) and "documents" in data["word"]:
+            formatted += "\n📄 MICROSOFT WORD DOCUMENTS:\n"
+            formatted += "-" * 30 + "\n"
+            documents = data["word"]["documents"]
+            if not documents:
+                formatted += "No documents found matching the query.\n"
+            else:
+                for i, doc in enumerate(documents[:5], 1):
+                    formatted += f"\n{i}. Document: {doc.get('name', 'Unknown')}\n"
+                    formatted += f"   Modified: {doc.get('lastModifiedDateTime', 'Unknown')}\n"
+                    if doc.get('content'):
+                        content_preview = doc['content'][:500]
+                        if len(doc['content']) > 500:
+                            content_preview += "..."
+                        formatted += f"   Content Preview:\n   {content_preview}\n"
+        # Excel
+        if "excel" in data and isinstance(data["excel"], dict) and "workbooks" in data["excel"]:
+            formatted += "\n📊 MICROSOFT EXCEL WORKBOOKS:\n"
+            formatted += "-" * 30 + "\n"
+            workbooks = data["excel"]["workbooks"]
+            if not workbooks:
+                formatted += "No workbooks found matching the query.\n"
+            else:
+                for i, wb in enumerate(workbooks[:5], 1):
+                    formatted += f"\n{i}. Workbook: {wb.get('name', 'Unknown')}\n"
+                    formatted += f"   Modified: {wb.get('lastModifiedDateTime', 'Unknown')}\n"
+                    if wb.get('content'):
+                        content_preview = wb['content'][:500]
+                        if len(wb['content']) > 500:
+                            content_preview += "..."
+                        formatted += f"   Content Preview:\n   {content_preview}\n"
+        # PowerPoint
+        if "powerpoint" in data and isinstance(data["powerpoint"], dict) and "presentations" in data["powerpoint"]:
+            formatted += "\n📊 MICROSOFT POWERPOINT PRESENTATIONS:\n"
+            formatted += "-" * 30 + "\n"
+            presentations = data["powerpoint"]["presentations"]
+            if not presentations:
+                formatted += "No presentations found matching the query.\n"
+            else:
+                for i, pres in enumerate(presentations[:5], 1):
+                    formatted += f"\n{i}. Presentation: {pres.get('name', 'Unknown')}\n"
+                    formatted += f"   Modified: {pres.get('lastModifiedDateTime', 'Unknown')}\n"
+                    if pres.get('content'):
+                        content_preview = pres['content'][:500]
+                        if len(pres['content']) > 500:
+                            content_preview += "..."
+                        formatted += f"   Content Preview:\n   {content_preview}\n"
+        # OneDrive/Files
+        if "onedrive" in data and isinstance(data["onedrive"], dict) and "files" in data["onedrive"]:
+            formatted += "\n📁 ONEDRIVE FILES:\n"
+            formatted += "-" * 30 + "\n"
+            files = data["onedrive"]["files"]
+            if not files:
+                formatted += "No files found matching the query.\n"
+            else:
+                for i, file in enumerate(files[:10], 1):
+                    formatted += f"\n{i}. File: {file.get('name', 'Unknown')}\n"
+                    formatted += f"   Modified: {file.get('lastModified', 'Unknown')}\n"
+                    if file.get('webUrl'):
+                        formatted += f"   URL: {file['webUrl']}\n"
+                    if file.get('content'):
+                        content_preview = file['content'][:500]
+                        if len(file['content']) > 500:
+                            content_preview += "..."
+                        formatted += f"   Content Preview:\n   {content_preview}\n"
+        # Outlook
+        if "outlook" in data and isinstance(data["outlook"], dict) and "messages" in data["outlook"]:
+            formatted += "\n📧 OUTLOOK MESSAGES:\n"
+            formatted += "-" * 30 + "\n"
+            messages = data["outlook"]["messages"]
+            if not messages:
+                formatted += "No messages found matching the query.\n"
+            else:
+                for i, msg in enumerate(messages[:10], 1):
+                    formatted += f"\n{i}. From: {msg.get('from', 'Unknown')}\n"
+                    formatted += f"   Subject: {msg.get('subject', 'No subject')}\n"
+                    body_preview = msg.get('body', '')[:300]
+                    if msg.get('body', '') and len(msg['body']) > 300:
+                        body_preview += "..."
+                    formatted += f"   Preview: {body_preview}\n"
+        # OneNote
+        if "onenote" in data and isinstance(data["onenote"], dict) and "pages" in data["onenote"]:
+            formatted += "\n📓 ONENOTE PAGES:\n"
+            formatted += "-" * 30 + "\n"
+            pages = data["onenote"]["pages"]
+            if not pages:
+                formatted += "No pages found matching the query.\n"
+            else:
+                for i, page in enumerate(pages[:10], 1):
+                    formatted += f"\n{i}. Page: {page.get('title', 'Unknown')}\n"
+                    formatted += f"   Section: {page.get('parentSection', 'Unknown')}\n"
+                    formatted += f"   Modified: {page.get('lastModifiedDateTime', 'Unknown')}\n"
+                    if page.get('contentPreview'):
+                        formatted += f"   Preview: {page['contentPreview'][:200]}...\n"
+        # Microsoft To Do
+        if "todo" in data and isinstance(data["todo"], dict) and "tasks" in data["todo"]:
+            formatted += "\n✅ MICROSOFT TO DO:\n"
+            formatted += "-" * 30 + "\n"
+            tasks = data["todo"]["tasks"]
+            if not tasks:
+                formatted += "No tasks found matching the query.\n"
+            else:
+                for i, task in enumerate(tasks[:10], 1):
+                    formatted += f"\n{i}. Task: {task.get('title', 'No title')}\n"
+                    formatted += f"   List: {task.get('listName', 'Unknown')}\n"
+                    formatted += f"   Status: {'Completed' if task.get('isCompleted') else 'Pending'}\n"
+                    if task.get('body', {}).get('content'):
+                        formatted += f"   Notes: {task['body']['content'][:200]}...\n"
+                    if task.get('dueDateTime'):
+                        formatted += f"   Due: {task['dueDateTime']['dateTime']}\n"
+        # Exchange Calendar
+        if "exchange" in data and isinstance(data["exchange"], dict) and "events" in data["exchange"]:
+            formatted += "\n📅 EXCHANGE CALENDAR:\n"
+            formatted += "-" * 30 + "\n"
+            events = data["exchange"]["events"]
+            if not events:
+                formatted += "No calendar events found matching the query.\n"
+            else:
+                for i, event in enumerate(events[:10], 1):
+                    formatted += f"\n{i}. Event: {event.get('subject', 'No subject')}\n"
+                    formatted += f"   Start: {event.get('start', {}).get('dateTime', 'Unknown')}\n"
+                    formatted += f"   End: {event.get('end', {}).get('dateTime', 'Unknown')}\n"
+                    if event.get('location', {}).get('displayName'):
+                        formatted += f"   Location: {event['location']['displayName']}\n"
+                    if event.get('bodyPreview'):
+                        formatted += f"   Preview: {event['bodyPreview'][:200]}...\n"
+        return formatted
+    # Helper methods to format data for Slack
+    def _format_slack_data(self, data: Dict[str, Any]) -> str:
+        formatted = "\n💬 SLACK MESSAGES:\n"
+        formatted += "-" * 30 + "\n"
+        if "messages" in data and isinstance(data["messages"], list):
+            messages = data["messages"]
+            if not messages:
+                formatted += "No messages found matching the query.\n"
+            else:
+                for i, msg in enumerate(messages[:15], 1):
+                    formatted += f"\n{i}. User: {msg.get('user', 'Unknown')}\n"
+                    formatted += f"   Channel: #{msg.get('channel', 'Unknown')}\n"
+                    formatted += f"   Message: {msg.get('text', '')}\n"
+                    if msg.get('ts'):
+                        # Convert timestamp to readable format if needed
+                        formatted += f"   Time: {msg['ts']}\n"
+        # Slack channels with messages
+        if "channels" in data and isinstance(data["channels"], list):
+            formatted += "\n📢 SLACK CHANNELS:\n"
+            formatted += "-" * 30 + "\n"
+            for channel in data["channels"][:10]:
+                formatted += f"\nChannel: #{channel.get('name', 'Unknown')}\n"
+                if channel.get('messages'):
+                    for msg in channel['messages'][:5]:
+                        formatted += f"  • {msg.get('user', 'Unknown')}: {msg.get('text', '')}\n"
+        # Slack files
+        if "files" in data and isinstance(data["files"], list):
+            formatted += "\n📎 SLACK FILES:\n"
+            formatted += "-" * 30 + "\n"
+            files = data["files"]
+            if not files:
+                formatted += "No files found matching the query.\n"
+            else:
+                for i, file in enumerate(files[:10], 1):
+                    formatted += f"\n{i}. File: {file.get('name', 'Unknown')}\n"
+                    formatted += f"   Type: {file.get('mimetype', 'Unknown')}\n"
+                    formatted += f"   Size: {self._format_file_size(file.get('size', 0))}\n"
+                    if file.get('preview'):
+                        formatted += f"   Preview: {file['preview'][:200]}...\n"
+        return formatted
+    # Helper method to format file sizes
+    def _format_file_size(self, size_bytes: int) -> str:
+        if size_bytes < 1024:
+            return f"{size_bytes} B"
+        elif size_bytes < 1024 * 1024:
+            return f"{size_bytes / 1024:.1f} KB"
+        elif size_bytes < 1024 * 1024 * 1024:
+            return f"{size_bytes / (1024 * 1024):.1f} MB"
+        else:
+            return f"{size_bytes / (1024 * 1024 * 1024):.1f} GB"
+if __name__ == "__main__":
+    # Example usage
+    client = MCPClient()
+    print(asyncio.run(client.fetch_app_data(
+        provider="google",
+        services=["gmail"],
+        query="summarize the information from my last 5 emails",
+        user_id="4c1d92c5-ecec-45d5-b8fd-cb0ce5292403",
+        access_token="ya29.a0AS3H6Nw9WnmYv7goOaxsZiwm6qDdaQq4h6tLwD69VVFPa6s7wwPYtzV3EgPIQHMnW_xRIpbcsDzTNmeOs-8gKhnB0RoW27Kuvv75eWcRed5BcWa08JWH5FFeNoSvzr_lZswEV1PZ4e5R4xNXSrtWmV4vJ-UPmwG48HIZn2lkaCgYKAW8SARcSFQHGX2MiUPjegzd64tClSBXeJNUPvw0175"
+    )))

src/query_processing/query_processor.py CHANGED Viewed

@@ -1,5 +1,6 @@
 import spacy
 import json
 from langchain.prompts import ChatPromptTemplate
 from src.utils.api_key_manager import with_api_manager
 from src.helpers.helper import remove_markdown
@@ -117,9 +118,10 @@ Your response should be in JSON format with the following structure (do not incl
 for internet search purposes, return the original query in plain text without any formatting and/or markdown.
 Intent: {intent}
-Query: {query}"""
             prompt = ChatPromptTemplate.from_template(template)
-            messages = prompt.format_messages(intent=intent, query=query)
         else:
             template = \
 """You are an expert in information retrieval and query analysis.
@@ -158,9 +160,10 @@ Your response should be in JSON format with the following structure (do not incl
 [IMPORTANT] If the query is simple or if it is not beneficial to decompose the query,
 for internet search purposes, return the original query in plain text without any formatting and/or markdown.
-Query: {query}"""
             prompt = ChatPromptTemplate.from_template(template)
-            messages = prompt.format_messages(query=query)
         try:
             response = await llm.ainvoke(messages)
@@ -288,9 +291,16 @@ Previous Queries:
 {context}
 Query:
-{query}"""
                 prompt = ChatPromptTemplate.from_template(template)
-                messages = prompt.format_messages(intent=intent, context=context, query=query)
             else:
                 template = \
 """You are an expert in information retrieval and query analysis.
@@ -342,9 +352,15 @@ Intent:
 {intent}
 Query:
-{query}"""
                 prompt = ChatPromptTemplate.from_template(template)
-                messages = prompt.format_messages(intent=intent, query=query)
         else:
             if context:
                 template = \
@@ -435,9 +451,15 @@ Previous Queries:
 {context}
 Query:
-{query}"""
                 prompt = ChatPromptTemplate.from_template(template)
-                messages = prompt.format_messages(context=context, query=query)
             else:
                 template = \
 """You are an expert in information retrieval and query analysis.
@@ -485,9 +507,14 @@ Your response should be in JSON format with the following structure (do not incl
 for internet search purposes, return the original query in plain text without any formatting and/or markdown.
 Query:
-{query}"""
                 prompt = ChatPromptTemplate.from_template(template)
-                messages = prompt.format_messages(query=query)
         try:
             response = await llm.ainvoke(messages)
@@ -550,9 +577,15 @@ Your response should ONLY contain the modified query in plain text without any f
 Query: {query}
 Context:
-{context}"""
         prompt = ChatPromptTemplate.from_template(template)
-        messages = prompt.format_messages(query=query, context=combined_context)
         response = await llm.ainvoke(messages)
         return response.content.strip()
@@ -570,10 +603,16 @@ Your analysis should be concise, detailed, and to the point. It should also cont
 [IMPORTANT] Your response should ONLY be the intent analysis of the user's query without any formatting.
 Do not include the reasoning process or the 10 versions of the query in your response.
-Original query: {query}"""
         prompt = ChatPromptTemplate.from_template(template)
-        response = await llm.ainvoke(prompt.format_messages(query=query))
         return response.content.strip()
 if __name__ == "__main__":

 import spacy
 import json
+from datetime import datetime, timezone
 from langchain.prompts import ChatPromptTemplate
 from src.utils.api_key_manager import with_api_manager
 from src.helpers.helper import remove_markdown
 for internet search purposes, return the original query in plain text without any formatting and/or markdown.
 Intent: {intent}
+Query: {query}
+Current date & time in ISO format (UTC timezone): {date}"""
             prompt = ChatPromptTemplate.from_template(template)
+            messages = prompt.format_messages(intent=intent, query=query, date=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'))
         else:
             template = \
 """You are an expert in information retrieval and query analysis.
 [IMPORTANT] If the query is simple or if it is not beneficial to decompose the query,
 for internet search purposes, return the original query in plain text without any formatting and/or markdown.
+Query: {query}
+Current date & time in ISO format (UTC timezone): {date}"""
             prompt = ChatPromptTemplate.from_template(template)
+            messages = prompt.format_messages(query=query, date=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'))
         try:
             response = await llm.ainvoke(messages)
 {context}
 Query:
+{query}
+Current date & time in ISO format (UTC timezone): {date}"""
                 prompt = ChatPromptTemplate.from_template(template)
+                messages = prompt.format_messages(
+                    intent=intent,
+                    context=context,
+                    query=query,
+                    date=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
+                )
             else:
                 template = \
 """You are an expert in information retrieval and query analysis.
 {intent}
 Query:
+{query}
+Current date & time in ISO format (UTC timezone): {date}"""
                 prompt = ChatPromptTemplate.from_template(template)
+                messages = prompt.format_messages(
+                    intent=intent,
+                    query=query,
+                    date=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
+                )
         else:
             if context:
                 template = \
 {context}
 Query:
+{query}
+Current date & time in ISO format (UTC timezone): {date}"""
                 prompt = ChatPromptTemplate.from_template(template)
+                messages = prompt.format_messages(
+                    context=context,
+                    query=query,
+                    date=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
+                )
             else:
                 template = \
 """You are an expert in information retrieval and query analysis.
 for internet search purposes, return the original query in plain text without any formatting and/or markdown.
 Query:
+{query}
+Current date & time in ISO format (UTC timezone): {date}"""
                 prompt = ChatPromptTemplate.from_template(template)
+                messages = prompt.format_messages(
+                    query=query,
+                    date=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
+                )
         try:
             response = await llm.ainvoke(messages)
 Query: {query}
 Context:
+{context}
+Current date & time in ISO format (UTC timezone): {date}"""
         prompt = ChatPromptTemplate.from_template(template)
+        messages = prompt.format_messages(
+            query=query,
+            context=combined_context,
+            date=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
+        )
         response = await llm.ainvoke(messages)
         return response.content.strip()
 [IMPORTANT] Your response should ONLY be the intent analysis of the user's query without any formatting.
 Do not include the reasoning process or the 10 versions of the query in your response.
+Original query: {query}
+Current date & time in ISO format (UTC timezone): {date}"""
         prompt = ChatPromptTemplate.from_template(template)
+        messages = prompt.format_messages(
+            query=query,
+            date=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
+        )
+        response = await llm.ainvoke(messages)
         return response.content.strip()
 if __name__ == "__main__":

src/rag/graph_rag.py CHANGED Viewed

@@ -408,7 +408,7 @@ class GraphRAG:
                 })
                 filtered_urls = await self.search_engine.filter_urls(
                     sub_query,
-                    "extensive research dynamic structure",
                     results
                 )
                 await self.emit_event("search_results_filtered", {
@@ -823,6 +823,16 @@ class GraphRAG:
                         else:
                             await self.emit_event("sub_query_failed", {"sub_query": sub_query})
                 print("Graph building complete, processing final tasks...")
                 await self.emit_event("search_process_completed", {
                     "depth": depth,
@@ -1321,17 +1331,11 @@ class GraphRAG:
         net.options["layout"] = {"improvedLayout": True}
         net.options["interaction"] = {"dragNodes": True}
-        original_dir = os.getcwd()
-        os.chdir(os.getenv("WRITABLE_DIR", "/tmp"))
         net.save_graph("temp_graph.html")
         with open("temp_graph.html", "r", encoding="utf-8") as f:
             html_str = f.read()
         os.remove("temp_graph.html")
-        os.chdir(original_dir)
         return html_str
     def verify_graph_integrity(self):
@@ -1519,7 +1523,7 @@ Present your analysis as a detailed, well-formatted report.""",
             answer = graph_search.query_graph(query)
             response = ""
-            async for chunk in reasoner.reason(query, answer):
                 response += chunk
                 print(response, end="", flush=True)

                 })
                 filtered_urls = await self.search_engine.filter_urls(
                     sub_query,
+                    "ultra",
                     results
                 )
                 await self.emit_event("search_results_filtered", {
                         else:
                             await self.emit_event("sub_query_failed", {"sub_query": sub_query})
+                for idx, (sub_query, future) in enumerate(futures.items(), 1):
+                    if future.done() and future.result().strip():
+                        print(f"Sub-query {idx} processed successfully")
+                    else:
+                        child_futures = all_child_futures.get(sub_query)
+                        if any(cf.done() and cf.result().strip() for cf in child_futures):
+                            print(f"Sub-query {idx} processed successfully because of child nodes")
+                        else:
+                            print(f"Sub-query {idx} failed to process because of child nodes")
                 print("Graph building complete, processing final tasks...")
                 await self.emit_event("search_process_completed", {
                     "depth": depth,
         net.options["layout"] = {"improvedLayout": True}
         net.options["interaction"] = {"dragNodes": True}
         net.save_graph("temp_graph.html")
         with open("temp_graph.html", "r", encoding="utf-8") as f:
             html_str = f.read()
         os.remove("temp_graph.html")
         return html_str
     def verify_graph_integrity(self):
             answer = graph_search.query_graph(query)
             response = ""
+            async for chunk in reasoner.answer(query, answer):
                 response += chunk
                 print(response, end="", flush=True)

src/rag/neo4j_graphrag.py CHANGED Viewed

@@ -12,7 +12,7 @@ from src.query_processing.query_processor import QueryProcessor
 from src.reasoning.reasoner import Reasoner
 from src.utils.api_key_manager import APIKeyManager
 from src.search.search_engine import SearchEngine
-from src.crawl.crawler import CustomCrawler #, Crawler
 from sentence_transformers import SentenceTransformer
 from bert_score.scorer import BERTScorer
 import numpy as np
@@ -51,7 +51,7 @@ class Neo4jGraphRAG:
             model_type="roberta-base",
             lang="en",
             rescale_with_baseline=True,
-            device= "cpu" # "cuda" if torch.cuda.is_available() else "cpu"
         )
         # Counters and tracking
@@ -705,7 +705,7 @@ class Neo4jGraphRAG:
                 # Filter the URLs based on the query
                 filtered_urls = await self.search_engine.filter_urls(
                     sub_query,
-                    "extensive research dynamic structure",
                     results
                 )
                 # Emit an event with the filtered URLs
@@ -785,6 +785,11 @@ class Neo4jGraphRAG:
             except Exception as e:
                 print(f"Error processing node {node_id}: {str(e)}")
                 future.set_exception(e)
                 raise
@@ -829,12 +834,22 @@ class Neo4jGraphRAG:
                             self, node_id, modified_query, session_id, future, depth, max_tokens_allowed
                             )
                 except Exception as e:
                     if not future.done():
                         future.set_exception(e)
                     raise
             except Exception as e:
                 print(f"Error processing dependent node {node_id}: {str(e)}")
                 if not future.done():
                     future.set_exception(e)
                 raise
@@ -1159,6 +1174,12 @@ class Neo4jGraphRAG:
             # Process completion tasks
             if depth == 0:
                 print("Graph building complete, processing final tasks...")
                 # Create cross-connections
                 create_cross_connections(self)
                 print("All cross-connections have been created!")
@@ -1180,6 +1201,8 @@ class Neo4jGraphRAG:
                                 node1, node2, query, threshold
                             )
     async def process_graph(
             self,
             query: str,
@@ -1763,6 +1786,7 @@ class Neo4jGraphRAG:
     def prune_edges(self, max_edges: int = 1000):
         """Prune excess edges while preserving node data."""
         try:
             with self.transaction() as tx:
                 try:
                     # Count current edges
@@ -1811,6 +1835,8 @@ class Neo4jGraphRAG:
                         tx.commit()
                         print(f"Pruned edges. Kept top {max_edges} edges by weight.")
                 except Exception as e:
                     tx.rollback()
                     raise e
@@ -2153,7 +2179,7 @@ Present your analysis as a detailed, well-formatted report.""",
     - Evaluate the differences in economy, trade, and military
     - Evaluate the differences in technology and infrastructure
 2. What were the similarities between the two civilizations?
-    - Evaluate the similarities in governance, society, and culture
     - Evaluate the similarities in economy, trade, and military
     - Evaluate the similarities in technology and infrastructure
 3. How did these two civilizations influence each other?
@@ -2186,7 +2212,7 @@ Present your analysis as a detailed, well-formatted report.""",
         # Query the graph and generate a response
         answer = graph_search.query_graph(query)
         response = ""
-        async for chunk in reasoner.reason(query, answer):
             response += chunk
             print(response, end="", flush=True)

 from src.reasoning.reasoner import Reasoner
 from src.utils.api_key_manager import APIKeyManager
 from src.search.search_engine import SearchEngine
+from src.crawl.crawler import Crawler, CustomCrawler
 from sentence_transformers import SentenceTransformer
 from bert_score.scorer import BERTScorer
 import numpy as np
             model_type="roberta-base",
             lang="en",
             rescale_with_baseline=True,
+            device= "cuda" if torch.cuda.is_available() else "cpu"
         )
         # Counters and tracking
                 # Filter the URLs based on the query
                 filtered_urls = await self.search_engine.filter_urls(
                     sub_query,
+                    "ultra",
                     results
                 )
                 # Emit an event with the filtered URLs
             except Exception as e:
                 print(f"Error processing node {node_id}: {str(e)}")
+                if depth == 0:
+                    await self.emit_event("sub_query_failed", {
+                        "node_id": node_id,
+                        "sub_query": sub_query
+                    })
                 future.set_exception(e)
                 raise
                             self, node_id, modified_query, session_id, future, depth, max_tokens_allowed
                             )
                 except Exception as e:
+                    if depth == 0:
+                        await self.emit_event("sub_query_failed", {
+                            "node_id": node_id,
+                            "sub_query": sub_query
+                        })
                     if not future.done():
                         future.set_exception(e)
                     raise
             except Exception as e:
                 print(f"Error processing dependent node {node_id}: {str(e)}")
+                if depth == 0:
+                    await self.emit_event("sub_query_failed", {
+                        "node_id": node_id,
+                        "sub_query": sub_query
+                    })
                 if not future.done():
                     future.set_exception(e)
                 raise
             # Process completion tasks
             if depth == 0:
                 print("Graph building complete, processing final tasks...")
+                await self.emit_event("search_process_completed", {
+                    "depth": depth,
+                    "sub_queries": sub_queries,
+                    "roles": roles
+                })
                 # Create cross-connections
                 create_cross_connections(self)
                 print("All cross-connections have been created!")
                                 node1, node2, query, threshold
                             )
+                print("All similarity-based edges have been added!")
     async def process_graph(
             self,
             query: str,
     def prune_edges(self, max_edges: int = 1000):
         """Prune excess edges while preserving node data."""
         try:
+            print(f"Pruning edges to keep top {max_edges} edges by weight...")
             with self.transaction() as tx:
                 try:
                     # Count current edges
                         tx.commit()
                         print(f"Pruned edges. Kept top {max_edges} edges by weight.")
+                    print("No pruning needed. Current edge count is within limits.")
                 except Exception as e:
                     tx.rollback()
                     raise e
     - Evaluate the differences in economy, trade, and military
     - Evaluate the differences in technology and infrastructure
 2. What were the similarities between the two civilizations?
+    - Evaluate the similarities in governance, society, and culture
     - Evaluate the similarities in economy, trade, and military
     - Evaluate the similarities in technology and infrastructure
 3. How did these two civilizations influence each other?
         # Query the graph and generate a response
         answer = graph_search.query_graph(query)
         response = ""
+        async for chunk in reasoner.answer(query, answer):
             response += chunk
             print(response, end="", flush=True)

src/reasoning/reasoner.py CHANGED Viewed

@@ -1,3 +1,4 @@
 from langchain.prompts import ChatPromptTemplate
 from langchain_core.prompts import ChatPromptTemplate
 from src.utils.api_key_manager import APIKeyManager, with_api_manager
@@ -9,43 +10,182 @@ class Reasoner:
         self.model = self.manager.get_llm()
     @with_api_manager(streaming=True)
-    async def reason(
         self,
         query,
         context=None,
         *,
         llm
     ):
         if context is None:
             template = \
-"""You are an expert at reasoning.
-Your task is to reason about the given user query and provide an answer.
-Rules:
-1. Your response should only be the answer in valid markdown format.
-2. You must use proper reasoning and logic to answer the query for your internal use but do not show your reasoning process in the response.
 Query:
-{query}"""
             prompt = ChatPromptTemplate.from_template(template)
-            messages = prompt.format_messages(query=query)
         else:
             template = \
-"""You are an expert at reasoning.
-Given the user query and the relevant context, your task is to reason and provide an answer.
-Rules:
-1. Your response should only be the answer in valid markdown format.
-2. You must use proper reasoning and logic to answer the query for your internal use but do not show your reasoning process in the response.
-3. You must not mention the context/documents provided to you in the response. Make it sound like you are the one who is answering the query.
-Context:
-[{context}]
 Query:
-{query}"""
             prompt = ChatPromptTemplate.from_template(template)
-            messages = prompt.format_messages(context=context, query=query)
         try:
             async for chunk in llm.astream(messages):
@@ -101,6 +241,72 @@ Document:
         response = await llm.ainvoke(messages)
         return response.content.strip()
 if __name__ == "__main__":
     import asyncio
     from src.crawl.crawler import Crawler
@@ -115,4 +321,5 @@ if __name__ == "__main__":
         rotate_proxy=False,
         return_html=True
     ))
-    print(contents)

+from datetime import datetime, timezone
 from langchain.prompts import ChatPromptTemplate
 from langchain_core.prompts import ChatPromptTemplate
 from src.utils.api_key_manager import APIKeyManager, with_api_manager
         self.model = self.manager.get_llm()
     @with_api_manager(streaming=True)
+    async def answer(
         self,
         query,
         context=None,
+        query_type="general",
         *,
         llm
     ):
         if context is None:
             template = \
+"""You are an AI model skilled in web search and crafting detailed, engaging, and well-structured answers.
+You excel at summarizing web pages and extracting relevant information to create professional, blog-style responses.
+Your task is to provide answers that are:
+- **Informative and relevant**: Thoroughly address the user's query.
+- **Well-structured**: Include clear headings and subheadings, and use a professional tone to present information concisely and logically.
+- **Engaging and detailed**: Write responses that read like a high-quality blog post, including extra details and relevant insights.
+- **Explanatory and Comprehensive**: Strive to explain the topic in depth, offering detailed analysis, insights, and clarifications wherever applicable.
+### Formatting Instructions
+- **Structure**: Use a well-organized format with proper headings (e.g., "## Example heading 1" or "## Example heading 2").
+Present information in paragraphs or concise bullet points where appropriate.
+- **Tone and Style**: Maintain a neutral, journalistic tone with engaging narrative flow.
+Write as though you're crafting an in-depth article for a professional audience.
+- **Markdown Usage**: Format your response with Markdown for clarity. Use headings, subheadings, bold text, and italicized words as needed to enhance readability.
+- **Length and Depth**: Provide comprehensive coverage of the topic. Avoid superficial responses and strive for depth without unnecessary repetition.
+Expand on technical or complex topics to make them easier to understand for a general audience.
+- **No main heading/title**: Start your response directly with the introduction unless asked to provide a specific title.
+- **Conclusion or Summary**: Include a concluding paragraph that synthesizes the provided information or suggests potential next steps, where appropriate.
+### Special Instructions
+- If the query involves technical, historical, or complex topics, provide detailed background and explanatory sections to ensure clarity.
+- If the user provides vague input or if relevant information is missing, explain what additional details might help refine the search.
+- If no relevant information is found, say:
+"Hmm, sorry I could not find any relevant information on this topic. Would you like me to search again or ask something else?"
+Be transparent about limitations and suggest alternatives or ways to reframe the query.
+### User instructions
+- These instructions are shared to you by the user as part of the query itself.
+- You will have to follow them and give them higher priority than the above instructions.
+- If the user has provided specific instructions or preferences, incorporate them into your response while adhering to the overall guidelines.
+- If no instructions are provided, follow the general guidelines and instructions above.
+### Example Output
+- Begin with a brief introduction summarizing the event or query topic.
+- Follow with detailed sections under clear headings, covering all aspects of the query if possible.
+- Provide explanations or historical context as needed to enhance understanding.
+- End with a conclusion or overall perspective if relevant.
 Query:
+{query}
+Current date & time in ISO format (UTC timezone): {date}"""
             prompt = ChatPromptTemplate.from_template(template)
+            messages = prompt.format_messages(query=query, date=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'))
+        elif query_type == "basic" and "[USER PROVIDED" in context:
+            template = \
+"""You are an AI model skilled in web search and crafting detailed, engaging, and well-structured answers.
+You excel at summarizing web pages and extracting relevant information to create professional, blog-style responses.
+Your task is to provide answers that are:
+- **Informative and relevant**: Thoroughly address the user's query.
+- **Well-structured**: Include clear headings and subheadings, and use a professional tone to present information concisely and logically.
+- **Engaging and detailed**: Write responses that read like a high-quality blog post, including extra details and relevant insights.
+- **Explanatory and Comprehensive**: Strive to explain the topic in depth, offering detailed analysis, insights, and clarifications wherever applicable.
+### Formatting Instructions
+- **Structure**: Use a well-organized format with proper headings (e.g., "## Example heading 1" or "## Example heading 2").
+Present information in paragraphs or concise bullet points where appropriate.
+- **Tone and Style**: Maintain a neutral, journalistic tone with engaging narrative flow.
+Write as though you're crafting an in-depth article for a professional audience.
+- **Markdown Usage**: Format your response with Markdown for clarity. Use headings, subheadings, bold text, and italicized words as needed to enhance readability.
+- **Length and Depth**: Provide comprehensive coverage of the topic. Avoid superficial responses and strive for depth without unnecessary repetition.
+Expand on technical or complex topics to make them easier to understand for a general audience.
+- **No main heading/title**: Start your response directly with the introduction unless asked to provide a specific title.
+- **Conclusion or Summary**: Include a concluding paragraph that synthesizes the provided information or suggests potential next steps, where appropriate.
+### Special Instructions
+- If the query involves technical, historical, or complex topics, provide detailed background and explanatory sections to ensure clarity.
+- If the user provides vague input or if relevant information is missing, explain what additional details might help refine the search.
+- All user-provided files and/or links must be given higher priority to those sources when crafting the response.
+- If no relevant information is found, say:
+"Hmm, sorry I could not find any relevant information on this topic. Would you like me to search again or ask something else?"
+Be transparent about limitations and suggest alternatives or ways to reframe the query.
+### User instructions
+- These instructions are shared to you by the user as part of the query itself.
+- You will have to follow them and give them higher priority than the above instructions.
+- If the user has provided specific instructions or preferences, incorporate them into your response while adhering to the overall guidelines.
+- If no instructions are provided, follow the general guidelines and instructions above.
+### Example Output
+- Begin with a brief introduction summarizing the event or query topic.
+- Follow with detailed sections under clear headings, covering all aspects of the query if possible.
+- Provide explanations or historical context as needed to enhance understanding.
+- End with a conclusion or overall perspective if relevant.
+Context:
+{context}
+Query:
+{query}
+Current date & time in ISO format (UTC timezone): {date}"""
+            prompt = ChatPromptTemplate.from_template(template)
+            messages = prompt.format_messages(context=context, query=query, date=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'))
         else:
             template = \
+"""You are an AI model skilled in web search and crafting detailed, engaging, and well-structured answers.
+You excel at summarizing web pages and extracting relevant information to create professional, blog-style responses.
+Your task is to provide answers that are:
+- **Informative and relevant**: Thoroughly address the user's query using the given context.
+- **Well-structured**: Include clear headings and subheadings, and use a professional tone to present information concisely and logically.
+- **Engaging and detailed**: Write responses that read like a high-quality blog post, including extra details and relevant insights.
+- **Cited and credible**: Use inline citations with [number] notation to refer to the context source(s) for each fact or detail included.
+- **Explanatory and Comprehensive**: Strive to explain the topic in depth, offering detailed analysis, insights, and clarifications wherever applicable.
+### Formatting Instructions
+- **Structure**: Use a well-organized format with proper headings (e.g., "## Example heading 1" or "## Example heading 2").
+Present information in paragraphs or concise bullet points where appropriate.
+- **Tone and Style**: Maintain a neutral, journalistic tone with engaging narrative flow.
+Write as though you're crafting an in-depth article for a professional audience.
+- **Markdown Usage**: Format your response with Markdown for clarity. Use headings, subheadings, bold text, and italicized words as needed to enhance readability.
+- **Length and Depth**: Provide comprehensive coverage of the topic. Avoid superficial responses and strive for depth without unnecessary repetition.
+Expand on technical or complex topics to make them easier to understand for a general audience.
+- **No main heading/title**: Start your response directly with the introduction unless asked to provide a specific title.
+- **Conclusion or Summary**: Include a concluding paragraph that synthesizes the provided information or suggests potential next steps, where appropriate.
+### [IMPORTANT] Citation Requirements
+- Cite every single fact, statement, or sentence using [number] notation corresponding to the source from the provided `context`.
+Each source in the `context` will be in the following format, where N is the source number:-
+[SOURCE N START]
+source content...
+[SOURCE N END]
+- Integrate citations naturally at the end of sentences or clauses as appropriate.
+For example, "The Eiffel Tower is one of the most visited landmarks in the world[1]."
+- [IMPORTANT] If applicable, use multiple sources for a single detail, such as, "Paris is a cultural hub, attracting millions of visitors annually[1][2]."
+*DO NOT* use two numbers in the same citation marker, e.g., [1,2] is *NOT* valid.
+- Always prioritize credibility and accuracy by linking all statements back to their respective context sources.
+- Avoid citing unsupported assumptions or personal interpretations; if no source supports a statement, clearly indicate the limitation.
+### Special Instructions
+- If the query involves technical, historical, or complex topics, provide detailed background and explanatory sections to ensure clarity.
+- If the user provides vague input or if relevant information is missing, explain what additional details might help refine the search.
+- If the context contains any user-provided files and/or links, ensure to give higher priority to those sources when crafting the response.
+- If no relevant information is found, say:
+"Hmm, sorry I could not find any relevant information on this topic. Would you like me to search again or ask something else?"
+Be transparent about limitations and suggest alternatives or ways to reframe the query.
+### User instructions
+- These instructions are shared to you by the user as part of the query itself.
+- You will have to follow them and give them higher priority than the above instructions.
+- If the user has provided specific instructions or preferences, incorporate them into your response while adhering to the overall guidelines.
+- If no instructions are provided, follow the general guidelines and instructions above.
+### Example Output
+- Begin with a brief introduction summarizing the event or query topic.
+- Follow with detailed sections under clear headings, covering all aspects of the query if possible.
+- Provide explanations or historical context as needed to enhance understanding.
+- End with a conclusion or overall perspective if relevant.
+Context:
+{context}
 Query:
+{query}
+Current date & time in ISO format (UTC timezone): {date}"""
             prompt = ChatPromptTemplate.from_template(template)
+            messages = prompt.format_messages(context=context, query=query, date=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'))
         try:
             async for chunk in llm.astream(messages):
         response = await llm.ainvoke(messages)
         return response.content.strip()
+    @with_api_manager()
+    async def get_excerpts(
+        self,
+        answer_text,
+        source_docs,
+        *,
+        llm
+    ):
+        template= \
+"""You are an expert at generating excerpts from long documents.
+Your task is to find and extract the most relevant, contiguous sentence(s) or short passage from the Source Documents that directly supports the Answer Text.
+The Source Documents are formatted with markers like [SOURCE N START] and [SOURCE N END], where N is the source number.
+The Answer Text uses citation markers like [N], where N directly corresponds to the source number N in the Source Documents.
+In case of multiple citations, the Answer Text's citation markers will be like [N][M][...etc] (or in some cases, [N, M, ...etc]).
+[IMPORTANT] Rules:
+1. You must carefully read and analyse the Answer Text and the Source Documents.
+2. The excerpts should be concise but detailed, precise and accurate.
+3. Focus on extracting key information, facts, and data that are directly relevant to the answer.
+4. Include specific details, numbers, and quotes when they are important.
+5. Ensure the excerpts are verbatim and extracted directly from the context without any paraphrasing or alteration.
+6. Your output should be a valid python list as shown in the output format below.
+7. If you cannot find any relevant excerpts, say "Excerpt not found".
+Output Format:
+[
+    {{<statement 1>: {{<source number>: <extracted excerpt 1>,
+                        <source number>: <extracted excerpt 2>,
+                        and so on...}}
+    }},
+    {{<statement 2>: {{<source number>: <extracted excerpt 1>,
+                        <source number>: <extracted excerpt 2>,
+                        and so on...}}
+    }},
+    ...and so on
+]
+Example Output:
+[
+    {{"The Treaty of Waitangi is a foundational document in New Zealand's history.": {{
+            1: "The Treaty of Waitangi, signed in 1840, is considered the founding document of New Zealand."
+        }}
+    }},
+    {{"Signed in 1840, the principles of the Treaty are often debated.": {{
+            1: "The Treaty of Waitangi, signed in 1840, is considered the founding document of New Zealand.",
+            2: "The principles of the Treaty are often debated in legal and political contexts."
+        }}
+    }},
+    {{"The Treaty can arguably lead to a civil war in New Zealand.": {{
+            "NA": "Excerpt not found"
+        }}
+    }}
+]
+Source Documents:
+{source_docs}
+Answer Text:
+{answer_text}"""
+        prompt = ChatPromptTemplate.from_template(template)
+        messages = prompt.format_messages(answer_text=answer_text, source_docs=source_docs)
+        response = await llm.ainvoke(messages)
+        return response.content.strip()
 if __name__ == "__main__":
     import asyncio
     from src.crawl.crawler import Crawler
         rotate_proxy=False,
         return_html=True
     ))
+    print(contents)

src/search/search_engine.py CHANGED Viewed

@@ -245,10 +245,10 @@ Consider factors such as:
 Rules:
 1. Rerank the URLs based on their relevance to the query according to the criteria listed above, from best match to worst match.
 2. Once reranked, select the top best matched results according to the category of the query as defined below:
-   - Simple External Lookup: Select upto 3 top best matched results
-   - Complex Moderate Decomposition: Select upto 4 top best matched results
-   - Complex Advanced Decomposition: Select upto 5 top best matched results
-   - Extensive Research Dynamic Structuring: Select upto 6 top best matched results
 3. [IMPORTANT] Select the MINIMUM number of results (based on the categories above) that are required to answer the query.
 4. The response should only contain a JSON array of objects, each containing 'link', 'title' and 'snippet' keys after reranking and filtering.
@@ -315,7 +315,7 @@ if __name__ == "__main__":
             print(f"Time taken to fetch search results: {end - start:.2f} seconds")
             # filtered_search = search_engine.filter_urls(
             #     optimized_query,
-            #     category="Simple External Lookup",
             #     search_results=search_results,
             #     num_results=2
             # )

 Rules:
 1. Rerank the URLs based on their relevance to the query according to the criteria listed above, from best match to worst match.
 2. Once reranked, select the top best matched results according to the category of the query as defined below:
+   - Advanced: Select upto 3 top best matched results
+   - Pro: Select upto 4 top best matched results
+   - Super: Select upto 5 top best matched results
+   - Ultra: Select upto 6 top best matched results
 3. [IMPORTANT] Select the MINIMUM number of results (based on the categories above) that are required to answer the query.
 4. The response should only contain a JSON array of objects, each containing 'link', 'title' and 'snippet' keys after reranking and filtering.
             print(f"Time taken to fetch search results: {end - start:.2f} seconds")
             # filtered_search = search_engine.filter_urls(
             #     optimized_query,
+            #     category="Advanced",
             #     search_results=search_results,
             #     num_results=2
             # )

src/utils/api_key_manager.py CHANGED Viewed

@@ -24,66 +24,45 @@ class APIKeyManager:
     # Define supported models
     SUPPORTED_MODELS = {
         "openai": [
-            "gpt-3.5-turbo",
-            "gpt-3.5-turbo-instruct",
-            "gpt-3.5-turbo-1106",
-            "gpt-3.5-turbo-0125",
-            "gpt-4-0314",
-            "gpt-4-0613",
-            "gpt-4",
-            "gpt-4-1106-preview",
-            "gpt-4-0125-preview",
-            "gpt-4-turbo-preview",
-            "gpt-4-turbo-2024-04-09",
-            "gpt-4-turbo",
-            "o1-mini-2024-09-12",
             "o1-mini",
-            "o1-preview-2024-09-12",
-            "o1-preview",
             "o1",
             "gpt-4o-mini-2024-07-18",
             "gpt-4o-mini",
             "chatgpt-4o-latest",
             "gpt-4o-2024-05-13",
             "gpt-4o-2024-08-06",
             "gpt-4o-2024-11-20",
-            "gpt-4o"
         ],
         "google": [
-            "gemini-1.5-flash",
-            "gemini-1.5-flash-latest",
-            "gemini-1.5-flash-exp-0827",
-            "gemini-1.5-flash-001",
-            "gemini-1.5-flash-002",
-            "gemini-1.5-flash-8b-exp-0924",
-            "gemini-1.5-flash-8b-exp-0827",
-            "gemini-1.5-flash-8b-001",
-            "gemini-1.5-flash-8b",
-            "gemini-1.5-flash-8b-latest",
-            "gemini-1.5-pro",
-            "gemini-1.5-pro-latest",
-            "gemini-1.5-pro-001",
-            "gemini-1.5-pro-002",
-            "gemini-1.5-pro-exp-0827",
-            "gemini-1.0-pro",
-            "gemini-1.0-pro-latest",
-            "gemini-1.0-pro-001",
-            "gemini-pro",
-            "gemini-exp-1114",
-            "gemini-exp-1121",
             "gemini-2.0-pro-exp-02-05",
             "gemini-2.0-flash-lite-preview-02-05",
             "gemini-2.0-flash-exp",
             "gemini-2.0-flash",
             "gemini-2.0-flash-thinking-exp-1219",
         ],
         "xai": [
-            "grok-beta",
-            "grok-vision-beta",
-            "grok-2-vision-1212",
-            "grok-2-1212"
         ],
         "anthropic": [
             "claude-3-5-sonnet-20241022",
             "claude-3-5-sonnet-latest",
             "claude-3-5-haiku-20241022",
@@ -275,14 +254,14 @@ class APIKeyManager:
         api_key = self.get_next_api_key(provider)
         print(f"Using provider={provider}, model_name={model_name}, "
               f"temperature={temperature}, top_p={top_p}, key={api_key}")
         kwargs = {
             "model": model_name,
             "temperature": temperature,
             "top_p": top_p,
             "max_retries": 0,
             "streaming": streaming,
-            "api_key": api_key,
         }
         if max_tokens is not None:
@@ -611,5 +590,5 @@ if __name__ == "__main__":
             except Exception as e:
                 raise Exception(f"Error with {model_name}: {str(e)}")
-    # test_without_load_balancing(model_name="gemini-exp-1121", prompt=prompt, test_count=50)
-    asyncio.run(test_load_balancing(prompt=prompt, test_count=100, stream=True))

     # Define supported models
     SUPPORTED_MODELS = {
         "openai": [
             "o1-mini",
             "o1",
+            "o1-pro",
+            "o3-mini",
+            "o3",
+            "o4-mini",
             "gpt-4o-mini-2024-07-18",
             "gpt-4o-mini",
             "chatgpt-4o-latest",
             "gpt-4o-2024-05-13",
             "gpt-4o-2024-08-06",
             "gpt-4o-2024-11-20",
+            "gpt-4o",
+            "gpt-4.1-nano",
+            "gpt-4.1-mini",
+            "gpt-4.1"
         ],
         "google": [
             "gemini-2.0-pro-exp-02-05",
             "gemini-2.0-flash-lite-preview-02-05",
             "gemini-2.0-flash-exp",
             "gemini-2.0-flash",
             "gemini-2.0-flash-thinking-exp-1219",
+            "gemini-2.5-flash-lite-preview-06-17",
+            "gemini-2.5-flash-preview-04-17",
+            "gemini-2.5-flash",
+            "gemini-2.5-pro"
         ],
         "xai": [
+            "grok-2",
+            "grok-3-mini-latest",
+            "grok-3-mini-fast-latest",
+            "grok-3-latest",
+            "grok-3-fast-latest"
         ],
         "anthropic": [
+            "claude-opus-4-20250514",
+            "claude-sonnet-4-20250514",
+            "claude-3-7-sonnet-20250219",
             "claude-3-5-sonnet-20241022",
             "claude-3-5-sonnet-latest",
             "claude-3-5-haiku-20241022",
         api_key = self.get_next_api_key(provider)
         print(f"Using provider={provider}, model_name={model_name}, "
               f"temperature={temperature}, top_p={top_p}, key={api_key}")
         kwargs = {
             "model": model_name,
             "temperature": temperature,
             "top_p": top_p,
             "max_retries": 0,
             "streaming": streaming,
+            "api_key": api_key
         }
         if max_tokens is not None:
             except Exception as e:
                 raise Exception(f"Error with {model_name}: {str(e)}")
+    test_without_load_balancing(model_name="gemini-2.5-flash-lite-preview-06-17", prompt=prompt, test_count=50)
+    # asyncio.run(test_load_balancing(prompt=prompt, test_count=100, stream=True))