Initial relase. Tested. Working

app.py

@@ -0,0 +1,203 @@
+import os
+import json
+import importlib
+import streamlit as st
+from typing import List, Dict, Any
+from dotenv import load_dotenv
+from global_vars import translations, t
+
+# Constantes
+CONFIG_FILE = "config.json"
+
+def load_config() -> Dict[str, Any]:
+    if os.path.exists(CONFIG_FILE):
+        with open(CONFIG_FILE, 'r') as f:
+            return json.load(f)
+    return {}
+
+def save_config(config: Dict[str, Any]):
+    with open(CONFIG_FILE, 'w') as f:
+        json.dump(config, f, indent=2)
+
+# Fonction pour mettre à jour la langue
+def set_lang(language):
+    st.session_state.lang = language
+
+# Fonction de traduction
+def t(key: str) -> str:
+    return translations[st.session_state.lang].get(key, key)
+
+class Plugin:
+    def __init__(self, name, plugin_manager):
+        self.name = name
+        self.plugin_manager = plugin_manager
+
+    def get_config_fields(self) -> Dict[str, Any]:
+        return {}
+
+    def get_config_ui(self, config):
+        updated_config = {}
+        for field, params in self.get_config_fields().items():
+            if params['type'] == 'select':
+                updated_config[field] = st.selectbox(
+                    params['label'],
+                    options=[option[0] for option in params['options']],
+                    format_func=lambda x: dict(params['options'])[x],
+                    index=[option[0] for option in params['options']].index(config.get(field, params['default']))
+                )
+            elif params['type'] == 'textarea':
+                updated_config[field] = st.text_area(
+                    params['label'],
+                    value=config.get(field, params['default'])
+                )
+            else:
+                updated_config[field] = st.text_input(
+                    params['label'],
+                    value=config.get(field, params['default']),
+                    type="password" if field.startswith("pass") else "default"
+                )
+        return updated_config
+
+    def get_tabs(self) -> List[Dict[str, Any]]:
+        return []
+
+    def run(self, config: Dict[str, Any]):
+        pass
+
+    def get_sidebar_config_ui(self, config: Dict[str, Any]) -> Dict[str, Any]:
+        return {}
+
+class PluginManager:
+    def __init__(self):
+        self.plugins: Dict[str, Plugin] = {}
+        self.starred_plugins: Set[str] = set()
+
+    def load_plugins(self):
+        plugins_dir = 'plugins'
+        for filename in os.listdir(plugins_dir):
+            if filename.endswith('.py'):
+                module_name = filename[:-3]
+                module = importlib.import_module(f'plugins.{module_name}')
+                plugin_class = getattr(module, f'{module_name.capitalize()}Plugin')
+                self.plugins[module_name] = plugin_class(module_name, self)
+
+    def get_plugin(self, plugin_name: str) -> Plugin:
+        return self.plugins.get(plugin_name)
+
+    def get_all_config_ui(self, config: Dict[str, Any]) -> Dict[str, Dict[str, Any]]:
+        all_ui = {}
+        for plugin_name, plugin in sorted(self.plugins.items()):
+             with st.expander(f"{'⭐ ' if plugin_name in self.starred_plugins else ''}{t('configurations')} {plugin_name}"):
+                all_ui[plugin_name] = plugin.get_config_ui(config.get(plugin_name, {}))
+                if st.button(f"{'Unstar' if plugin_name in self.starred_plugins else 'Star'} {plugin_name}"):
+                    if plugin_name in self.starred_plugins:
+                        self.starred_plugins.remove(plugin_name)
+                    else:
+                        self.starred_plugins.add(plugin_name)
+                    self.save_starred_plugins(config)
+                    st.rerun()
+        return all_ui
+
+    def get_all_tabs(self) -> List[Dict[str, Any]]:
+        all_tabs = []
+        for plugin_name, plugin in sorted(self.plugins.items()):
+            tabs = plugin.get_tabs()
+            for tab in tabs:
+                tab['id'] = plugin_name
+                tab['starred'] = plugin_name in self.starred_plugins
+            all_tabs.extend(tabs)
+        return all_tabs
+
+
+    def load_starred_plugins(self, config: Dict[str, Any]):
+        self.starred_plugins = set(config.get('starred_plugins', []))
+
+    def save_starred_plugins(self, config: Dict[str, Any]):
+        config['starred_plugins'] = list(self.starred_plugins)
+        save_config(config)
+
+    def run_plugin(self, plugin_name: str, config: Dict[str, Any]):
+        if plugin_name in self.plugins:
+            self.plugins[plugin_name].run(config)
+
+    def save_config(self, config):
+        save_config(config)
+
+def main():
+    st.set_page_config(page_title="Veille", layout="wide")
+    # Initialisation du gestionnaire de plugins
+    plugin_manager = PluginManager()
+    plugin_manager.load_plugins()
+
+    # Chargement de la configuration
+    config = load_config()
+    plugin_manager.load_starred_plugins(config)
+
+    # Initialisation de la langue dans st.session_state
+    if 'lang' not in st.session_state:
+        st.session_state.lang = config['common']['language']
+    st.title(t("page_title"))
+
+    load_dotenv()
+    LLM_KEY = os.getenv("LLM_API_KEY")
+    config['llm_key'] = LLM_KEY
+
+    # Création des onglets avec des identifiants uniques
+    tabs = [{"id": "configurations", "name": t("configurations")}] + [{"id": tab['plugin'], "name": tab['name'], "starred" : tab['starred']} for tab in plugin_manager.get_all_tabs()]
+
+    # Gestion de la langue
+    if 'lang' not in st.session_state:
+        st.session_state.lang = "fr"
+
+    new_lang = st.sidebar.selectbox("Choose your language / Choisissez votre langue", options=["en", "fr"], index=["en", "fr"].index(st.session_state.lang), key="lang_selector")
+
+    if new_lang != st.session_state.lang:
+        st.session_state.lang = new_lang
+        st.rerun()
+
+    # Ajout des éléments de configuration de la sidebar pour chaque plugin
+    for plugin_name, plugin in plugin_manager.plugins.items():
+        sidebar_config = plugin.get_sidebar_config_ui(config.get(plugin_name, {}))
+        if sidebar_config:
+            #st.sidebar.markdown(f"**{plugin_name} Configuration**")
+            for key, value in sidebar_config.items():
+                config.setdefault(plugin_name, {})[key] = value
+
+    # Gestion de l'onglet sélectionné
+    if 'selected_tab_id' not in st.session_state:
+        st.session_state.selected_tab_id = "configurations"
+
+    # Sort tabs alphabetically, with starred tabs first
+    sorted_tabs = sorted(tabs, key=lambda x: (not x.get('starred', False), x['name']))
+    tab_names = [f"{'⭐ ' if tab.get('starred', False) else ''}{tab['name']}" for tab in sorted_tabs]
+
+    selected_tab_index = [tab["id"] for tab in sorted_tabs].index(st.session_state.selected_tab_id)
+    selected_tab = st.sidebar.radio(t("navigation"), tab_names, index=selected_tab_index, key="tab_selector")
+
+    new_selected_tab_id = next(tab["id"] for tab in sorted_tabs if f"{'⭐ ' if tab.get('starred', False) else ''}{tab['name']}" == selected_tab)
+
+    if new_selected_tab_id != st.session_state.selected_tab_id:
+        st.session_state.selected_tab_id = new_selected_tab_id
+        st.rerun()
+
+    if st.session_state.selected_tab_id == "configurations":
+        st.header(t("configurations"))
+        all_config_ui = plugin_manager.get_all_config_ui(config)
+
+        for plugin_name, ui_config in all_config_ui.items():
+            with st.expander(f"{t('configurations')} {plugin_name}"):
+                config[plugin_name] = ui_config
+
+        if st.button(t("save_button")):
+            save_config(config)
+            st.success(t("success_message"))
+
+    else:
+        # Exécution du plugin correspondant à l'onglet sélectionné
+        for tab in plugin_manager.get_all_tabs():
+            if tab['plugin'] == st.session_state.selected_tab_id:
+                plugin_manager.run_plugin(tab['plugin'], config)
+                break
+
+if __name__ == "__main__":
+    main()

bart_ft.py

@@ -0,0 +1,89 @@
+import torch
+from torch.utils.data import Dataset, DataLoader
+from transformers import BartForSequenceClassification, BartTokenizer, get_linear_schedule_with_warmup
+from transformers import AdamW
+from tqdm import tqdm
+import gc
+from plugins.scansite import ScansitePlugin  # Assurez-vous que l'import est correct
+
+torch.cuda.empty_cache()
+
+# Définition du dataset personnalisé
+class PreferenceDataset(Dataset):
+    def __init__(self, data, tokenizer, max_length=128):
+        self.data = data
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        _, title, score = self.data[idx]
+        encoding = self.tokenizer(title, truncation=True, padding='max_length', max_length=self.max_length, return_tensors='pt')
+        # Convertir le score en label binaire (0 ou 1)
+        label = 1 if score > 0 else 0
+        return {key: val.squeeze(0) for key, val in encoding.items()}, torch.tensor(label, dtype=torch.long)
+
+# Fonction principale de finetuning
+def finetune(num_epochs=2, lr=2e-5, weight_decay=0.01, batch_size=1, model_name='facebook/bart-large-mnli', output_model='./bart-large-ft', num_warmup_steps=0):
+    print(f"Fine-tuning parameters:\n"
+          f"num_epochs: {num_epochs}\n"
+          f"learning rate (lr): {lr}\n"
+          f"weight_decay: {weight_decay}\n"
+          f"batch_size: {batch_size}\n"
+          f"model_name: {model_name}\n"
+          f"num_warmup_steps: {num_warmup_steps}")
+
+    # Récupérer les données de référence
+    scansite_plugin = ScansitePlugin("scansite", None)  # Vous devrez peut-être ajuster ceci en fonction de votre structure
+    reference_data_valid, reference_data_rejected = scansite_plugin.get_reference_data()
+
+    # Combiner les données valides et rejetées
+    all_data = reference_data_valid + [(url, title, 0) for url, title in reference_data_rejected]
+
+    tokenizer = BartTokenizer.from_pretrained(model_name)
+    model = BartForSequenceClassification.from_pretrained(model_name, num_labels=2, ignore_mismatched_sizes=True)
+
+    dataset = PreferenceDataset(all_data, tokenizer)
+    dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)
+
+    optimizer = AdamW(model.parameters(), lr=lr, weight_decay=weight_decay)
+
+    total_steps = len(dataloader) * num_epochs
+    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=total_steps)
+
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    model.to(device)
+    model.gradient_checkpointing_enable()
+
+    for epoch in range(num_epochs):
+        model.train()
+        for batch in tqdm(dataloader, desc=f"Epoch {epoch + 1}/{num_epochs}"):
+            optimizer.zero_grad()
+
+            inputs, labels = batch
+            inputs = {k: v.to(device) for k, v in inputs.items()}
+            labels = labels.to(device)
+
+            outputs = model(**inputs, labels=labels)
+            loss = outputs.loss
+
+            loss.backward()
+            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
+            optimizer.step()
+            scheduler.step()
+
+            del inputs, outputs, labels
+            torch.cuda.empty_cache()
+            gc.collect()
+            #print(f"Finetuning en cours round {epoch}/{num_epochs}")
+    print("Finetuning terminé sauvegarde en cours.")
+    model.save_pretrained(output_model)
+    tokenizer.save_pretrained(output_model)
+
+    print("Finetuning terminé et modèle sauvegardé.")
+
+# Appel par défaut si le script est exécuté directement
+if __name__ == "__main__":
+    finetune()

embeddings_ft.py

@@ -0,0 +1,95 @@
+import torch
+from torch.utils.data import Dataset, DataLoader
+from sentence_transformers import SentenceTransformer, losses
+from tqdm import tqdm
+import gc
+from plugins.scansite import ScansitePlugin
+
+torch.cuda.empty_cache()
+
+class PreferenceDataset(Dataset):
+    def __init__(self, data, tokenizer, max_length=128):
+        self.data = data
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        url, title, score = self.data[idx]
+        encoded = self.tokenizer(title, padding='max_length', truncation=True, max_length=self.max_length, return_tensors="pt")
+        return {key: val.squeeze(0) for key, val in encoded.items()}, torch.tensor(score, dtype=torch.float)
+
+def collate_fn(batch):
+    input_ids = torch.stack([item[0]['input_ids'] for item in batch])
+    attention_mask = torch.stack([item[0]['attention_mask'] for item in batch])
+    scores = torch.stack([item[1] for item in batch])
+    return {'input_ids': input_ids, 'attention_mask': attention_mask}, scores
+
+def finetune(model_name='nomic-ai/nomic-embed-text-v1', output_model_name="embeddings-ft", num_epochs=2, learning_rate=2e-5, weight_decay=0.01, batch_size=8, num_warmup_steps=0):
+    print(f"Fine-tuning parameters:\n"
+          f"num_epochs: {num_epochs}\n"
+          f"learning rate (lr): {learning_rate}\n"
+          f"weight_decay: {weight_decay}\n"
+          f"batch_size: {batch_size}\n"
+          f"model_name: {model_name}\n"
+          f"num_warmup_steps: {num_warmup_steps}")
+
+    scansite_plugin = ScansitePlugin("scansite", None)
+    reference_data_valid, reference_data_rejected = scansite_plugin.get_reference_data()
+
+    valid_data_with_scores = [(url, title, (score - 1) / 8 + 0.5) for url, title, score in reference_data_valid]
+    rejected_data_with_scores = [(url, title, 0.0) for url, title in reference_data_rejected]
+
+    all_data = valid_data_with_scores + rejected_data_with_scores
+
+    model = SentenceTransformer(model_name, trust_remote_code=True)
+    tokenizer = model.tokenizer
+
+    dataset = PreferenceDataset(all_data, tokenizer)
+    dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_fn)
+
+    loss_function = torch.nn.MSELoss()
+
+    optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate, weight_decay=weight_decay)
+
+    total_steps = len(dataloader) * num_epochs
+    scheduler = torch.optim.lr_scheduler.LinearLR(optimizer, start_factor=1.0, end_factor=0.1, total_iters=total_steps)
+
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    model.to(device)
+
+    for epoch in range(num_epochs):
+        model.train()
+        for batch in tqdm(dataloader, desc=f"Epoch {epoch + 1}/{num_epochs}"):
+            input_data, scores = batch
+            input_data = {k: v.to(device) for k, v in input_data.items()}
+            scores = scores.to(device)
+
+            optimizer.zero_grad()
+
+            embeddings = model(input_data)['sentence_embedding']
+
+            # Calcul de la similarité cosinus
+            embeddings_norm = torch.nn.functional.normalize(embeddings, p=2, dim=1)
+            cosine_similarities = torch.sum(embeddings_norm, dim=1)
+
+            # Calcul de la perte
+            loss = loss_function(cosine_similarities, scores)
+
+            loss.backward()
+            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
+            optimizer.step()
+            scheduler.step()
+
+            del embeddings, cosine_similarities
+            torch.cuda.empty_cache()
+            gc.collect()
+
+    model.save(output_model_name)
+
+    print("Finetuning terminé et modèle sauvegardé.")
+
+if __name__ == "__main__":
+    finetune()

plugins/common.py

@@ -0,0 +1,44 @@
+from global_vars import t, translations
+from app import Plugin
+import streamlit as st
+import torch
+
+# Ajout des traductions spécifiques à ce plugin
+translations["en"].update({
+    "work_directory": "Work Directory",
+    "page_title": "News watcher",
+})
+translations["fr"].update({
+    "work_directory": "Répertoire de travail",
+    "page_title": "Outil de veille",
+})
+
+class CommonPlugin(Plugin):
+    def get_config_fields(self):
+        return {
+            "work_directory": {
+                "type": "text",
+                "label": t("work_directory"),
+                "default": "/home/joriel/Vidéos"
+            },
+            "language": {
+                "type": "select",
+                "label": t("preferred_language"),
+                "options": [("fr", "Français"), ("en", "Anglais")],
+                "default": "fr"
+            },
+        }
+
+    def get_tabs(self):
+        return [{"name": "Commun", "plugin": "common"}]
+
+    def run(self, config):
+        st.header("Common Plugin")
+        st.write(f"{t('work_directory')}: {config['common']['work_directory']}")
+        torch.cuda.empty_cache()
+        st.write("CUDA memory reset")
+
+def remove_quotes(s):
+    if s.startswith('"') and s.endswith('"'):
+        return s[1:-1]
+    return s

plugins/ragllm.py

@@ -0,0 +1,306 @@
+from global_vars import translations, t
+from app import Plugin
+import streamlit as st
+import yaml
+from litellm import completion, embedding
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity, euclidean_distances, manhattan_distances
+import os
+from typing import List, Dict, Any
+import requests
+import torch
+from transformers import AutoTokenizer, AutoModel
+
+DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
+MAX_LENGTH = 512
+CHUNK_SIZE = 200  # Nombre de mots par chunk
+
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0]
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+# Ajout des traductions spécifiques à ce plugin
+translations["en"].update({
+    "rag_plugin_loaded": "RAG LLM Plugin loaded",
+    "rag_enter_text": "Enter RAG text:",
+    "rag_enter_question": "Enter your question:",
+    "rag_button_get_answer": "Get an answer",
+    "rag_success_text_processed": "RAG text processed successfully!",
+    "rag_warning_enter_text": "Please enter RAG text.",
+    "rag_warning_process_text_first": "Please process the RAG text first.",
+    "rag_warning_enter_question": "Please enter a question.",
+    "rag_answer": "Answer:",
+    "rag_citations": "Citations:",
+    "rag_model_provider": "Model Provider",
+    "rag_llm_model": "LLM Model",
+    "rag_embedder_model": "Embedding Model",
+    "rag_similarity_method": "Similarity Method",
+    "rag_llm_sys_prompt": "System prompt for LLM",
+    "rag_chunk_size": "Chunk size",
+    "rag_top_k_chunks": "Number of chunks to use",
+    "rag_default_sys_prompt": "You are an AI assistant. Your task is to analyze the provided context and answer questions based ONLY on this context. If the information is not in the context, clearly state that.",
+    "rag_error_fetching_models_ollama": "Error fetching Ollama models: ",
+    "rag_error_calling_llm": "Error calling LLM: ",
+    "rag_processing" : "Processing...",
+})
+
+translations["fr"].update({
+    "rag_plugin_loaded": "Plugin RAG LLM chargé",
+    "rag_enter_text": "Entrez le texte RAG :",
+    "rag_enter_question": "Entrez votre question :",
+    "rag_button_get_answer": "Obtenir une réponse",
+    "rag_success_text_processed": "Texte RAG traité avec succès!",
+    "rag_warning_enter_text": "Veuillez entrer du texte RAG.",
+    "rag_warning_process_text_first": "Veuillez d'abord traiter le texte RAG.",
+    "rag_warning_enter_question": "Veuillez entrer une question.",
+    "rag_answer": "Réponse :",
+    "rag_citations": "Citations :",
+    "rag_model_provider": "Fournisseur de modèle",
+    "rag_llm_model": "Modèle LLM",
+    "rag_embedder_model": "Modèle d'embedding",
+    "rag_similarity_method": "Méthode de similarité",
+    "rag_llm_sys_prompt": "Prompt système pour le LLM",
+    "rag_chunk_size": "Taille des chunks",
+    "rag_top_k_chunks": "Nombre de chunks à utiliser",
+    "rag_default_sys_prompt": "Tu es un assistant IA. Ta tâche est d'analyser le contexte fourni et de répondre aux questions en te basant UNIQUEMENT sur ce contexte. Si l'information n'est pas dans le contexte, dis-le clairement.",
+    "rag_error_fetching_models_ollama": "Erreur lors de la récupération des modèles Ollama : ",
+    "rag_error_calling_llm": "Erreur lors de l'appel au LLM : ",
+    "rag_processing" : "En cours de traitement...",
+})
+
+class RagllmPlugin(Plugin):
+    def __init__(self, name: str, plugin_manager):
+        super().__init__(name, plugin_manager)
+        self.config = self.load_llm_config()
+        self.embeddings = None
+        self.chunks = None
+
+    def load_llm_config(self) -> Dict:
+        with open('.llm-config.yml', 'r') as file:
+            return yaml.safe_load(file)
+
+    def get_tabs(self):
+        return [{"name": "RAG", "plugin": "ragllm"}]
+
+    def get_config_fields(self):
+        return {
+            "provider": {
+                "type": "select",
+                "label": t("rag_model_provider"),
+                "options": [("ollama", "Ollama"), ("groq", "Groq")],
+                "default": "ollama"
+            },
+            "llm_model": {
+                "type": "select",
+                "label": t("rag_llm_model"),
+                "options": [("none", "À charger...")],
+                "default": "ollama/qwen2"
+            },
+            "embedder": {
+                "type": "select",
+                "label": t("rag_embedder_model"),
+                "options": [
+                    ("sentence-transformers/all-MiniLM-L6-v2", "all-MiniLM-L6-v2"),
+                    ("nomic-ai/nomic-embed-text-v1.5", "nomic-embed-text-v1.5")
+                ],
+                "default": "sentence-transformers/all-MiniLM-L6-v2"
+            },
+            "similarity_method": {
+                "type": "select",
+                "label": t("rag_similarity_method"),
+                "options": [
+                    ("cosine", "Cosinus"),
+                    ("euclidean", "Distance euclidienne"),
+                    ("manhattan", "Distance de Manhattan")
+                ],
+                "default": "cosine"
+            },
+            "llm_sys_prompt": {
+                "type": "textarea",
+                "label": t("rag_llm_sys_prompt"),
+                "default": t("rag_default_sys_prompt")
+            },
+            "chunk_size": {
+                "type": "number",
+                "label": t("rag_chunk_size"),
+                "default": 200
+            },
+            "top_k": {
+                "type": "number",
+                "label": t("rag_top_k_chunks"),
+                "default": 3
+            }
+        }
+
+    def get_config_ui(self, config):
+        updated_config = {}
+        for field, params in self.get_config_fields().items():
+            if params['type'] == 'select':
+                if field == 'llm_model':
+                    provider = config.get('provider', 'ollama')
+                    models = self.get_available_models(provider)
+                    try:
+                        default_index = models.index(config.get(field, params['default']))
+                    except ValueError:
+                        default_index = 0
+                    updated_config[field] = st.selectbox(
+                        params['label'],
+                        options=models,
+                        index=default_index
+                    )
+                else:
+                    options_list = [option[0] for option in params['options']]
+                    try:
+                        default_index = options_list.index(config.get(field, params['default']))
+                    except ValueError:
+                        default_index = 0
+                    updated_config[field] = st.selectbox(
+                        params['label'],
+                        options=options_list,
+                        format_func=lambda x: dict(params['options'])[x],
+                        index=default_index
+                    )
+            elif params['type'] == 'textarea':
+                updated_config[field] = st.text_area(
+                    params['label'],
+                    value=config.get(field, params['default'])
+                )
+            elif params['type'] == 'number':
+                updated_config[field] = st.number_input(
+                    params['label'],
+                    value=int(config.get(field, params['default'])),
+                    step=1
+                )
+            else:
+                updated_config[field] = st.text_input(
+                    params['label'],
+                    value=config.get(field, params['default'])
+                )
+        return updated_config
+
+    def get_sidebar_config_ui(self, config: Dict[str, Any]) -> Dict[str, Any]:
+        available_models = self.get_available_models('ollama') + self.get_available_models('groq')
+        default_model = config.get('llm_model', available_models[0] if available_models else None)
+        selected_model = st.sidebar.selectbox(
+            t("rag_llm_model"),
+            options=available_models,
+            index=available_models.index(default_model) if default_model in available_models else 0,
+            key="ragllm_llm_model"
+        )
+        return {"llm_model": selected_model}
+
+    def get_available_models(self, provider: str) -> List[str]:
+        if provider == 'ollama':
+            try:
+                response = requests.get("http://localhost:11434/api/tags")
+                models = response.json()["models"]
+                return [f"ollama/{model['name']}" for model in models] + ["ollama/qwen2"]
+            except Exception as e:
+                st.error(f"{t('rag_error_fetching_models_ollama')}{str(e)}")
+                return ["ollama/qwen2"]
+        elif provider == 'groq':
+            return ["groq/llama3-70b-8192", "groq/mixtral-8x7b-32768"]
+        else:
+            return ["none"]
+
+    def process_rag_text(self, rag_text: str, chunk_size: int, embedder):
+        rag_text = rag_text.replace('\\n', ' ').replace('\\\'', "'")
+        mots = rag_text.split()
+        self.chunks = [' '.join(mots[i:i+chunk_size]) for i in range(0, len(mots), chunk_size)]
+        self.embeddings = np.vstack([self.get_embedding(c, embedder) for c in self.chunks])
+
+    def get_embedding(self, text: str, model: str) -> np.ndarray:
+        tokenizer = AutoTokenizer.from_pretrained(model)
+        model = AutoModel.from_pretrained(model, trust_remote_code=True).to(DEVICE)
+        inputs = tokenizer(text, padding=True, truncation=True, max_length=MAX_LENGTH, return_tensors="pt").to(DEVICE)
+        with torch.no_grad():
+            model_output = model(**inputs)
+        return mean_pooling(model_output, inputs['attention_mask']).cpu().numpy()
+
+    def calculate_similarity(self, query_embedding: np.ndarray, method: str) -> np.ndarray:
+        if method == 'cosine':
+            return cosine_similarity(query_embedding.reshape(1, -1), self.embeddings)[0]
+        elif method == 'euclidean':
+            return -euclidean_distances(query_embedding.reshape(1, -1), self.embeddings)[0]
+        elif method == 'manhattan':
+            return -manhattan_distances(query_embedding.reshape(1, -1), self.embeddings)[0]
+        else:
+            raise ValueError("Méthode de similarité non reconnue")
+
+    def get_context(self, query: str, config: Dict[str, Any]) -> tuple:
+        query_embedding = self.get_embedding(query, config['ragllm']['embedder'])
+        similarities = self.calculate_similarity(query_embedding, config['ragllm']['similarity_method'])
+        top_indices = np.argsort(similarities)[-config['ragllm']['top_k']:][::-1]
+        context = "\n\n".join([self.chunks[i] for i in top_indices])
+        return context, [self.chunks[i] for i in top_indices]
+
+    def call_llm(self, prompt: str, sysprompt: str) -> str:
+        try:
+            llm_model = st.session_state.ragllm_llm_model
+            #print(f"---------------------------------------\nCalling LLM {llm_model} \n with sysprompt {sysprompt} \n and prompt {prompt} \n and context len of {len(context)}")
+            messages = [
+                {"role": "system", "content": sysprompt},
+                {"role": "user", "content": prompt}
+            ]
+            response = completion(model=llm_model, messages=messages)
+            return response['choices'][0]['message']['content']
+        except Exception as e:
+            return f"{t('rag_error_calling_llm')}{str(e)}"
+
+    def free_llm(self):
+        try:
+            llm_model = st.session_state.ragllm_llm_model
+            if llm_model.startswith("ollama/"):
+                ollama_model = llm_model.split("/")[1]
+                response = requests.post(
+                    "http://localhost:11434/api/generate",
+                    json={
+                        "model": ollama_model,
+                        "prompt": "bye",
+                        "keep_alive": 0
+                    }
+                )
+                return response.json()['response']
+        except Exception as e:
+            return f"{t('rag_error_calling_llm')}{str(e)}"
+
+    def process_with_llm(self, prompt: str, sysprompt: str, context: str) -> str:
+        return self.call_llm(f"Contexte : {context}\n\nQuestion : {prompt}", sysprompt)
+
+    def run(self, config):
+        st.write(t("rag_plugin_loaded"))
+
+        # Initialiser rag_text avec la valeur de session_state si elle existe, sinon utiliser une chaîne vide
+        if 'rag_text' not in st.session_state:
+            st.session_state.rag_text = ""
+        if 'rag_question' not in st.session_state:
+            st.session_state.rag_question = "Question"
+
+        rag_text = st.text_area(t("rag_enter_text"), height=200, value=st.session_state.rag_text, key="rag_text_key")
+        user_prompt = st.text_area(t("rag_enter_question"), value=st.session_state.rag_question, key="rag_prompt_key")
+        st.session_state.rag_text = rag_text  # Mettre à jour la valeur dans session_state
+        st.session_state.rag_question = user_prompt
+
+        if st.button(t("rag_button_get_answer"), key="get_answer_button"):
+
+            with st.spinner(t("rag_processing")):
+                if rag_text:
+                    self.process_rag_text(rag_text, config['ragllm']['chunk_size'], config['ragllm']['embedder'])
+                    st.success(t("rag_success_text_processed"))
+                else:
+                    st.warning(t("rag_warning_enter_text"))
+                if user_prompt and self.embeddings is not None:
+                    context, citations = self.get_context(user_prompt, config)
+                    response = self.process_with_llm(user_prompt, config['ragllm']['llm_sys_prompt'], context)
+
+                    st.write(t("rag_answer"))
+                    st.write(response)
+
+                    st.write(t("rag_citations"))
+                    for i, citation in enumerate(citations, 1):
+                        st.write(f"{i}. {citation[:100]}...")
+                elif self.embeddings is None:
+                    st.warning(t("rag_warning_process_text_first"))
+                else:
+                    st.warning(t("rag_warning_enter_question"))

plugins/scansite.py

@@ -0,0 +1,503 @@
+from app import Plugin
+import streamlit as st
+import sqlite3
+import requests
+from bs4 import BeautifulSoup
+from datetime import datetime
+import ollama
+from global_vars import t, translations
+
+# Ajout des traductions spécifiques à ce plugin
+translations["en"].update({
+    "scansite_title": "News Aggregator",
+    "total_links": "Total number of links",
+    "annotated_links": "Number of annotated links",
+    "known_tags": "Known tags",
+    "reset_database": "Reset database",
+    "database_reset_success": "Database reset successfully",
+    "launch_scan": "Launch scan",
+    "scan_complete": "Scan complete",
+    "no_articles": "No articles to display.",
+    "page": "Page",
+    "previous_page": "Previous page",
+    "next_page": "Next page",
+    "new_articles": "New Articles",
+    "rated_articles": "Rated Articles",
+    "clicked_not_rated": "Clicked but not rated Articles",
+    "tagged_articles": "Tagged Articles",
+    "ignored_articles": "Ignored Articles",
+    "excluded_articles": "Excluded Articles",
+    "rating": "Rating",
+    "tags": "Tags",
+    "exclude": "Exclude",
+    "sources": "Sources",
+    "update": "Update",
+    "delete": "Delete",
+    "add_new_source": "Add a new source (URL)",
+    "add_source": "Add source",
+    "new_tag": "New tag",
+    "new_tag_description": "New tag description",
+    "add_tag": "Add tag",
+    "work_directory": "Work Directory",
+})
+
+translations["fr"].update({
+    "scansite_title": "Agrégateur de Nouvelles",
+    "total_links": "Nombre total de liens",
+    "annotated_links": "Nombre de liens annotés",
+    "known_tags": "Tags connus",
+    "reset_database": "Réinitialiser la base de données",
+    "database_reset_success": "Base de données réinitialisée",
+    "launch_scan": "Lancer le scan",
+    "scan_complete": "Scan terminé",
+    "no_articles": "Aucun article à afficher.",
+    "page": "Page",
+    "previous_page": "Page précédente",
+    "next_page": "Page suivante",
+    "new_articles": "Nouveaux Articles",
+    "rated_articles": "Articles Notés",
+    "clicked_not_rated": "Articles Cliqués non notés",
+    "tagged_articles": "Articles Tagués",
+    "ignored_articles": "Articles Ignorés",
+    "excluded_articles": "Articles Exclus",
+    "rating": "Note",
+    "tags": "Tags",
+    "exclude": "Exclure",
+    "sources": "Sources",
+    "update": "Mettre à jour",
+    "delete": "Supprimer",
+    "add_new_source": "Ajouter une nouvelle source (URL)",
+    "add_source": "Ajouter source",
+    "new_tag": "Nouveau tag",
+    "new_tag_description": "Description du nouveau tag",
+    "add_tag": "Ajouter tag",
+    "work_directory": "Répertoire de travail",
+})
+
+class ScansitePlugin(Plugin):
+    def __init__(self, name, plugin_manager):
+        super().__init__(name, plugin_manager)
+        self.conn = self.get_connection()
+        self.c = self.conn.cursor()
+        self.init_db()
+
+    def get_connection(self):
+        return sqlite3.connect('news_app.db', check_same_thread=False)
+
+    def init_db(self):
+        current_version = self.get_db_version()
+        if current_version < 1:
+            self.c.execute('''CREATE TABLE IF NOT EXISTS sources
+                             (id INTEGER PRIMARY KEY, url TEXT, title TEXT)''')
+            self.c.execute('''CREATE TABLE IF NOT EXISTS articles
+                             (id INTEGER PRIMARY KEY, source_id INTEGER, url TEXT UNIQUE, title TEXT, date TEXT,
+                              is_new INTEGER, is_excluded INTEGER DEFAULT 0)''')
+            self.c.execute('''CREATE TABLE IF NOT EXISTS user_actions
+                             (id INTEGER PRIMARY KEY, article_id INTEGER, action TEXT, rating INTEGER, tags TEXT, timestamp TEXT)''')
+            self.c.execute('''CREATE TABLE IF NOT EXISTS tags
+                             (id INTEGER PRIMARY KEY, name TEXT UNIQUE, description TEXT)''')
+            self.set_db_version(1)
+
+        # Add more version upgrades here
+        # if current_version < 2:
+        #     self.c.execute('''ALTER TABLE articles ADD COLUMN new_column TEXT''')
+        #     self.set_db_version(2)
+
+        self.conn.commit()
+
+    def get_db_version(self):
+        self.c.execute('''CREATE TABLE IF NOT EXISTS db_version (version INTEGER)''')
+        self.c.execute('SELECT version FROM db_version')
+        result = self.c.fetchone()
+        return result[0] if result else 0
+
+    def set_db_version(self, version):
+        self.c.execute('INSERT OR REPLACE INTO db_version (rowid, version) VALUES (1, ?)', (version,))
+        self.conn.commit()
+
+    def get_tabs(self):
+        return [{"name": t("scansite_title"), "plugin": "scansite"}]
+
+    def run(self, config):
+        st.title(t("scansite_title"))
+
+        total_links, annotated_links = self.get_stats()
+        st.write(f"{t('total_links')} : {total_links}")
+        st.write(f"{t('annotated_links')} : {annotated_links}")
+
+        all_tags = self.get_all_tags()
+        st.write(f"{t('known_tags')} :", ", ".join(all_tags))
+
+        if st.button(t("reset_database")):
+            self.reset_database()
+            st.success(t("database_reset_success"))
+
+        if st.button(t("launch_scan")):
+            self.launch_scan()
+            st.success(t("scan_complete"))
+
+        self.display_tabs()
+
+    def get_stats(self):
+        total_links = self.c.execute("SELECT COUNT(*) FROM articles WHERE is_excluded = 0").fetchone()[0]
+        annotated_links = self.c.execute("""
+            SELECT COUNT(DISTINCT article_id) FROM user_actions
+            WHERE action IN ('click', 'rate', 'tag')
+        """).fetchone()[0]
+        return total_links, annotated_links
+
+    def get_all_tags(self):
+        return [row[0] for row in self.c.execute("SELECT name FROM tags").fetchall()]
+
+    def reset_database(self):
+        self.c.execute("DROP TABLE IF EXISTS sources")
+        self.c.execute("DROP TABLE IF EXISTS articles")
+        self.c.execute("DROP TABLE IF EXISTS user_actions")
+        self.c.execute("DROP TABLE IF EXISTS tags")
+        self.conn.commit()
+        self.init_db()
+
+    def launch_scan(self):
+        sources = self.c.execute("SELECT * FROM sources").fetchall()
+        for source in sources:
+            self.mark_not_new(source[0])
+            links = self.scan_new_links(source[0], source[1])
+            for link, title in links:
+                self.c.execute("""
+                    INSERT OR IGNORE INTO articles (source_id, url, title, date, is_new, is_excluded)
+                    VALUES (?, ?, ?, ?, 1, 0)
+                """, (source[0], link, title, datetime.now().strftime('%Y-%m-%d')))
+        self.conn.commit()
+
+    def display_tabs(self):
+        tab1, tab2, tab3, tab4, tab5, tab6 = st.tabs([
+            t("new_articles"), t("rated_articles"), t("clicked_not_rated"),
+            t("tagged_articles"), t("ignored_articles"), t("excluded_articles")
+        ])
+
+        all_tags = self.get_all_tags()
+
+        with tab1:
+            st.header(t("new_articles"))
+            self.display_paginated_articles(self.get_new_articles(), all_tags, "nouveaux")
+
+        with tab2:
+            st.header(t("rated_articles"))
+            self.display_paginated_articles(self.get_rated_articles(), all_tags, "notes")
+
+        with tab3:
+            st.header(t("clicked_not_rated"))
+            self.display_paginated_articles(self.get_clicked_not_rated_articles(), all_tags, "cliques")
+
+        with tab4:
+            st.header(t("tagged_articles"))
+            self.display_paginated_articles(self.get_tagged_articles(), all_tags, "tagues")
+
+        with tab5:
+            st.header(t("ignored_articles"))
+            self.display_paginated_articles(self.get_ignored_articles(), all_tags, "ignores")
+
+        with tab6:
+            st.header(t("excluded_articles"))
+            self.display_paginated_articles(self.get_excluded_articles(), all_tags, "exclus")
+
+    def display_paginated_articles(self, articles, all_tags, tab_name, items_per_page=20):
+        if not articles:
+            st.write(t("no_articles"))
+            return
+
+        total_pages = (len(articles) - 1) // items_per_page + 1
+
+        page_key = f"{tab_name}_page"
+        if page_key not in st.session_state:
+            st.session_state[page_key] = 1
+
+        page = st.number_input(t("page"), min_value=1, max_value=total_pages, value=st.session_state[page_key], key=f"{tab_name}_number_input")
+        st.session_state[page_key] = page
+
+        start_idx = (page - 1) * items_per_page
+        end_idx = start_idx + items_per_page
+
+        for article in articles[start_idx:end_idx]:
+            self.display_article(article, all_tags, tab_name)
+
+        col1, col2, col3 = st.columns(3)
+        with col1:
+            if page > 1:
+                if st.button(t("previous_page"), key=f"{tab_name}_prev"):
+                    st.session_state[page_key] = page - 1
+                    st.rerun()
+        with col3:
+            if page < total_pages:
+                if st.button(t("next_page"), key=f"{tab_name}_next"):
+                    st.session_state[page_key] = page + 1
+                    st.rerun()
+        with col2:
+            st.write(f"{t('page')} {page}/{total_pages}")
+
+    def display_article(self, article, all_tags, tab_name):
+        article_id = article[0]
+
+        col1, col2, col3, col4, col5 = st.columns([3, 0.5, 1, 2, 1])
+
+        with col1:
+            summary_key = f"{tab_name}_summary_{article_id}"
+            if summary_key not in st.session_state:
+                st.session_state[summary_key] = None
+
+            if st.button(article[3], key=f"{tab_name}_article_{article_id}"):
+                summary = self.get_article_summary(article[2])
+                st.session_state[summary_key] = summary
+                self.c.execute("INSERT INTO user_actions (article_id, action, timestamp) VALUES (?, ?, ?)",
+                               (article_id, 'click', datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
+                self.c.execute("UPDATE articles SET is_new = 0 WHERE id = ?", (article_id,))
+                self.conn.commit()
+
+            if st.session_state[summary_key]:
+                st.write(st.session_state[summary_key])
+
+        with col2:
+            st.markdown(f"[🔗]({article[2]})")
+
+        with col3:
+            rating_key = f"{tab_name}_rating_{article_id}"
+            current_rating = self.get_article_rating(article_id)
+            rating = st.slider(t("rating"), 0, 5, current_rating, key=rating_key)
+            if rating != current_rating:
+                self.c.execute("INSERT INTO user_actions (article_id, action, rating, timestamp) VALUES (?, ?, ?, ?)",
+                               (article_id, 'rate', rating, datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
+                self.conn.commit()
+
+        with col4:
+            tags_key = f"{tab_name}_tags_{article_id}"
+            current_tags = self.get_article_tags(article_id)
+            selected_tags = st.multiselect(t("tags"), all_tags, default=current_tags, key=tags_key)
+            if set(selected_tags) != set(current_tags):
+                tags_str = ','.join(selected_tags)
+                self.c.execute("INSERT INTO user_actions (article_id, action, tags, timestamp) VALUES (?, ?, ?, ?)",
+                               (article_id, 'tag', tags_str, datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
+                self.conn.commit()
+
+        with col5:
+            exclude_key = f"{tab_name}_exclude_{article_id}"
+            if st.button(t("exclude"), key=exclude_key):
+                self.c.execute("UPDATE articles SET is_excluded = 1 WHERE id = ?", (article_id,))
+                self.conn.commit()
+                st.rerun()
+
+    def get_config_ui(self, config):
+        updated_config = {}
+
+        updated_config['sources'] = st.header(t("sources"))
+        sources = self.c.execute("SELECT * FROM sources").fetchall()
+        for source in sources:
+            col1, col2, col3 = st.columns([3, 1, 1])
+            with col1:
+                new_title = st.text_input(f"{t('update')} {source[1]}", value=source[2], key=f"source_title_{source[0]}")
+            with col2:
+                if st.button(t("update"), key=f"update_source_{source[0]}"):
+                    self.c.execute("UPDATE sources SET title = ? WHERE id = ?", (new_title, source[0]))
+                    self.conn.commit()
+            with col3:
+                if st.button(t("delete"), key=f"delete_source_{source[0]}"):
+                    self.c.execute("DELETE FROM sources WHERE id = ?", (source[0],))
+                    self.conn.commit()
+
+        new_url = st.text_input(t("add_new_source"))
+        if st.button(t("add_source")):
+            title = self.fetch_page_title(new_url)
+            self.c.execute("INSERT INTO sources (url, title) VALUES (?, ?)", (new_url, title))
+            self.conn.commit()
+
+        st.header(t("tags"))
+        tags = self.get_all_tags_with_descriptions()
+        for tag, description in tags:
+            col1, col2, col3, col4 = st.columns([2, 3, 1, 1])
+            with col1:
+                st.text(tag)
+            with col2:
+                new_description = st.text_input(f"{t('update')} {tag}", value=description, key=f"tag_desc_{tag}")
+            with col3:
+                if st.button(t("update"), key=f"update_tag_{tag}"):
+                    self.add_or_update_tag(tag, new_description)
+            with col4:
+                if st.button(t("delete"), key=f"delete_tag_{tag}"):
+                    self.delete_tag(tag)
+
+        new_tag = st.text_input(t("new_tag"))
+        new_tag_description = st.text_input(t("new_tag_description"))
+        if st.button(t("add_tag")):
+            self.add_or_update_tag(new_tag, new_tag_description)
+
+        # Ajout des configurations modifiées au dictionnaire updated_config
+        updated_config["sources"] = sources
+        updated_config["new_source_url"] = new_url
+        updated_config["tags"] = tags
+        updated_config["new_tag"] = new_tag
+        updated_config["new_tag_description"] = new_tag_description
+
+        return updated_config
+
+    def fetch_page_title(self, url):
+        try:
+            response = requests.get(url)
+            soup = BeautifulSoup(response.text, 'html.parser')
+            return soup.title.string
+        except:
+            return url
+
+    def mark_not_new(self, source_id):
+        self.c.execute("UPDATE articles SET is_new = 0 WHERE source_id = ?", (source_id,))
+        self.conn.commit()
+
+    def scan_new_links(self, source_id, url):
+        links = self.scan_links(url)
+        filtered_links = []
+        for link, title in links:
+            self.c.execute("SELECT id, is_excluded FROM articles WHERE url = ?", (link,))
+            result = self.c.fetchone()
+            if result is None:
+                filtered_links.append((link, title))
+        return filtered_links
+
+    def scan_links(self, url):
+        links = set()
+        try:
+            response = requests.get(url)
+            soup = BeautifulSoup(response.text, 'html.parser')
+            for link in soup.find_all('a'):
+                href = link.get('href')
+                title = link.text.strip() or href
+                if href and href.startswith('http'):
+                    try:
+                        article_response = requests.get(href)
+                        article_soup = BeautifulSoup(article_response.text, 'html.parser')
+                        if article_soup.find('article'):
+                            links.add((href, title))
+                    except:
+                        pass
+        except:
+            st.error(f"Erreur lors du scan de {url}")
+        return list(links)
+
+    def get_article_summary(self, url, model="qwen2"):
+        prompt = f"Résumez brièvement l'article à cette URL : {url}"
+        response = ollama.generate(model=model, prompt=prompt)
+        return response['response']
+
+    def get_new_articles(self):
+        return self.c.execute("""
+            SELECT * FROM articles
+            WHERE is_new = 1
+            AND is_excluded = 0
+            AND id NOT IN (
+                SELECT DISTINCT article_id
+                FROM user_actions
+                WHERE action IN ('click', 'rate', 'tag')
+            )
+            ORDER BY date DESC
+        """).fetchall()
+
+    def get_rated_articles(self):
+        return self.c.execute("""
+            SELECT DISTINCT a.*
+            FROM articles a
+            JOIN user_actions ua ON a.id = ua.article_id
+            WHERE ua.action = 'rate'
+            AND a.is_excluded = 0
+            ORDER BY ua.timestamp DESC
+        """).fetchall()
+
+    def get_clicked_not_rated_articles(self):
+        return self.c.execute("""
+            SELECT DISTINCT a.*
+            FROM articles a
+            JOIN user_actions ua ON a.id = ua.article_id
+            WHERE ua.action = 'click'
+            AND a.is_excluded = 0
+            AND a.id NOT IN (
+                SELECT article_id
+                FROM user_actions
+                WHERE action IN ('rate', 'tag')
+            )
+            ORDER BY ua.timestamp DESC
+        """).fetchall()
+
+    def get_tagged_articles(self):
+        return self.c.execute("""
+            SELECT DISTINCT a.*
+            FROM articles a
+            JOIN user_actions ua ON a.id = ua.article_id
+            WHERE ua.action = 'tag'
+            AND a.is_excluded = 0
+            AND a.id NOT IN (
+                SELECT article_id
+                FROM user_actions
+                WHERE action IN ('rate', 'click')
+            )
+            ORDER BY ua.timestamp DESC
+        """).fetchall()
+
+    def get_ignored_articles(self):
+        return self.c.execute("""
+            SELECT * FROM articles
+            WHERE is_new = 0
+            AND is_excluded = 0
+            AND id NOT IN (
+                SELECT DISTINCT article_id
+                FROM user_actions
+                WHERE action IN ('click', 'rate', 'tag')
+            )
+            ORDER BY date DESC
+        """).fetchall()
+
+    def get_excluded_articles(self):
+        return self.c.execute("""
+            SELECT * FROM articles
+            WHERE is_excluded = 1
+            ORDER BY date DESC
+        """).fetchall()
+
+    def get_article_rating(self, article_id):
+        self.c.execute("SELECT rating FROM user_actions WHERE article_id = ? AND action = 'rate' ORDER BY timestamp DESC LIMIT 1", (article_id,))
+        result = self.c.fetchone()
+        return result[0] if result else 0
+
+    def get_article_tags(self, article_id):
+        self.c.execute("SELECT tags FROM user_actions WHERE article_id = ? AND action = 'tag' ORDER BY timestamp DESC LIMIT 1", (article_id,))
+        result = self.c.fetchone()
+        return result[0].split(',') if result and result[0] else []
+
+    def get_all_tags_with_descriptions(self):
+        return self.c.execute("SELECT name, description FROM tags").fetchall()
+
+    def add_or_update_tag(self, name, description):
+        self.c.execute("INSERT OR REPLACE INTO tags (name, description) VALUES (?, ?)", (name, description))
+        self.conn.commit()
+
+    def delete_tag(self, name):
+        self.c.execute("DELETE FROM tags WHERE name = ?", (name,))
+        self.conn.commit()
+
+    def get_reference_data(self):
+        # Récupérer les articles avec leur rating
+        self.c.execute("""
+            SELECT a.id, a.url, a.title, COALESCE(ua.rating, 0) as rating
+            FROM articles a
+            LEFT JOIN (
+                SELECT article_id, rating
+                FROM user_actions
+                WHERE action = 'rate'
+                GROUP BY article_id
+                HAVING MAX(timestamp)
+            ) ua ON a.id = ua.article_id
+            WHERE a.is_excluded = 0
+            ORDER BY rating DESC, a.date DESC
+        """)
+        articles = self.c.fetchall()
+
+        # Séparer les articles en valides (notés) et rejetés (non notés)
+        reference_data_valid = [(article[1], article[2], article[3]) for article in articles if article[3] > 0]
+        reference_data_rejected = [(article[1], article[2]) for article in articles if article[3] == 0]
+
+        return reference_data_valid, reference_data_rejected

plugins/webrankings.py

@@ -0,0 +1,285 @@
+import streamlit as st
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+import torch
+from transformers import pipeline
+from sentence_transformers import SentenceTransformer, util
+import concurrent.futures
+import time
+import sys
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+from transformers import AutoTokenizer, AutoModel
+import numpy as np
+from scipy import stats
+from PyDictionary import PyDictionary
+import matplotlib.pyplot as plt
+from scipy import stats
+import litellm
+import re
+import sentencepiece
+import random
+from global_vars import t, translations
+from app import Plugin
+
+from embeddings_ft import finetune as finetune_embeddings
+from bart_ft import finetune as finetune_bart
+
+from webrankings_helper import *
+from plugins.scansite import ScansitePlugin
+
+#from data import reference_data_valid, reference_data_rejected
+#reference_data = reference_data_valid + reference_data_rejected
+
+# Ajout des traductions spécifiques à ce plugin
+translations["en"].update({
+    "webrankings_title": "Comparative os sorter",
+    "clear_memory": "Clear Memory",
+    "enter_topic": "Enter the topic you're interested in (e.g. longevity):",
+    "use_keyword_expansion": "Use keyword expansion",
+    "test_content": "Also test link content in addition to titles",
+    "select_llm_models": "Select LLM models to use",
+    "select_zero_shot_models": "Select zero-shot models to use",
+    "select_embedding_models": "Select embedding models to use",
+    "analyze_button": "Analyze",
+    "loading_models": "Loading models and analyzing links...",
+    "expanded_keywords": "Expanded keywords:",
+    "analysis_completed": "Analysis completed in {:.2f} seconds",
+    "evaluation_results": "Evaluation results with optimal thresholds:",
+    "summary_table": "Summary table of scores",
+    "optimal_thresholds": "Optimal thresholds:",
+    "spearman_comparison": "Comparison of Spearman correlations",
+    "methods": "Methods",
+    "spearman_correlation": "Spearman correlation coefficient",
+    "results_for": "Results for {}",
+    "device_info": "Device used for inference: {}",
+    "finetune_bart_title": "BART Fine-tuning Interface",
+    "finetune_embeddings_title": "Embeddings Fine-tuning Interface",
+})
+
+translations["fr"].update({
+    "webrankings_title": "Analyseur de classeurs",
+    "clear_memory": "Vider la mémoire",
+    "enter_topic": "Entrez le sujet qui vous intéresse (ex: longévité):",
+    "use_keyword_expansion": "Utiliser l'expansion des mots-clés",
+    "test_content": "Tester aussi le contenu des liens en plus des titres",
+    "select_llm_models": "Sélectionnez les modèles LLM à utiliser",
+    "select_zero_shot_models": "Sélectionnez les modèles zero-shot à utiliser",
+    "select_embedding_models": "Sélectionnez les modèles d'embedding à utiliser",
+    "analyze_button": "Analyser",
+    "loading_models": "Chargement des modèles et analyse des liens...",
+    "expanded_keywords": "Mots-clés étendus :",
+    "analysis_completed": "Analyse terminée en {:.2f} secondes",
+    "evaluation_results": "Résultats d'évaluation avec les seuils optimaux :",
+    "summary_table": "Tableau récapitulatif des scores",
+    "optimal_thresholds": "Seuils optimaux :",
+    "spearman_comparison": "Comparaison des corrélations de Spearman",
+    "methods": "Méthodes",
+    "spearman_correlation": "Coefficient de corrélation de Spearman",
+    "results_for": "Résultats pour {}",
+    "device_info": "Dispositif utilisé pour l'inférence : {}",
+    "finetune_bart_title": "Interface de Fine-tuning BART",
+    "finetune_embeddings_title": "Interface de Fine-tuning des Embeddings",
+})
+
+
+# Liste des modèles LLM
+llm_models = [] #["ollama/llama3", "ollama/llama3.1", "ollama/qwen2", "ollama/phi3:medium-128k", "ollama/openhermes"]
+
+# Liste des modèles zero-shot
+zero_shot_models = [
+    ("facebook/bart-large-mnli", "facebook/bart-large-mnli"),
+    ("bart-large-ft", "./bart-large-ft")
+    #("cross-encoder/nli-deberta-v3-base", "cross-encoder/nli-deberta-v3-base")
+]
+
+# Liste des modèles d'embedding
+embedding_models = [
+    ("paraphrase-MiniLM-L6-v2", "paraphrase-MiniLM-L6-v2"),
+    ("all-MiniLM-L6-v2", "all-MiniLM-L6-v2"),
+    ("nomic-embed-text-v1", "nomic-ai/nomic-embed-text-v1"),
+    ("embeddings-ft", "./embeddings-ft")
+]
+
+
+class WebrankingsPlugin(Plugin):
+    def __init__(self, name, plugin_manager):
+        super().__init__(name, plugin_manager)
+        self.scansite_plugin = ScansitePlugin('scansite', plugin_manager)
+
+    def get_tabs(self):
+        return [
+            {"name": t("webrankings_title"), "plugin": "webrankings"}
+        ]
+
+    def run(self, config):
+        tab1, tab2, tab3 = st.tabs([t("webrankings_title"), t("finetune_bart_title"), t("finetune_embeddings_title")])
+        reference_data_valid, reference_data_rejected = self.scansite_plugin.get_reference_data()
+        reference_data = reference_data_valid + [(url, title, 0) for url, title in reference_data_rejected]
+
+        with tab1:
+            st.title(t("webrankings_title"))
+            if st.button(t("clear_memory")):
+                torch.cuda.empty_cache()
+                torch.cuda.synchronize()
+                clear_globals()
+                reset_cuda_context()
+
+            topic = st.text_input(t("enter_topic"), value="longevity, health, healthspan, lifespan")
+            use_synonyms = st.checkbox(t("use_keyword_expansion"), value=False)
+            check_content = st.checkbox(t("test_content"), value=False)
+
+            selected_llm_models = st.multiselect(t("select_llm_models"), llm_models, default=llm_models)
+            selected_zero_shot_models = st.multiselect(t("select_zero_shot_models"), [m[0] for m in zero_shot_models], default=[m[0] for m in zero_shot_models])
+            selected_embedding_models = st.multiselect(t("select_embedding_models"), [m[0] for m in embedding_models], default=[m[0] for m in embedding_models])
+
+            if st.button(t("analyze_button")):
+                with st.spinner(t("loading_models")):
+                    device = "cuda" if torch.cuda.is_available() else "cpu"
+
+                    # Préparation des modèles
+                    zero_shot_classifiers = {name: pipeline("zero-shot-classification", model=model, device=device)
+                                             for name, model in zero_shot_models if name in selected_zero_shot_models}
+                    embedding_models_dict = {}
+                    for name, model in embedding_models:
+                        import os
+                        if name == "embeddings-ft":
+                            if os.path.exists('./embeddings-ft'):
+                                embedding_models_dict[name] = SentenceTransformer('./embeddings-ft', trust_remote_code=True).to(device)
+                        else:
+                            embedding_models_dict[name] = SentenceTransformer(model, trust_remote_code=True).to(device)
+                    bert_models = [AutoModel.from_pretrained('bert-base-uncased').to(device)]
+                    tfidf_objects = [TfidfVectorizer()]
+                    #release_vram(zero_shot_classifiers, embedding_models_dict, bert_models, tfidf_objects)
+
+                    # Expansion des mots-clés (utilisant le premier modèle LLM sélectionné)
+                    if use_synonyms and selected_llm_models:
+                        expanded_query = []
+                        for word in topic.split():
+                            expanded_query.extend(expand_keywords_llm(word, llm_model=selected_llm_models[0]))
+                        expanded_query = " ".join(expanded_query)
+                        st.write("Mots-clés étendus :", expanded_query)
+                    else:
+                        expanded_query = topic
+
+                    start_time = time.time()
+                    # Analyse pour chaque lien
+                    results = []
+                    for title, link,note in reference_data:
+                        result = analyze_link(
+                            title, link, topic, zero_shot_classifiers, embedding_models_dict,
+                            expanded_query, selected_llm_models, check_content
+                        )
+                        if result is not None:
+                            results.append(result)
+                    end_time = time.time()
+
+                    # Libération de la mémoire VRAM et des autres ressources
+                    release_vram(zero_shot_classifiers, embedding_models_dict, bert_models, tfidf_objects)
+
+                    # Création du DataFrame avec tous les résultats
+                    df = pd.DataFrame(results)
+                    print(f"Analyse terminée en {end_time - start_time:.2f} secondes")
+
+                    st.success(t("analysis_completed").format(end_time - start_time))
+
+                    # Évaluation et affichage des résultats
+                    evaluation_results = {}
+                    optimal_thresholds = {}
+                    for column in df.columns:
+                        if column != "Titre":
+                            method_scores = df.set_index("Titre")[column].to_dict()
+
+                            optimal_threshold = find_optimal_threshold(
+                                [item[0] for item in reference_data_valid],
+                                [item[0] for item in reference_data_rejected],
+                                method_scores
+                            )
+                            optimal_thresholds[column] = optimal_threshold
+
+                            evaluation_results[column] = evaluate_ranking(
+                                [item[0] for item in reference_data_valid],
+                                [item[0] for item in reference_data_rejected],
+                                method_scores,
+                                optimal_threshold, False
+                            )
+
+                    # Affichage des résultats
+                    st.write(t("evaluation_results"))
+                    eval_df = pd.DataFrame(evaluation_results).T
+                    st.dataframe(eval_df)
+
+                    st.subheader(t("summary_table"))
+                    st.dataframe(df)
+
+                    st.write(t("optimal_thresholds"))
+                    st.json(optimal_thresholds)
+
+                    # Graphique de comparaison des corrélations de Spearman
+                    spearman_scores = [results['spearman_correlation'] for results in evaluation_results.values()]
+                    plt.figure(figsize=(15, 8))
+                    plt.bar(evaluation_results.keys(), spearman_scores)
+                    plt.title(t("spearman_comparison"))
+                    plt.xlabel(t("methods"))
+                    plt.ylabel(t("spearman_correlation"))
+                    plt.xticks(rotation=90, ha='right')
+                    plt.tight_layout()
+                    st.pyplot(plt)
+
+                    # Affichage des résultats pour chaque méthode
+                    for column in df.columns:
+                        if column != "Titre":
+                            st.subheader(f"Résultats pour {column}")
+                            df_method = df[["Titre", column]].sort_values(column, ascending=False)
+                            threshold = find_optimal_threshold(
+                                [item[0] for item in reference_data_valid],
+                                [item[0] for item in reference_data_rejected],
+                                df_method.set_index("Titre")[column].to_dict()
+                            )
+                            df_method = df_method[df_method[column] > threshold]
+                            st.dataframe(df_method)
+
+        with tab2:
+            st.title(t("finetune_bart_title"))
+            num_epochs = st.number_input("Nombre d'époques", min_value=1, max_value=10, value=2)
+            lr = st.number_input("Learning Rate", min_value=1e-6, max_value=1e-1, value=2e-5, format="%.6f", step=1e-5)
+            weight_decay = st.number_input("Poids de Décroissance (Weight Decay)", min_value=0.0, max_value=0.1, value=0.01, step=0.005)
+            batch_size = st.number_input("Taille du Batch", min_value=1, max_value=16, value=1)
+            start = st.slider("Score initial des données valides", min_value=0.0, max_value=1.0, value=0.9, step=0.01)
+            model_name = st.text_input("Nom du modèle", value='facebook/bart-large-mnli')
+            num_warmup_steps = st.number_input("Nombre d'étapes de Warmup", min_value=0, max_value=100, value=0)
+
+            # Bouton pour lancer le fine-tuning
+            if st.button("Lancer le fine-tuning"):
+                with st.spinner("Fine-tuning en cours..."):
+                    finetune_bart(num_epochs=num_epochs, lr=lr, weight_decay=weight_decay,
+                             batch_size=batch_size, model_name=model_name, output_model='./bart-large-ft',
+                             num_warmup_steps=num_warmup_steps)
+                    st.success("Fine-tuning terminé et modèle sauvegardé.")
+
+        with tab3:
+            st.title(t("finetune_embeddings_title"))
+            num_epochs_emb = st.number_input("Nombre d'époques (Embeddings)", min_value=1, max_value=100, value=10)
+            lr_emb = st.number_input("Learning Rate (Embeddings)", min_value=1e-6, max_value=1e-1, value=2e-5, format="%.6f", step=5e-6)
+            weight_decay_emb = st.number_input("Poids de Décroissance (Weight Decay) (Embeddings)", min_value=0.0, max_value=0.1, value=0.01, step=0.005)
+            batch_size_emb = st.number_input("Taille du Batch (Embeddings)", min_value=1, max_value=32, value=16)
+            start_emb = st.slider("Score initial des données valides (Embeddings)", min_value=0.0, max_value=1.0, value=0.9, step=0.01)
+            model_name_emb = st.selectbox("Modèle d'embeddings de base", ["nomic-ai/nomic-embed-text-v1", "all-MiniLM-L6-v2", "paraphrase-MiniLM-L6-v2"])
+            margin_erb = st.slider("Marge (Embeddings)", min_value=0.0, max_value=1.0, value=0.5, step=0.01)
+
+            # Bouton pour lancer le fine-tuning des embeddings
+            if st.button("Lancer le fine-tuning des embeddings"):
+                with st.spinner("Fine-tuning des embeddings en cours..."):
+
+                    finetune_embeddings(model_name=model_name_emb, output_model_name="./embeddings-ft",
+                                        num_epochs=num_epochs_emb,
+                                        learning_rate=lr_emb,
+                                        weight_decay=weight_decay_emb,
+                                        batch_size=batch_size_emb,
+                                        )
+                    st.success("Fine-tuning des embeddings terminé et modèle sauvegardé.")
+
+        # Affichage de l'information sur le dispositif utilisé
+        device = "GPU (CUDA)" if torch.cuda.is_available() else "CPU"
+        st.sidebar.info(t("device_info").format(device))

webrankings_helper.py

@@ -0,0 +1,410 @@
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+import torch
+from transformers import pipeline
+from sentence_transformers import SentenceTransformer, util
+import concurrent.futures
+import time
+import sys
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+from transformers import AutoTokenizer, AutoModel
+import numpy as np
+from scipy import stats
+from PyDictionary import PyDictionary
+import matplotlib.pyplot as plt
+from scipy import stats
+import litellm
+import re
+import sentencepiece
+import random
+
+def score_with_llm(title, topic, llm_model):
+    prompt = f"""Evaluate the relevance of the following article to the topic '{topic}'.
+Article title: {title}
+Give a final relevance score between 0 and 1, where 1 is very relevant and 0 is not relevant at all.
+Respond only with a number between 0 and 1."""
+
+    try:
+        response = litellm.completion(
+            model=llm_model,
+            messages=[{"role": "user", "content": prompt}],
+            max_tokens=10
+        )
+
+        score_match = re.search(r'\d+(\.\d+)?', response.choices[0].message.content.strip())
+        if score_match:
+            score = float(score_match.group())
+            print(f"Score LLM : {score}")
+            return max(0, min(score, 1))
+        else:
+            print(f"Could not extract a score from LLM response: {response.choices[0].message.content}")
+            return None
+    except Exception as e:
+        print(f"Error in scoring with LLM {llm_model}: {str(e)}")
+        return None
+
+def expand_keywords_llm(keyword, max_synonyms=3, llm_model="ollama/qwen2"):
+    prompt = f"""Please provide up to {max_synonyms} synonyms or closely related terms for the word or phrase: "{keyword}".
+    Return only the list of synonyms, separated by commas, without any additional explanation."""
+
+    try:
+        response = litellm.completion(
+            model=llm_model,
+            messages=[{"role": "user", "content": prompt}],
+            max_tokens=50
+        )
+
+        synonyms = [s.strip() for s in response.choices[0].message.content.split(',')]
+        return [keyword] + synonyms[:max_synonyms]
+    except Exception as e:
+        print(f"Error in expanding keywords with LLM {llm_model}: {str(e)}")
+        return [keyword]
+
+# Fonction pour obtenir les liens de la page d'accueil
+def get_homepage_links(url):
+    response = requests.get(url)
+    soup = BeautifulSoup(response.text, 'html.parser')
+    links = soup.find_all('a', href=True)
+    return [(link.text.strip(), link['href']) for link in links if link.text.strip()]
+
+# Fonction pour obtenir le contenu d'un article
+def get_article_content(url):
+    try:
+        print(f"Récupération du contenu de : {url}")
+        response = requests.get(url)
+        print(f"Taille de la réponse HTTP : {len(response.content)} octets")  # Affiche le nombre d'octets de la réponse HTTP
+        soup = BeautifulSoup(response.text, 'html.parser')
+        print(f"Taille de l'objet soup : {sys.getsizeof(soup)} octets")  # Affiche la taille en mémoire de l'objet soup
+        article = soup.find('article')
+        if article:
+            paragraphs = article.find_all('p')
+            content = ' '.join([p.text for p in paragraphs])
+            print(f"Paragraphes récupéré : {len(content)} caractères")
+            return content
+        print("Aucun contenu d'article trouvé")
+        return ""
+    except Exception as e:
+        print(f"Erreur lors de la récupération du contenu : {str(e)}")
+        return ""
+
+# Fonction pour l'analyse zero-shot
+def zero_shot_analysis(text, topic, classifier):
+    if not text:
+        print("Texte vide pour l'analyse zero-shot")
+        return 0.0
+    result = classifier(text, candidate_labels=[topic, f"not {topic}"], multi_label=False)
+    print(f"Score zero-shot : {result['scores'][0]}")
+    return result['scores'][0]
+
+# Fonction pour l'analyse par embeddings
+def embedding_analysis(text, topic_embedding, model):
+    if not text:
+        print("Texte vide pour l'analyse par embeddings")
+        return 0.0
+    text_embedding = model.encode([text], convert_to_tensor=True)
+    similarity = util.pytorch_cos_sim(text_embedding, topic_embedding).item()
+    print(f"Score embedding : {similarity}")
+    return similarity
+
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+#import nltk
+#from nltk.corpus import wordnet
+#nltk.download('wordnet')
+
+def preprocess_text(text):
+    # Tokenize the text
+    tokens = text.lower().split()
+    # Expand each token with its synonyms
+    expanded_tokens = []
+    for token in tokens:
+        synonyms = set()
+        for syn in wordnet.synsets(token):
+            for lemma in syn.lemmas():
+                synonyms.add(lemma.name())
+        expanded_tokens.extend(list(synonyms))
+    return ' '.join(expanded_tokens)
+
+def improved_tfidf_similarity(texts, query):
+    # Preprocess texts and query
+    preprocessed_texts = [preprocess_text(text) for text in texts]
+    preprocessed_query = preprocess_text(query)
+
+    # Combine texts and query for vectorization
+    all_texts = preprocessed_texts + [preprocessed_query]
+
+    # Use TfidfVectorizer with custom parameters
+    vectorizer = TfidfVectorizer(ngram_range=(1, 2), min_df=1, smooth_idf=True)
+    tfidf_matrix = vectorizer.fit_transform(all_texts)
+
+    # Calculate cosine similarity
+    cosine_similarities = cosine_similarity(tfidf_matrix[-1], tfidf_matrix[:-1]).flatten()
+
+    # Normalize similarities to avoid zero scores
+    normalized_similarities = (cosine_similarities - cosine_similarities.min()) / (cosine_similarities.max() - cosine_similarities.min())
+
+    return normalized_similarities
+
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+import numpy as np
+
+def improved_tfidf_similarity_v2(texts, query):
+    # Combine texts and query, treating each word or phrase as a separate document
+    all_docs = [word.strip() for text in texts for word in text.split(',')] + [word.strip() for word in query.split(',')]
+
+    # Create TF-IDF matrix
+    vectorizer = TfidfVectorizer()
+    tfidf_matrix = vectorizer.fit_transform(all_docs)
+
+    # Calculate document vectors by summing the TF-IDF vectors of their words
+    doc_vectors = []
+    query_vector = np.zeros((1, tfidf_matrix.shape[1]))
+
+    current_doc = 0
+    for i, doc in enumerate(all_docs):
+        if i < len(all_docs) - len(query.split(',')):  # If it's part of the texts
+            if current_doc == len(texts):
+                break
+            if doc in texts[current_doc]:
+                doc_vectors.append(tfidf_matrix[i].toarray())
+            else:
+                current_doc += 1
+                doc_vectors.append(tfidf_matrix[i].toarray())
+        else:  # If it's part of the query
+            query_vector += tfidf_matrix[i].toarray()
+
+    doc_vectors = np.array([np.sum(doc, axis=0) for doc in doc_vectors])
+
+    # Calculate cosine similarity
+    similarities = cosine_similarity(query_vector, doc_vectors).flatten()
+
+    # Normalize similarities to avoid zero scores
+    normalized_similarities = (similarities - similarities.min()) / (similarities.max() - similarities.min() + 1e-8)
+
+    return normalized_similarities
+
+# Example usage:
+# texts = ["longevity, health, aging", "computer science, AI"]
+# query = "longevity, life extension, anti-aging"
+# results = improved_tfidf_similarity_v2(texts, query)
+# print(results)
+
+# Nouvelles fonctions
+def tfidf_similarity(texts, query):
+    vectorizer = TfidfVectorizer()
+    tfidf_matrix = vectorizer.fit_transform(texts + [query])
+    cosine_similarities = cosine_similarity(tfidf_matrix[-1], tfidf_matrix[:-1]).flatten()
+    return cosine_similarities
+
+def bert_similarity(texts, query, model_name='bert-base-uncased'):
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModel.from_pretrained(model_name)
+
+    def get_embedding(text):
+        inputs = tokenizer(text, return_tensors='pt', truncation=True, padding=True, max_length=512)
+        with torch.no_grad():
+            outputs = model(**inputs)
+        return outputs.last_hidden_state.mean(dim=1).squeeze().numpy()
+
+    query_embedding = get_embedding(query)
+    text_embeddings = [get_embedding(text) for text in texts]
+
+    similarities = [cosine_similarity([query_embedding], [text_embedding])[0][0] for text_embedding in text_embeddings]
+    return similarities
+
+# Fonction principale d'analyse modifiée
+def analyze_link(title, link, topic, zero_shot_classifiers, embedding_models, expanded_query, llm_models, testcontent):
+    print(f"\nAnalyse de : {title}")
+
+    results = {
+        "Titre": title,
+        #"TF-IDF (titre)": improved_tfidf_similarity_v2([title], expanded_query)[0],
+        #"BERT (titre)": bert_similarity([title], expanded_query)[0],
+    }
+
+    # Zero-shot analysis
+    for name, classifier in zero_shot_classifiers.items():
+        results[f"Zero-shot (titre) - {name}"] = zero_shot_analysis(title, topic, classifier)
+
+    # Embedding analysis
+    for name, model in embedding_models.items():
+        topic_embedding = model.encode([expanded_query], convert_to_tensor=True)
+        results[f"Embeddings (titre) - {name}"] = embedding_analysis(title, topic_embedding, model)
+
+    # LLM analysis
+    for model in llm_models:
+        results[f"LLM Score - {model}"] = score_with_llm(title, topic, model)
+
+    if testcontent:
+        content = get_article_content(link)
+        #results["TF-IDF (contenu)"] = improved_tfidf_similarity_v2([content], expanded_query)[0]
+        #results["BERT (contenu)"]= bert_similarity([content], expanded_query)[0]
+
+        # Zero-shot analysis
+        for name, classifier in zero_shot_classifiers.items():
+            results[f"Zero-shot (contenu) - {name}"] = zero_shot_analysis(content, topic, classifier)
+
+        # Embedding analysis
+        for name, model in embedding_models.items():
+            topic_embedding = model.encode([expanded_query], convert_to_tensor=True)
+            results[f"Embeddings (contenu) - {name}"] = embedding_analysis(content, topic_embedding, model)
+
+        # LLM analysis
+        for model in llm_models:
+            results[f"LLM Content Score - {model}"] = score_with_llm(content, topic, model)
+
+
+    return results
+
+from scipy import stats
+
+def evaluate_ranking(reference_data_valid, reference_data_rejected, method_scores, threshold, silent):
+    simple_score = 0
+    true_positives = 0
+    false_positives = 0
+    true_negatives = 0
+    false_negatives = 0
+
+    # Créer une liste de tous les éléments avec leur statut (1 pour valide, 0 pour rejeté)
+    all_items = [(item, 1) for item in reference_data_valid] + [(item, 0) for item in reference_data_rejected]
+
+    # Trier les éléments selon leur score dans la méthode
+    all_items_temp = all_items.copy()
+    # correct false positive if method spit out same score for all
+    #random.shuffle(all_items_temp)
+    all_items_temp.reverse()
+    sorted_method = sorted([(item, method_scores.get(item, 0)) for item, _ in all_items_temp],
+                           key=lambda x: x[1], reverse=True)
+
+    # Créer des listes pour le calcul de la corrélation de Spearman
+    reference_ranks = []
+    method_ranks = []
+
+    for i, (item, status) in enumerate(all_items):
+        method_score = method_scores.get(item, 0)
+        method_rank = next(j for j, (it, score) in enumerate(sorted_method) if it == item)
+
+        reference_ranks.append(i)
+        method_ranks.append(method_rank)
+
+        if status == 1:  # Item valide
+            if method_score >= threshold:
+                simple_score += 1
+                true_positives += 1
+            else:
+                simple_score -= 1
+                false_negatives += 1
+        else:  # Item rejeté
+            if method_score < threshold:
+                simple_score += 1
+                true_negatives += 1
+            else:
+                simple_score -= 1
+                false_positives += 1
+
+    # Calculer le coefficient de corrélation de Spearman
+    if not silent:
+        print("+++")
+        print(reference_ranks)
+        print("---")
+        print(method_ranks)
+    spearman_corr, _ = stats.spearmanr(reference_ranks, method_ranks)
+
+    # Calculer la précision, le rappel et le F1-score
+    precision = true_positives / (true_positives + false_positives) if (true_positives + false_positives) > 0 else 0
+    recall = true_positives / (true_positives + false_negatives) if (true_positives + false_negatives) > 0 else 0
+    f1_score = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0
+
+    return {
+        "simple_score": simple_score,
+        "spearman_correlation": spearman_corr,
+        "precision": precision,
+        "recall": recall,
+        "f1_score": f1_score,
+    }
+
+def find_optimal_threshold(reference_data_valid, reference_data_rejected, method_scores):
+    best_score = float('-inf')
+    best_threshold = 0
+
+    for threshold in np.arange(0, 1.05, 0.05):
+        result = evaluate_ranking(
+            reference_data_valid,
+            reference_data_rejected,
+            method_scores,
+            threshold, True
+        )
+        if result['simple_score'] > best_score:
+            best_score = result['simple_score']
+            best_threshold = threshold
+
+    return best_threshold
+
+def reset_cuda_context():
+    torch.cuda.empty_cache()
+    torch.cuda.ipc_collect()
+    if torch.cuda.is_available():
+        torch.cuda.set_device(torch.cuda.current_device())
+        torch.cuda.synchronize()
+
+import gc
+def clear_models():
+    global zero_shot_classifiers, embedding_models_dict, bert_models, tfidf_objects
+
+    for classifier in zero_shot_classifiers.values():
+        del classifier
+    zero_shot_classifiers.clear()
+
+    for model in embedding_models_dict.values():
+        del model
+    embedding_models_dict.clear()
+
+    for model in bert_models:
+        del model
+    bert_models.clear()
+
+    for vectorizer in tfidf_objects:
+        del vectorizer
+    tfidf_objects.clear()
+
+    torch.cuda.empty_cache()
+    gc.collect()
+
+def clear_globals():
+    for name in list(globals()):
+        if isinstance(globals()[name], (torch.nn.Module, torch.Tensor)):
+            del globals()[name]
+
+def release_vram(zero_shot_classifiers, embedding_models, bert_models, tfidf_objects):
+    # Supprimer les objets zero-shot classifiers
+    for model in zero_shot_classifiers.values():
+        del model
+
+    # Supprimer les objets embedding models
+    for model in embedding_models.values():
+        del model
+
+    # Supprimer les objets bert models
+    for model in bert_models:
+        del model
+
+    # Supprimer les objets tfidf objects
+    for obj in tfidf_objects:
+        del obj
+
+    # Vider le cache de la mémoire GPU
+    torch.cuda.empty_cache()
+    torch.cuda.synchronize()
+    gc.collect()
+    clear_globals()
+    reset_cuda_context()
+
+def load_finetuned_model(model_path):
+    checkpoint = torch.load(model_path)
+    base_model = AutoModel.from_pretrained(checkpoint['base_model_name'])
+    model = EmbeddingModel(base_model)
+    model.load_state_dict(checkpoint['model_state_dict'])
+    return model