frontend/src/pages/LeaderboardPage/components/Leaderboard/hooks/useDataUtils.js

import { useMemo } from "react";
import {
  looksLikeRegex,
  parseSearchQuery,
  getValueByPath,
} from "../utils/searchUtils";
import { ALLOWED_MODELS, isModelAllowed } from "../constants/allowedModels";

// Calculate min/max averages
export const useAverageRange = (data) => {
  return useMemo(() => {
    const averages = data.map((item) => item.model.average_score);
    return {
      minAverage: Math.min(...averages),
      maxAverage: Math.max(...averages),
    };
  }, [data]);
};

// Generate colors for scores
export const useColorGenerator = (minAverage, maxAverage) => {
  return useMemo(() => {
    const colorCache = new Map();
    return (value) => {
      const cached = colorCache.get(value);
      if (cached) return cached;

      const normalizedValue = (value - minAverage) / (maxAverage - minAverage);
      const red = Math.round(255 * (1 - normalizedValue) * 1);
      const green = Math.round(255 * normalizedValue) * 1;
      // const color = `rgba(${red}, ${green}, 0, 1)`;
      const color = `rgba(${red}, 0, ${green}, 1)`;
      colorCache.set(value, color);
      return color;
    };
  }, [minAverage, maxAverage]);
};

// Process data with boolean standardization
export const useProcessedData = (data, averageMode, visibleColumns) => {
  return useMemo(() => {
    // First filter and process existing models
    let processed = data.map((item) => {
      // Calculate average score for Greek datasets
      const greekDatasets = ['multifin', 'qa', 'fns', 'finnum', 'fintext'];
      const greekScores = greekDatasets
        .filter(dataset => item.evaluations[dataset]?.normalized_score !== undefined)
        .map(dataset => item.evaluations[dataset].normalized_score);
      
      const greekAverage = greekScores.length > 0
        ? greekScores.reduce((a, b) => a + b, 0) / greekScores.length
        : null;

      // Add Greek average to evaluations object
      const enhancedEvaluations = {
        ...item.evaluations,
        greek_average: greekAverage
      };
      
      // Calculate average score for all visible evaluations (including greek_average, but excluding specific Greek datasets)
      const includedEvaluations = {};
      // Copy all non-Greek evaluation data
      Object.entries(item.evaluations).forEach(([key, value]) => {
        if (!greekDatasets.includes(key)) {
          includedEvaluations[key] = value;
        }
      });
      // Add Greek average
      if (greekAverage !== null) {
        includedEvaluations.greek_average = { normalized_score: greekAverage };
      }
      
      const evaluationScores = Object.entries(includedEvaluations)
        .filter(([key]) => {
          if (averageMode === "all") return true;
          return visibleColumns.includes(`evaluations.${key}.normalized_score`);
        })
        .map(([, value]) => value.normalized_score);

      const average =
        evaluationScores.length > 0
          ? evaluationScores.reduce((a, b) => a + b, 0) /
            evaluationScores.length
          : averageMode === "visible"
          ? null
          : 0;

      // Boolean standardization
      const standardizedFeatures = {
        ...item.features,
        is_moe: Boolean(item.features.is_moe),
        is_flagged: Boolean(item.features.is_flagged),
        is_highlighted_by_maintainer: Boolean(
          item.features.is_highlighted_by_maintainer
        ),
        is_merged: Boolean(item.features.is_merged),
        is_not_available_on_hub: Boolean(item.features.is_not_available_on_hub),
      };

      return {
        ...item,
        features: standardizedFeatures,
        evaluations: enhancedEvaluations, // Use enhanced evaluations
        model: {
          ...item.model,
          has_chat_template: Boolean(item.model.has_chat_template),
          average_score: average,
        },
      };
    });

    // Create mapping of existing models, check which ones are in the allowed list
    const existingModelsMap = {};
    const filteredModels = [];
    
    processed.forEach(model => {
      if (isModelAllowed(model.model.name)) {
        existingModelsMap[model.model.name] = model;
        filteredModels.push(model);
      }
    });
    
    // Add "missing" entries, create placeholders for models in the allowed list that don't exist
    ALLOWED_MODELS.forEach(allowedModelName => {
      // Check if a matching model already exists
      const modelExists = Object.keys(existingModelsMap).some(name => 
        name.toLowerCase().includes(allowedModelName.toLowerCase())
      );
      
      if (!modelExists) {
        // Create a "missing" placeholder
        filteredModels.push({
          id: `missing-${allowedModelName}`,
          model: {
            name: allowedModelName,
            average_score: null,
            type: "Unknown",
          },
          evaluations: {
            greek_average: null
          },
          features: {
            is_moe: false,
            is_flagged: false,
            is_highlighted_by_maintainer: false,
            is_merged: false,
            is_not_available_on_hub: true,
          },
          metadata: {
            submission_date: new Date().toISOString(),
          },
          isMissing: true, // Mark as missing
        });
      }
    });

    // Sort the results
    filteredModels.sort((a, b) => {
      // Place missing models at the end
      if (a.isMissing && !b.isMissing) return 1;
      if (!a.isMissing && b.isMissing) return -1;
      
      // If both are missing or both are not missing, sort by average score
      if (a.model.average_score === null && b.model.average_score === null)
        return 0;
      if (a.model.average_score === null) return 1;
      if (b.model.average_score === null) return -1;
      return b.model.average_score - a.model.average_score;
    });

    return filteredModels.map((item, index) => ({
      ...item,
      static_rank: index + 1,
    }));
  }, [data, averageMode, visibleColumns]);
};

// Common filtering logic
export const useFilteredData = (
  processedData,
  selectedPrecisions,
  selectedTypes,
  paramsRange,
  searchValue,
  selectedBooleanFilters,
  rankingMode,
  pinnedModels = [],
  isOfficialProviderActive = false
) => {
  return useMemo(() => {
    const pinnedData = processedData.filter((row) => {
      return pinnedModels.includes(row.id);
    });
    const unpinnedData = processedData.filter((row) => {
      return !pinnedModels.includes(row.id);
    });

    let filteredUnpinned = unpinnedData;

    // Filter by official providers
    if (isOfficialProviderActive) {
      filteredUnpinned = filteredUnpinned.filter(
        (row) =>
          row.features?.is_highlighted_by_maintainer ||
          row.metadata?.is_highlighted_by_maintainer
      );
    }

    // Filter by precision
    if (selectedPrecisions.length > 0) {
      filteredUnpinned = filteredUnpinned.filter((row) =>
        selectedPrecisions.includes(row.model.precision)
      );
    }

    // Filter by type
    if (selectedTypes.length > 0) {
      filteredUnpinned = filteredUnpinned.filter((row) => {
        const modelType = row.model.type?.toLowerCase().trim();
        return selectedTypes.some((type) => modelType?.includes(type));
      });
    }

    // Filter by parameters
    filteredUnpinned = filteredUnpinned.filter((row) => {
      // Skip parameter filtering if no filter is active
      if (paramsRange[0] === -1 && paramsRange[1] === 140) return true;

      const params =
        row.metadata?.params_billions || row.features?.params_billions;
      if (params === undefined || params === null) return false;
      return params >= paramsRange[0] && params < paramsRange[1];
    });

    // Filter by search
    if (searchValue) {
      const searchQueries = searchValue
        .split(";")
        .map((q) => q.trim())
        .filter((q) => q);
      if (searchQueries.length > 0) {
        filteredUnpinned = filteredUnpinned.filter((row) => {
          return searchQueries.some((query) => {
            const { specialSearches, textSearch } = parseSearchQuery(query);

            const specialSearchMatch = specialSearches.every(
              ({ field, value }) => {
                const fieldValue = getValueByPath(row, field)
                  ?.toString()
                  .toLowerCase();
                return fieldValue?.includes(value.toLowerCase());
              }
            );

            if (!specialSearchMatch) return false;
            if (!textSearch) return true;

            const modelName = row.model.name.toLowerCase();
            const searchLower = textSearch.toLowerCase();

            if (looksLikeRegex(textSearch)) {
              try {
                const regex = new RegExp(textSearch, "i");
                return regex.test(modelName);
              } catch (e) {
                return modelName.includes(searchLower);
              }
            } else {
              return modelName.includes(searchLower);
            }
          });
        });
      }
    }

    // Filter by booleans
    if (selectedBooleanFilters.length > 0) {
      filteredUnpinned = filteredUnpinned.filter((row) => {
        return selectedBooleanFilters.every((filter) => {
          const filterValue =
            typeof filter === "object" ? filter.value : filter;

          // Maintainer's Highlight keeps positive logic
          if (filterValue === "is_highlighted_by_maintainer") {
            return row.features[filterValue];
          }

          // For all other filters, invert the logic
          if (filterValue === "is_not_available_on_hub") {
            return row.features[filterValue];
          }

          return !row.features[filterValue];
        });
      });
    }

    // Create ordered array of pinned models respecting pinnedModels order
    const orderedPinnedData = pinnedModels
      .map((pinnedModelId) =>
        pinnedData.find((item) => item.id === pinnedModelId)
      )
      .filter(Boolean);

    // Combine all filtered data
    const allFilteredData = [...filteredUnpinned, ...orderedPinnedData];

    // Sort all data by average_score for dynamic_rank
    const sortedByScore = [...allFilteredData].sort((a, b) => {
      // Si les scores moyens sont différents, trier par score
      if (a.model.average_score !== b.model.average_score) {
        if (a.model.average_score === null && b.model.average_score === null)
          return 0;
        if (a.model.average_score === null) return 1;
        if (b.model.average_score === null) return -1;
        return b.model.average_score - a.model.average_score;
      }

      // Si les scores sont égaux, comparer le nom du modèle et la date de soumission
      if (a.model.name === b.model.name) {
        // Si même nom, trier par date de soumission (la plus récente d'abord)
        const dateA = new Date(a.metadata?.submission_date || 0);
        const dateB = new Date(b.metadata?.submission_date || 0);
        return dateB - dateA;
      }

      // Si noms différents, trier par nom
      return a.model.name.localeCompare(b.model.name);
    });

    // Create Map to store dynamic_ranks
    const dynamicRankMap = new Map();
    sortedByScore.forEach((item, index) => {
      dynamicRankMap.set(item.id, index + 1);
    });

    // Add ranks to final data
    const finalData = [...orderedPinnedData, ...filteredUnpinned].map(
      (item) => {
        return {
          ...item,
          dynamic_rank: dynamicRankMap.get(item.id),
          rank: item.isPinned
            ? pinnedModels.indexOf(item.id) + 1
            : rankingMode === "static"
            ? item.static_rank
            : dynamicRankMap.get(item.id),
          isPinned: pinnedModels.includes(item.id),
        };
      }
    );

    return finalData;
  }, [
    processedData,
    selectedPrecisions,
    selectedTypes,
    paramsRange,
    searchValue,
    selectedBooleanFilters,
    rankingMode,
    pinnedModels,
    isOfficialProviderActive,
  ]);
};

// Column visibility management
export const useColumnVisibility = (visibleColumns = []) => {
  // Create secure visibility object
  const columnVisibility = useMemo(() => {
    // Check visible columns
    const safeVisibleColumns = Array.isArray(visibleColumns)
      ? visibleColumns
      : [];

    const visibility = {};
    try {
      safeVisibleColumns.forEach((columnKey) => {
        if (typeof columnKey === "string") {
          visibility[columnKey] = true;
        }
      });
    } catch (error) {
      console.warn("Error in useColumnVisibility:", error);
    }
    return visibility;
  }, [visibleColumns]);

  return columnVisibility;
};