name: Nightly Evaluation Run

on:
  schedule:
    - cron: '0 3 * * *'  # Run at 3am UTC every day
  workflow_dispatch:  # Allow manual triggering

jobs:
  run-evals:
    runs-on: ubuntu-latest
    timeout-minutes: 1440  # 24 hours timeout
    steps:
      - uses: actions/checkout@v3
      
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.12'
          
      - name: Install dependencies
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          uv sync --frozen --extra dev
          
      - name: Run evaluations with checkpointing
        env:
          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
          HUGGINGFACE_ACCESS_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
          N_SENTENCES: 20
          MAX_LANGUAGES: 150
        run: |
          uv run huggingface-cli login --token ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
          uv run evals/download_data.py
          
          # Run evaluations with periodic checkpointing
          uv run python -c "
          import time
          import subprocess
          import json
          import os
          
          # Check if we have existing results to resume from
          if os.path.exists('results.json'):
              print('Found existing results.json, will resume from checkpoint')
          
          # Run the main evaluation
          try:
              subprocess.run(['uv', 'run', 'evals/main.py'], check=True)
          except subprocess.CalledProcessError as e:
              print(f'Evaluation failed: {e}')
              # Save current state even if failed
              if os.path.exists('results.json'):
                  print('Saving checkpoint before exit...')
              exit(1)
          "
          
      - name: Commit changes
        env:
          GH_PAT: ${{ secrets.GH_PAT }}
        run: |
          git config --local user.email "github-actions[bot]@users.noreply.github.com"
          git config --local user.name "github-actions[bot]"
          git config --local --unset-all http.https://github.com/.extraheader
          git remote set-url origin https://${GH_PAT}@github.com/datenlabor-bmz/ai-language-monitor.git
          git add results.json models.json languages.json checkpoint.json
          git commit -m "Update evaluation results" || echo "No changes to commit"
          git push origin HEAD:main 
      
      - name: Upload to Hugging Face
        env:
          HUGGINGFACE_ACCESS_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
        run: |
          uv run python -c '
          from huggingface_hub import upload_folder
          import os
          
          upload_folder(
              folder_path=".",
              path_in_repo="/",
              allow_patterns=["results.json", "models.json", "languages.json"],
              repo_id="fair-forward/evals-for-every-language",
              repo_type="space",
              token=os.environ["HUGGINGFACE_ACCESS_TOKEN"],
              commit_message="Upload from nightly evaluation run",
          )
          '