meta_data.py

VLMEVALKIT_README = 'https://raw.githubusercontent.com/open-compass/VLMEvalKit/main/README.md'
# CONSTANTS-CITATION
CITATION_BUTTON_TEXT = r"""@article{guo2025sok,
  title={{Frontier AI's Impact on the Cybersecurity Landscape}},
  author={Guo, Wenbo and Potter, Yujin and Shi, Tianneng and Wang, Zhun and Zhang, Andy and Song, Dawn},
  journal={arXiv preprint arXiv:2504.05408},
  year={2025}
}
"""
CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
# CONSTANTS-TEXT
LEADERBORAD_INTRODUCTION = """# Frontier AI Cybersecurity Observatory
### Welcome to Frontier AI Cybersecurity Observatory! This leaderboard is a collection of benchmarks relevant to cybersecurity capabilities. 
This leaderboard covers {} benchmarks.

This leaderboard was last updated: {} """
# CONSTANTS-FIELDS
# META_FIELDS = [
#     'Model'
# ]

DEFAULT_TASK = [
    'Vulnerable Code Generation', 'Attack Generation', 'CTF', 'Cyber Knowledge', 'Pen Test', 'Vulnerability Detection', 'PoC Generation', 'Patching'
]

# The README file for each benchmark
LEADERBOARD_MD = {}

LEADERBOARD_MD['CyberSecEval-3'] = """CyberSecEval-3 is a security benchmarks for LLMs. CyberSecEval-3 assesses 8 different risks across two broad categories: risk to third parties, and risk to application developers and end users.  

Paper: https://arxiv.org/abs/2408.01605 
Code: https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks 
"""
LEADERBOARD_MD['SecCodePLT'] = """ SecCodePLT is a unified and comprehensive evaluation platform for code GenAIs' risks. This benchmark consists of insecure coding tasks and cyberattack helpfulness tasks. The helpfulness tasks are designed considering five attack steps: reconnaissance, weaponization & infiltration, C2 & execution, discovery, and collection.

Paper: https://arxiv.org/abs/2410.11096 
Code: https://github.com/CodeSecPLT/CodeSecPLT 
"""
LEADERBOARD_MD['RedCode'] = """RedCode is a benchmark for risky code execution and generation: (1) RedCode-Exec provides challenging prompts that could lead to risky code execution, aiming to evaluate code agents' ability to recognize and handle unsafe code. (2) RedCode-Gen provides 160 prompts with function signatures and docstrings as input to assess whether code agents will follow instructions to generate harmful code or software.

Paper: https://arxiv.org/abs/2411.07781
Code: https://github.com/AI-secure/RedCode 
"""
LEADERBOARD_MD['CyBench'] = """Cybench is a framework for specifying cybersecurity tasks and evaluating agents on those tasks. This includes 40 professional-level Capture the Flag (CTF) tasks from 4 distinct CTF competitions, chosen to be recent, meaningful, and spanning a wide range of difficulties.

Paper: https://arxiv.org/abs/2408.08926 
Code: https://github.com/andyzorigin/cybench 
"""
LEADERBOARD_MD['NYU CTF Bench'] = """This assesses LLMs in solving CTF challenges. This includes a diverse range of CTF challenges from popular competitions.

Paper: https://arxiv.org/abs/2406.05590 
Code: https://github.com/NYU-LLM-CTF/NYU_CTF_Bench 
"""
LEADERBOARD_MD['CyberBench'] = """CyberBench is a multi-task benchmark to evaluate the model knowledge in cybersecurity. 

Paper: https://zefang-liu.github.io/files/liu2024cyberbench_paper.pdf 
Code: https://github.com/jpmorganchase/CyberBench
"""
LEADERBOARD_MD['CyberMetric'] = """CyberMetric is designed to accurately test the general knowledge of LLMs in cybersecurity. CyberMetric-80, CyberMetric-500, CyberMetric-2000, and CyberMetric-10000 are multiple-choice Q&A benchmark datasets comprising 80, 500, 2000, and 10,000 questions, respectively.

Paper: https://arxiv.org/abs/2402.07688 
Code: https://github.com/cybermetric/CyberMetric/tree/main 
"""
LEADERBOARD_MD['TACTL'] = """Threat Actor Competency Test for LLMs (TACTL) is a multiple-choice benchmark as a challenging offensive cyber knowledge test. 

Paper: https://arxiv.org/abs/2502.15797
Code: They plan to open-source TACTL (https://gbhackers.com/mitre-releases-occult-framework/). 
"""
LEADERBOARD_MD['AutoPenBench'] = """AutoPenBench is an open benchmark for evaluating generative agents in automated penetration testing.

Paper: https://arxiv.org/abs/2410.03225 
Code: https://github.com/lucagioacchini/auto-pen-bench 
"""
LEADERBOARD_MD['PrimeVul'] = """PrimeVul is a dataset for training and evaluating code LMs for vulnerability detection.

Paper: https://arxiv.org/abs/2403.18624 
Code: https://github.com/DLVulDet/PrimeVul
"""
LEADERBOARD_MD['CRUXEval'] = """CRUXEval (Code Reasoning, Understanding, and eXecution Evaluation) is a benchmark consisting of 800 Python functions (3-13 lines).

Paper: https://arxiv.org/abs/2401.03065 
Code: https://github.com/facebookresearch/cruxeval 
"""
LEADERBOARD_MD['SWE-bench-verified'] = """This is a human-validated subset of SWE-bench that more reliably evaluates AI models' ability to solve real-world software issues.

Paper: https://openai.com/index/introducing-swe-bench-verified/ 
Code: https://github.com/swe-bench/SWE-bench 
"""