meta_data.py

VLMEVALKIT_README = 'https://raw.githubusercontent.com/open-compass/VLMEvalKit/main/README.md'
# CONSTANTS-CITATION
CITATION_BUTTON_TEXT = r"""@article{guo2025sok,

  title={{Frontier AI's Impact on the Cybersecurity Landscape}},

  author={Guo, Wenbo and Potter, Yujin and Shi, Tianneng and Wang, Zhun and Zhang, Andy and Song, Dawn},

  journal={arXiv preprint arXiv:2504.05408},

  year={2025}

}

"""
CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
# CONSTANTS-TEXT
LEADERBORAD_INTRODUCTION = """# Cybersecurity Leaderboard

### Welcome to the Cybersecurity Leaderboard! This leaderboard is a collection of benchmarks relevant to cybersecurity capabilities. 

This leaderboard covers {} benchmarks.



This leaderboard was last updated: {} """
# CONSTANTS-FIELDS
# META_FIELDS = [
#     'Model'
# ]

DEFAULT_TASK = [
    'Vulnerable Code Generation', 'Attack Generation', 'CTF', 'Cyber Knowledge', 'Pen Test', 'Vulnerability Detection', 'PoC Generation', 'Patching'
]

# The README file for each benchmark
LEADERBOARD_MD = {}

LEADERBOARD_MD['CyberSecEval-3'] = """CyberSecEval-3 is a security benchmarks for LLMs. CyberSecEval-3 assesses 8 different risks across two broad categories: risk to third parties, and risk to application developers and end users.  



Paper: https://arxiv.org/abs/2408.01605 

Code: https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks 

"""
LEADERBOARD_MD['SecCodePLT'] = """ SecCodePLT is a unified and comprehensive evaluation platform for code GenAIs' risks. This benchmark consists of insecure coding tasks and cyberattack helpfulness tasks. The helpfulness tasks are designed considering five attack steps: reconnaissance, weaponization & infiltration, C2 & execution, discovery, and collection.



Paper: https://arxiv.org/abs/2410.11096 

Code: https://github.com/CodeSecPLT/CodeSecPLT 

"""
LEADERBOARD_MD['RedCode'] = """RedCode is a benchmark for risky code execution and generation: (1) RedCode-Exec provides challenging prompts that could lead to risky code execution, aiming to evaluate code agents' ability to recognize and handle unsafe code. (2) RedCode-Gen provides 160 prompts with function signatures and docstrings as input to assess whether code agents will follow instructions to generate harmful code or software.



Paper: https://arxiv.org/abs/2411.07781

Code: https://github.com/AI-secure/RedCode 

"""
LEADERBOARD_MD['CyBench'] = """Cybench is a framework for specifying cybersecurity tasks and evaluating agents on those tasks. This includes 40 professional-level Capture the Flag (CTF) tasks from 4 distinct CTF competitions, chosen to be recent, meaningful, and spanning a wide range of difficulties.



Paper: https://arxiv.org/abs/2408.08926 

Code: https://github.com/andyzorigin/cybench 

"""
LEADERBOARD_MD['NYU CTF Bench'] = """This assesses LLMs in solving CTF challenges. This includes a diverse range of CTF challenges from popular competitions.



Paper: https://arxiv.org/abs/2406.05590 

Code: https://github.com/NYU-LLM-CTF/NYU_CTF_Bench 

"""
LEADERBOARD_MD['CyberBench'] = """CyberBench is a multi-task benchmark to evaluate the model knowledge in cybersecurity. 



Paper: https://zefang-liu.github.io/files/liu2024cyberbench_paper.pdf 

Code: https://github.com/jpmorganchase/CyberBench

"""
LEADERBOARD_MD['CyberMetric'] = """CyberMetric is designed to accurately test the general knowledge of LLMs in cybersecurity. CyberMetric-80, CyberMetric-500, CyberMetric-2000, and CyberMetric-10000 are multiple-choice Q&A benchmark datasets comprising 80, 500, 2000, and 10,000 questions, respectively.



Paper: https://arxiv.org/abs/2402.07688 

Code: https://github.com/cybermetric/CyberMetric/tree/main 

"""
LEADERBOARD_MD['TACTL'] = """Threat Actor Competency Test for LLMs (TACTL) is a multiple-choice benchmark as a challenging offensive cyber knowledge test. 



Paper: https://arxiv.org/abs/2502.15797

Code: They plan to open-source TACTL (https://gbhackers.com/mitre-releases-occult-framework/). 

"""
LEADERBOARD_MD['AutoPenBench'] = """AutoPenBench is an open benchmark for evaluating generative agents in automated penetration testing.



Paper: https://arxiv.org/abs/2410.03225 

Code: https://github.com/lucagioacchini/auto-pen-bench 

"""
LEADERBOARD_MD['PrimeVul'] = """PrimeVul is a dataset for training and evaluating code LMs for vulnerability detection.



Paper: https://arxiv.org/abs/2403.18624 

Code: https://github.com/DLVulDet/PrimeVul

"""
LEADERBOARD_MD['CRUXEval'] = """CRUXEval (Code Reasoning, Understanding, and eXecution Evaluation) is a benchmark consisting of 800 Python functions (3-13 lines).



Paper: https://arxiv.org/abs/2401.03065 

Code: https://github.com/facebookresearch/cruxeval 

"""
LEADERBOARD_MD['SWE-bench-verified'] = """This is a human-validated subset of SWE-bench that more reliably evaluates AI models' ability to solve real-world software issues.



Paper: https://openai.com/index/introducing-swe-bench-verified/ 

Code: https://github.com/swe-bench/SWE-bench 

"""