Spaces:
Runtime error
Runtime error
| VLMEVALKIT_README = 'https://raw.githubusercontent.com/open-compass/VLMEvalKit/main/README.md' | |
| # CONSTANTS-CITATION | |
| CITATION_BUTTON_TEXT = r"""@article{guo2025sok, | |
| title={{Frontier AI's Impact on the Cybersecurity Landscape}}, | |
| author={Guo, Wenbo and Potter, Yujin and Shi, Tianneng and Wang, Zhun and Zhang, Andy and Song, Dawn}, | |
| journal={arXiv preprint arXiv:2504.05408}, | |
| year={2025} | |
| } | |
| """ | |
| CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results" | |
| # CONSTANTS-TEXT | |
| LEADERBORAD_INTRODUCTION = """# Frontier AI Cybersecurity Observatory | |
| ### Welcome to Frontier AI Cybersecurity Observatory! This leaderboard is a collection of benchmarks relevant to cybersecurity capabilities. | |
| This leaderboard covers {} benchmarks. | |
| This leaderboard was last updated: {} """ | |
| # CONSTANTS-FIELDS | |
| # META_FIELDS = [ | |
| # 'Model' | |
| # ] | |
| DEFAULT_TASK = [ | |
| 'Vulnerable Code Generation', 'Attack Generation', 'CTF', 'Cyber Knowledge', 'Pen Test', 'Vulnerability Detection', 'PoC Generation', 'Patching' | |
| ] | |
| # The README file for each benchmark | |
| LEADERBOARD_MD = {} | |
| LEADERBOARD_MD['CyberSecEval-3'] = """CyberSecEval-3 is a security benchmarks for LLMs. CyberSecEval-3 assesses 8 different risks across two broad categories: risk to third parties, and risk to application developers and end users. | |
| Paper: https://arxiv.org/abs/2408.01605 | |
| Code: https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks | |
| """ | |
| LEADERBOARD_MD['SecCodePLT'] = """ SecCodePLT is a unified and comprehensive evaluation platform for code GenAIs' risks. This benchmark consists of insecure coding tasks and cyberattack helpfulness tasks. The helpfulness tasks are designed considering five attack steps: reconnaissance, weaponization & infiltration, C2 & execution, discovery, and collection. | |
| Paper: https://arxiv.org/abs/2410.11096 | |
| Code: https://github.com/CodeSecPLT/CodeSecPLT | |
| """ | |
| LEADERBOARD_MD['RedCode'] = """RedCode is a benchmark for risky code execution and generation: (1) RedCode-Exec provides challenging prompts that could lead to risky code execution, aiming to evaluate code agents' ability to recognize and handle unsafe code. (2) RedCode-Gen provides 160 prompts with function signatures and docstrings as input to assess whether code agents will follow instructions to generate harmful code or software. | |
| Paper: https://arxiv.org/abs/2411.07781 | |
| Code: https://github.com/AI-secure/RedCode | |
| """ | |
| LEADERBOARD_MD['CyBench'] = """Cybench is a framework for specifying cybersecurity tasks and evaluating agents on those tasks. This includes 40 professional-level Capture the Flag (CTF) tasks from 4 distinct CTF competitions, chosen to be recent, meaningful, and spanning a wide range of difficulties. | |
| Paper: https://arxiv.org/abs/2408.08926 | |
| Code: https://github.com/andyzorigin/cybench | |
| """ | |
| LEADERBOARD_MD['NYU CTF Bench'] = """This assesses LLMs in solving CTF challenges. This includes a diverse range of CTF challenges from popular competitions. | |
| Paper: https://arxiv.org/abs/2406.05590 | |
| Code: https://github.com/NYU-LLM-CTF/NYU_CTF_Bench | |
| """ | |
| LEADERBOARD_MD['CyberBench'] = """CyberBench is a multi-task benchmark to evaluate the model knowledge in cybersecurity. | |
| Paper: https://zefang-liu.github.io/files/liu2024cyberbench_paper.pdf | |
| Code: https://github.com/jpmorganchase/CyberBench | |
| """ | |
| LEADERBOARD_MD['CyberMetric'] = """CyberMetric is designed to accurately test the general knowledge of LLMs in cybersecurity. CyberMetric-80, CyberMetric-500, CyberMetric-2000, and CyberMetric-10000 are multiple-choice Q&A benchmark datasets comprising 80, 500, 2000, and 10,000 questions, respectively. | |
| Paper: https://arxiv.org/abs/2402.07688 | |
| Code: https://github.com/cybermetric/CyberMetric/tree/main | |
| """ | |
| LEADERBOARD_MD['TACTL'] = """Threat Actor Competency Test for LLMs (TACTL) is a multiple-choice benchmark as a challenging offensive cyber knowledge test. | |
| Paper: https://arxiv.org/abs/2502.15797 | |
| Code: They plan to open-source TACTL (https://gbhackers.com/mitre-releases-occult-framework/). | |
| """ | |
| LEADERBOARD_MD['AutoPenBench'] = """AutoPenBench is an open benchmark for evaluating generative agents in automated penetration testing. | |
| Paper: https://arxiv.org/abs/2410.03225 | |
| Code: https://github.com/lucagioacchini/auto-pen-bench | |
| """ | |
| LEADERBOARD_MD['PrimeVul'] = """PrimeVul is a dataset for training and evaluating code LMs for vulnerability detection. | |
| Paper: https://arxiv.org/abs/2403.18624 | |
| Code: https://github.com/DLVulDet/PrimeVul | |
| """ | |
| LEADERBOARD_MD['CRUXEval'] = """CRUXEval (Code Reasoning, Understanding, and eXecution Evaluation) is a benchmark consisting of 800 Python functions (3-13 lines). | |
| Paper: https://arxiv.org/abs/2401.03065 | |
| Code: https://github.com/facebookresearch/cruxeval | |
| """ | |
| LEADERBOARD_MD['SWE-bench-verified'] = """This is a human-validated subset of SWE-bench that more reliably evaluates AI models' ability to solve real-world software issues. | |
| Paper: https://openai.com/index/introducing-swe-bench-verified/ | |
| Code: https://github.com/swe-bench/SWE-bench | |
| """ |