File size: 5,014 Bytes
84eb7fa bb8ff6c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 |
VLMEVALKIT_README = 'https://raw.githubusercontent.com/open-compass/VLMEvalKit/main/README.md'
# CONSTANTS-CITATION
CITATION_BUTTON_TEXT = r"""@article{guo2025sok,
title={{Frontier AI's Impact on the Cybersecurity Landscape}},
author={Guo, Wenbo and Potter, Yujin and Shi, Tianneng and Wang, Zhun and Zhang, Andy and Song, Dawn},
journal={arXiv preprint arXiv:2504.05408},
year={2025}
}
"""
CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
# CONSTANTS-TEXT
LEADERBORAD_INTRODUCTION = """# Frontier AI Cybersecurity Observatory
### Welcome to Frontier AI Cybersecurity Observatory! This leaderboard is a collection of benchmarks relevant to cybersecurity capabilities.
This leaderboard covers {} benchmarks.
This leaderboard was last updated: {} """
# CONSTANTS-FIELDS
# META_FIELDS = [
# 'Model'
# ]
DEFAULT_TASK = [
'Vulnerable Code Generation', 'Attack Generation', 'CTF', 'Cyber Knowledge', 'Pen Test', 'Vulnerability Detection', 'PoC Generation', 'Patching'
]
# The README file for each benchmark
LEADERBOARD_MD = {}
LEADERBOARD_MD['CyberSecEval-3'] = """CyberSecEval-3 is a security benchmarks for LLMs. CyberSecEval-3 assesses 8 different risks across two broad categories: risk to third parties, and risk to application developers and end users.
Paper: https://arxiv.org/abs/2408.01605
Code: https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks
"""
LEADERBOARD_MD['SecCodePLT'] = """ SecCodePLT is a unified and comprehensive evaluation platform for code GenAIs' risks. This benchmark consists of insecure coding tasks and cyberattack helpfulness tasks. The helpfulness tasks are designed considering five attack steps: reconnaissance, weaponization & infiltration, C2 & execution, discovery, and collection.
Paper: https://arxiv.org/abs/2410.11096
Code: https://github.com/CodeSecPLT/CodeSecPLT
"""
LEADERBOARD_MD['RedCode'] = """RedCode is a benchmark for risky code execution and generation: (1) RedCode-Exec provides challenging prompts that could lead to risky code execution, aiming to evaluate code agents' ability to recognize and handle unsafe code. (2) RedCode-Gen provides 160 prompts with function signatures and docstrings as input to assess whether code agents will follow instructions to generate harmful code or software.
Paper: https://arxiv.org/abs/2411.07781
Code: https://github.com/AI-secure/RedCode
"""
LEADERBOARD_MD['CyBench'] = """Cybench is a framework for specifying cybersecurity tasks and evaluating agents on those tasks. This includes 40 professional-level Capture the Flag (CTF) tasks from 4 distinct CTF competitions, chosen to be recent, meaningful, and spanning a wide range of difficulties.
Paper: https://arxiv.org/abs/2408.08926
Code: https://github.com/andyzorigin/cybench
"""
LEADERBOARD_MD['NYU CTF Bench'] = """This assesses LLMs in solving CTF challenges. This includes a diverse range of CTF challenges from popular competitions.
Paper: https://arxiv.org/abs/2406.05590
Code: https://github.com/NYU-LLM-CTF/NYU_CTF_Bench
"""
LEADERBOARD_MD['CyberBench'] = """CyberBench is a multi-task benchmark to evaluate the model knowledge in cybersecurity.
Paper: https://zefang-liu.github.io/files/liu2024cyberbench_paper.pdf
Code: https://github.com/jpmorganchase/CyberBench
"""
LEADERBOARD_MD['CyberMetric'] = """CyberMetric is designed to accurately test the general knowledge of LLMs in cybersecurity. CyberMetric-80, CyberMetric-500, CyberMetric-2000, and CyberMetric-10000 are multiple-choice Q&A benchmark datasets comprising 80, 500, 2000, and 10,000 questions, respectively.
Paper: https://arxiv.org/abs/2402.07688
Code: https://github.com/cybermetric/CyberMetric/tree/main
"""
LEADERBOARD_MD['TACTL'] = """Threat Actor Competency Test for LLMs (TACTL) is a multiple-choice benchmark as a challenging offensive cyber knowledge test.
Paper: https://arxiv.org/abs/2502.15797
Code: They plan to open-source TACTL (https://gbhackers.com/mitre-releases-occult-framework/).
"""
LEADERBOARD_MD['AutoPenBench'] = """AutoPenBench is an open benchmark for evaluating generative agents in automated penetration testing.
Paper: https://arxiv.org/abs/2410.03225
Code: https://github.com/lucagioacchini/auto-pen-bench
"""
LEADERBOARD_MD['PrimeVul'] = """PrimeVul is a dataset for training and evaluating code LMs for vulnerability detection.
Paper: https://arxiv.org/abs/2403.18624
Code: https://github.com/DLVulDet/PrimeVul
"""
LEADERBOARD_MD['CRUXEval'] = """CRUXEval (Code Reasoning, Understanding, and eXecution Evaluation) is a benchmark consisting of 800 Python functions (3-13 lines).
Paper: https://arxiv.org/abs/2401.03065
Code: https://github.com/facebookresearch/cruxeval
"""
LEADERBOARD_MD['SWE-bench-verified'] = """This is a human-validated subset of SWE-bench that more reliably evaluates AI models' ability to solve real-world software issues.
Paper: https://openai.com/index/introducing-swe-bench-verified/
Code: https://github.com/swe-bench/SWE-bench
""" |