"""
AttackArgs Class
================
"""
from dataclasses import dataclass, field
import json
import os
import sys
import time
from typing import Dict, Optional
import textattack
from textattack.shared.utils import ARGS_SPLIT_TOKEN, load_module_from_file
from .attack import Attack
from .dataset_args import DatasetArgs
from .model_args import ModelArgs
ATTACK_RECIPE_NAMES = {
"alzantot": "textattack.attack_recipes.GeneticAlgorithmAlzantot2018",
"bae": "textattack.attack_recipes.BAEGarg2019",
"bert-attack": "textattack.attack_recipes.BERTAttackLi2020",
"faster-alzantot": "textattack.attack_recipes.FasterGeneticAlgorithmJia2019",
"deepwordbug": "textattack.attack_recipes.DeepWordBugGao2018",
"hotflip": "textattack.attack_recipes.HotFlipEbrahimi2017",
"input-reduction": "textattack.attack_recipes.InputReductionFeng2018",
"kuleshov": "textattack.attack_recipes.Kuleshov2017",
"morpheus": "textattack.attack_recipes.MorpheusTan2020",
"seq2sick": "textattack.attack_recipes.Seq2SickCheng2018BlackBox",
"textbugger": "textattack.attack_recipes.TextBuggerLi2018",
"textfooler": "textattack.attack_recipes.TextFoolerJin2019",
"pwws": "textattack.attack_recipes.PWWSRen2019",
"iga": "textattack.attack_recipes.IGAWang2019",
"pruthi": "textattack.attack_recipes.Pruthi2019",
"pso": "textattack.attack_recipes.PSOZang2020",
"checklist": "textattack.attack_recipes.CheckList2020",
"clare": "textattack.attack_recipes.CLARE2020",
"a2t": "textattack.attack_recipes.A2TYoo2021",
}
BLACK_BOX_TRANSFORMATION_CLASS_NAMES = {
"random-synonym-insertion": "textattack.transformations.RandomSynonymInsertion",
"word-deletion": "textattack.transformations.WordDeletion",
"word-swap-embedding": "textattack.transformations.WordSwapEmbedding",
"word-swap-homoglyph": "textattack.transformations.WordSwapHomoglyphSwap",
"word-swap-inflections": "textattack.transformations.WordSwapInflections",
"word-swap-neighboring-char-swap": "textattack.transformations.WordSwapNeighboringCharacterSwap",
"word-swap-random-char-deletion": "textattack.transformations.WordSwapRandomCharacterDeletion",
"word-swap-random-char-insertion": "textattack.transformations.WordSwapRandomCharacterInsertion",
"word-swap-random-char-substitution": "textattack.transformations.WordSwapRandomCharacterSubstitution",
"word-swap-wordnet": "textattack.transformations.WordSwapWordNet",
"word-swap-masked-lm": "textattack.transformations.WordSwapMaskedLM",
"word-swap-hownet": "textattack.transformations.WordSwapHowNet",
"word-swap-qwerty": "textattack.transformations.WordSwapQWERTY",
}
WHITE_BOX_TRANSFORMATION_CLASS_NAMES = {
"word-swap-gradient": "textattack.transformations.WordSwapGradientBased"
}
CONSTRAINT_CLASS_NAMES = {
#
# Semantics constraints
#
"embedding": "textattack.constraints.semantics.WordEmbeddingDistance",
"bert": "textattack.constraints.semantics.sentence_encoders.BERT",
"infer-sent": "textattack.constraints.semantics.sentence_encoders.InferSent",
"thought-vector": "textattack.constraints.semantics.sentence_encoders.ThoughtVector",
"use": "textattack.constraints.semantics.sentence_encoders.UniversalSentenceEncoder",
"muse": "textattack.constraints.semantics.sentence_encoders.MultilingualUniversalSentenceEncoder",
"bert-score": "textattack.constraints.semantics.BERTScore",
#
# Grammaticality constraints
#
"lang-tool": "textattack.constraints.grammaticality.LanguageTool",
"part-of-speech": "textattack.constraints.grammaticality.PartOfSpeech",
"goog-lm": "textattack.constraints.grammaticality.language_models.GoogleLanguageModel",
"gpt2": "textattack.constraints.grammaticality.language_models.GPT2",
"learning-to-write": "textattack.constraints.grammaticality.language_models.LearningToWriteLanguageModel",
"cola": "textattack.constraints.grammaticality.COLA",
#
# Overlap constraints
#
"bleu": "textattack.constraints.overlap.BLEU",
"chrf": "textattack.constraints.overlap.chrF",
"edit-distance": "textattack.constraints.overlap.LevenshteinEditDistance",
"meteor": "textattack.constraints.overlap.METEOR",
"max-words-perturbed": "textattack.constraints.overlap.MaxWordsPerturbed",
#
# Pre-transformation constraints
#
"repeat": "textattack.constraints.pre_transformation.RepeatModification",
"stopword": "textattack.constraints.pre_transformation.StopwordModification",
"max-word-index": "textattack.constraints.pre_transformation.MaxWordIndexModification",
}
SEARCH_METHOD_CLASS_NAMES = {
"beam-search": "textattack.search_methods.BeamSearch",
"greedy": "textattack.search_methods.GreedySearch",
"ga-word": "textattack.search_methods.GeneticAlgorithm",
"greedy-word-wir": "textattack.search_methods.GreedyWordSwapWIR",
"pso": "textattack.search_methods.ParticleSwarmOptimization",
}
GOAL_FUNCTION_CLASS_NAMES = {
#
# Classification goal functions
#
"targeted-classification": "textattack.goal_functions.classification.TargetedClassification",
"untargeted-classification": "textattack.goal_functions.classification.UntargetedClassification",
"input-reduction": "textattack.goal_functions.classification.InputReduction",
#
# Text goal functions
#
"minimize-bleu": "textattack.goal_functions.text.MinimizeBleu",
"non-overlapping-output": "textattack.goal_functions.text.NonOverlappingOutput",
"text-to-text": "textattack.goal_functions.text.TextToTextGoalFunction",
}
@dataclass
class AttackArgs:
"""Attack arguments to be passed to :class:`~textattack.Attacker`.
Args:
num_examples (:obj:`int`, 'optional`, defaults to :obj:`10`):
The number of examples to attack. :obj:`-1` for entire dataset.
num_successful_examples (:obj:`int`, `optional`, defaults to :obj:`None`):
The number of successful adversarial examples we want. This is different from :obj:`num_examples`
as :obj:`num_examples` only cares about attacking `N` samples while :obj:`num_successful_examples` aims to keep attacking
until we have `N` successful cases.
.. note::
If set, this argument overrides `num_examples` argument.
num_examples_offset (:obj: `int`, `optional`, defaults to :obj:`0`):
The offset index to start at in the dataset.
attack_n (:obj:`bool`, `optional`, defaults to :obj:`False`):
Whether to run attack until total of `N` examples have been attacked (and not skipped).
shuffle (:obj:`bool`, `optional`, defaults to :obj:`False`):
If :obj:`True`, we randomly shuffle the dataset before attacking. However, this avoids actually shuffling
the dataset internally and opts for shuffling the list of indices of examples we want to attack. This means
:obj:`shuffle` can now be used with checkpoint saving.
query_budget (:obj:`int`, `optional`, defaults to :obj:`None`):
The maximum number of model queries allowed per example attacked.
If not set, we use the query budget set in the :class:`~textattack.goal_functions.GoalFunction` object (which by default is :obj:`float("inf")`).
.. note::
Setting this overwrites the query budget set in :class:`~textattack.goal_functions.GoalFunction` object.
checkpoint_interval (:obj:`int`, `optional`, defaults to :obj:`None`):
If set, checkpoint will be saved after attacking every `N` examples. If :obj:`None` is passed, no checkpoints will be saved.
checkpoint_dir (:obj:`str`, `optional`, defaults to :obj:`"checkpoints"`):
The directory to save checkpoint files.
random_seed (:obj:`int`, `optional`, defaults to :obj:`765`):
Random seed for reproducibility.
parallel (:obj:`False`, `optional`, defaults to :obj:`False`):
If :obj:`True`, run attack using multiple CPUs/GPUs.
num_workers_per_device (:obj:`int`, `optional`, defaults to :obj:`1`):
Number of worker processes to run per device in parallel mode (i.e. :obj:`parallel=True`). For example, if you are using GPUs and :obj:`num_workers_per_device=2`,
then 2 processes will be running in each GPU.
log_to_txt (:obj:`str`, `optional`, defaults to :obj:`None`):
If set, save attack logs as a `.txt` file to the directory specified by this argument.
If the last part of the provided path ends with `.txt` extension, it is assumed to the desired path of the log file.
log_to_csv (:obj:`str`, `optional`, defaults to :obj:`None`):
If set, save attack logs as a CSV file to the directory specified by this argument.
If the last part of the provided path ends with `.csv` extension, it is assumed to the desired path of the log file.
csv_coloring_style (:obj:`str`, `optional`, defaults to :obj:`"file"`):
Method for choosing how to mark perturbed parts of the text. Options are :obj:`"file"`, :obj:`"plain"`, and :obj:`"html"`.
:obj:`"file"` wraps perturbed parts with double brackets :obj:`[[ <text> ]]` while :obj:`"plain"` does not mark the text in any way.
log_to_visdom (:obj:`dict`, `optional`, defaults to :obj:`None`):
If set, Visdom logger is used with the provided dictionary passed as a keyword arguments to :class:`~textattack.loggers.VisdomLogger`.
Pass in empty dictionary to use default arguments. For custom logger, the dictionary should have the following
three keys and their corresponding values: :obj:`"env", "port", "hostname"`.
log_to_wandb(:obj:`dict`, `optional`, defaults to :obj:`None`):
If set, WandB logger is used with the provided dictionary passed as a keyword arguments to :class:`~textattack.loggers.WeightsAndBiasesLogger`.
Pass in empty dictionary to use default arguments. For custom logger, the dictionary should have the following
key and its corresponding value: :obj:`"project"`.
disable_stdout (:obj:`bool`, `optional`, defaults to :obj:`False`):
Disable displaying individual attack results to stdout.
silent (:obj:`bool`, `optional`, defaults to :obj:`False`):
Disable all logging (except for errors). This is stronger than :obj:`disable_stdout`.
enable_advance_metrics (:obj:`bool`, `optional`, defaults to :obj:`False`):
Enable calculation and display of optional advance post-hoc metrics like perplexity, grammar errors, etc.
"""
num_examples: int = 10
num_successful_examples: int = None
num_examples_offset: int = 0
attack_n: bool = False
shuffle: bool = False
query_budget: int = None
checkpoint_interval: int = None
checkpoint_dir: str = "checkpoints"
random_seed: int = 765 # equivalent to sum((ord(c) for c in "TEXTATTACK"))
parallel: bool = False
num_workers_per_device: int = 1
log_to_txt: str = None
log_to_csv: str = None
log_summary_to_json: str = None
csv_coloring_style: str = "file"
log_to_visdom: dict = None
log_to_wandb: dict = None
disable_stdout: bool = False
silent: bool = False
enable_advance_metrics: bool = False
metrics: Optional[Dict] = None
def __post_init__(self):
if self.num_successful_examples:
self.num_examples = None
if self.num_examples:
assert (
self.num_examples >= 0 or self.num_examples == -1
), "`num_examples` must be greater than or equal to 0 or equal to -1."
if self.num_successful_examples:
assert (
self.num_successful_examples >= 0
), "`num_examples` must be greater than or equal to 0."
if self.query_budget:
assert self.query_budget > 0, "`query_budget` must be greater than 0."
if self.checkpoint_interval:
assert (
self.checkpoint_interval > 0
), "`checkpoint_interval` must be greater than 0."
assert (
self.num_workers_per_device > 0
), "`num_workers_per_device` must be greater than 0."
@classmethod
def _add_parser_args(cls, parser):
"""Add listed args to command line parser."""
default_obj = cls()
num_ex_group = parser.add_mutually_exclusive_group(required=False)
num_ex_group.add_argument(
"--num-examples",
"-n",
type=int,
default=default_obj.num_examples,
help="The number of examples to process, -1 for entire dataset.",
)
num_ex_group.add_argument(
"--num-successful-examples",
type=int,
default=default_obj.num_successful_examples,
help="The number of successful adversarial examples we want.",
)
parser.add_argument(
"--num-examples-offset",
"-o",
type=int,
required=False,
default=default_obj.num_examples_offset,
help="The offset to start at in the dataset.",
)
parser.add_argument(
"--query-budget",
"-q",
type=int,
default=default_obj.query_budget,
help="The maximum number of model queries allowed per example attacked. Setting this overwrites the query budget set in `GoalFunction` object.",
)
parser.add_argument(
"--shuffle",
action="store_true",
default=default_obj.shuffle,
help="If `True`, shuffle the samples before we attack the dataset. Default is False.",
)
parser.add_argument(
"--attack-n",
action="store_true",
default=default_obj.attack_n,
help="Whether to run attack until `n` examples have been attacked (not skipped).",
)
parser.add_argument(
"--checkpoint-dir",
required=False,
type=str,
default=default_obj.checkpoint_dir,
help="The directory to save checkpoint files.",
)
parser.add_argument(
"--checkpoint-interval",
required=False,
type=int,
default=default_obj.checkpoint_interval,
help="If set, checkpoint will be saved after attacking every N examples. If not set, no checkpoints will be saved.",
)
parser.add_argument(
"--random-seed",
default=default_obj.random_seed,
type=int,
help="Random seed for reproducibility.",
)
parser.add_argument(
"--parallel",
action="store_true",
default=default_obj.parallel,
help="Run attack using multiple GPUs.",
)
parser.add_argument(
"--num-workers-per-device",
default=default_obj.num_workers_per_device,
type=int,
help="Number of worker processes to run per device.",
)
parser.add_argument(
"--log-to-txt",
nargs="?",
default=default_obj.log_to_txt,
const="",
type=str,
help="Path to which to save attack logs as a text file. Set this argument if you want to save text logs. "
"If the last part of the path ends with `.txt` extension, the path is assumed to path for output file.",
)
parser.add_argument(
"--log-to-csv",
nargs="?",
default=default_obj.log_to_csv,
const="",
type=str,
help="Path to which to save attack logs as a CSV file. Set this argument if you want to save CSV logs. "
"If the last part of the path ends with `.csv` extension, the path is assumed to path for output file.",
)
parser.add_argument(
"--log-summary-to-json",
nargs="?",
default=default_obj.log_summary_to_json,
const="",
type=str,
help="Path to which to save attack summary as a JSON file. Set this argument if you want to save attack results summary in a JSON. "
"If the last part of the path ends with `.json` extension, the path is assumed to path for output file.",
)
parser.add_argument(
"--csv-coloring-style",
default=default_obj.csv_coloring_style,
type=str,
help='Method for choosing how to mark perturbed parts of the text in CSV logs. Options are "file" and "plain". '
'"file" wraps text with double brackets `[[ <text> ]]` while "plain" does not mark any text. Default is "file".',
)
parser.add_argument(
"--log-to-visdom",
nargs="?",
default=None,
const='{"env": "main", "port": 8097, "hostname": "localhost"}',
type=json.loads,
help="Set this argument if you want to log attacks to Visdom. The dictionary should have the following "
'three keys and their corresponding values: `"env", "port", "hostname"`. '
'Example for command line use: `--log-to-visdom {"env": "main", "port": 8097, "hostname": "localhost"}`.',
)
parser.add_argument(
"--log-to-wandb",
nargs="?",
default=None,
const='{"project": "textattack"}',
type=json.loads,
help="Set this argument if you want to log attacks to WandB. The dictionary should have the following "
'key and its corresponding value: `"project"`. '
'Example for command line use: `--log-to-wandb {"project": "textattack"}`.',
)
parser.add_argument(
"--disable-stdout",
action="store_true",
default=default_obj.disable_stdout,
help="Disable logging attack results to stdout",
)
parser.add_argument(
"--silent",
action="store_true",
default=default_obj.silent,
help="Disable all logging",
)
parser.add_argument(
"--enable-advance-metrics",
action="store_true",
default=default_obj.enable_advance_metrics,
help="Enable calculation and display of optional advance post-hoc metrics like perplexity, USE distance, etc.",
)
return parser
@classmethod
def create_loggers_from_args(cls, args):
"""Creates AttackLogManager from an AttackArgs object."""
assert isinstance(
args, cls
), f"Expect args to be of type `{type(cls)}`, but got type `{type(args)}`."
# Create logger
attack_log_manager = textattack.loggers.AttackLogManager(args.metrics)
# Get current time for file naming
timestamp = time.strftime("%Y-%m-%d-%H-%M")
# if '--log-to-txt' specified with arguments
if args.log_to_txt is not None:
if args.log_to_txt.lower().endswith(".txt"):
txt_file_path = args.log_to_txt
else:
txt_file_path = os.path.join(args.log_to_txt, f"{timestamp}-log.txt")
dir_path = os.path.dirname(txt_file_path)
dir_path = dir_path if dir_path else "."
if not os.path.exists(dir_path):
os.makedirs(os.path.dirname(txt_file_path))
color_method = "file"
attack_log_manager.add_output_file(txt_file_path, color_method)
# if '--log-to-csv' specified with arguments
if args.log_to_csv is not None:
if args.log_to_csv.lower().endswith(".csv"):
csv_file_path = args.log_to_csv
else:
csv_file_path = os.path.join(args.log_to_csv, f"{timestamp}-log.csv")
dir_path = os.path.dirname(csv_file_path)
dir_path = dir_path if dir_path else "."
if not os.path.exists(dir_path):
os.makedirs(dir_path)
color_method = (
None if args.csv_coloring_style == "plain" else args.csv_coloring_style
)
attack_log_manager.add_output_csv(csv_file_path, color_method)
# if '--log-summary-to-json' specified with arguments
if args.log_summary_to_json is not None:
if args.log_summary_to_json.lower().endswith(".json"):
summary_json_file_path = args.log_summary_to_json
else:
summary_json_file_path = os.path.join(
args.log_summary_to_json, f"{timestamp}-attack_summary_log.json"
)
dir_path = os.path.dirname(summary_json_file_path)
dir_path = dir_path if dir_path else "."
if not os.path.exists(dir_path):
os.makedirs(os.path.dirname(summary_json_file_path))
attack_log_manager.add_output_summary_json(summary_json_file_path)
# Visdom
if args.log_to_visdom is not None:
attack_log_manager.enable_visdom(**args.log_to_visdom)
# Weights & Biases
if args.log_to_wandb is not None:
attack_log_manager.enable_wandb(**args.log_to_wandb)
# Stdout
if not args.disable_stdout and not sys.stdout.isatty():
attack_log_manager.disable_color()
elif not args.disable_stdout:
attack_log_manager.enable_stdout()
return attack_log_manager
@dataclass
class _CommandLineAttackArgs:
"""Attack args for command line execution. This requires more arguments to
create ``Attack`` object as specified.
Args:
transformation (:obj:`str`, `optional`, defaults to :obj:`"word-swap-embedding"`):
Name of transformation to use.
constraints (:obj:`list[str]`, `optional`, defaults to :obj:`["repeat", "stopword"]`):
List of names of constraints to use.
goal_function (:obj:`str`, `optional`, defaults to :obj:`"untargeted-classification"`):
Name of goal function to use.
search_method (:obj:`str`, `optional`, defualts to :obj:`"greedy-word-wir"`):
Name of search method to use.
attack_recipe (:obj:`str`, `optional`, defaults to :obj:`None`):
Name of attack recipe to use.
.. note::
Setting this overrides any previous selection of transformation, constraints, goal function, and search method.
attack_from_file (:obj:`str`, `optional`, defaults to :obj:`None`):
Path of `.py` file from which to load attack from. Use `<path>^<variable_name>` to specifiy which variable to import from the file.
.. note::
If this is set, it overrides any previous selection of transformation, constraints, goal function, and search method
interactive (:obj:`bool`, `optional`, defaults to :obj:`False`):
If `True`, carry attack in interactive mode.
parallel (:obj:`bool`, `optional`, defaults to :obj:`False`):
If `True`, attack in parallel.
model_batch_size (:obj:`int`, `optional`, defaults to :obj:`32`):
The batch size for making queries to the victim model.
model_cache_size (:obj:`int`, `optional`, defaults to :obj:`2**18`):
The maximum number of items to keep in the model results cache at once.
constraint-cache-size (:obj:`int`, `optional`, defaults to :obj:`2**18`):
The maximum number of items to keep in the constraints cache at once.
"""
transformation: str = "word-swap-embedding"
constraints: list = field(default_factory=lambda: ["repeat", "stopword"])
goal_function: str = "untargeted-classification"
search_method: str = "greedy-word-wir"
attack_recipe: str = None
attack_from_file: str = None
interactive: bool = False
parallel: bool = False
model_batch_size: int = 32
model_cache_size: int = 2**18
constraint_cache_size: int = 2**18
@classmethod
def _add_parser_args(cls, parser):
"""Add listed args to command line parser."""
default_obj = cls()
transformation_names = set(BLACK_BOX_TRANSFORMATION_CLASS_NAMES.keys()) | set(
WHITE_BOX_TRANSFORMATION_CLASS_NAMES.keys()
)
parser.add_argument(
"--transformation",
type=str,
required=False,
default=default_obj.transformation,
help='The transformation to apply. Usage: "--transformation {transformation}:{arg_1}={value_1},{arg_3}={value_3}". Choices: '
+ str(transformation_names),
)
parser.add_argument(
"--constraints",
type=str,
required=False,
nargs="*",
default=default_obj.constraints,
help='Constraints to add to the attack. Usage: "--constraints {constraint}:{arg_1}={value_1},{arg_3}={value_3}". Choices: '
+ str(CONSTRAINT_CLASS_NAMES.keys()),
)
goal_function_choices = ", ".join(GOAL_FUNCTION_CLASS_NAMES.keys())
parser.add_argument(
"--goal-function",
"-g",
default=default_obj.goal_function,
help=f"The goal function to use. choices: {goal_function_choices}",
)
attack_group = parser.add_mutually_exclusive_group(required=False)
search_choices = ", ".join(SEARCH_METHOD_CLASS_NAMES.keys())
attack_group.add_argument(
"--search-method",
"--search",
"-s",
type=str,
required=False,
default=default_obj.search_method,
help=f"The search method to use. choices: {search_choices}",
)
attack_group.add_argument(
"--attack-recipe",
"--recipe",
"-r",
type=str,
required=False,
default=default_obj.attack_recipe,
help="full attack recipe (overrides provided goal function, transformation & constraints)",
choices=ATTACK_RECIPE_NAMES.keys(),
)
attack_group.add_argument(
"--attack-from-file",
type=str,
required=False,
default=default_obj.attack_from_file,
help="Path of `.py` file from which to load attack from. Use `<path>^<variable_name>` to specifiy which variable to import from the file.",
)
parser.add_argument(
"--interactive",
action="store_true",
default=default_obj.interactive,
help="Whether to run attacks interactively.",
)
parser.add_argument(
"--model-batch-size",
type=int,
default=default_obj.model_batch_size,
help="The batch size for making calls to the model.",
)
parser.add_argument(
"--model-cache-size",
type=int,
default=default_obj.model_cache_size,
help="The maximum number of items to keep in the model results cache at once.",
)
parser.add_argument(
"--constraint-cache-size",
type=int,
default=default_obj.constraint_cache_size,
help="The maximum number of items to keep in the constraints cache at once.",
)
return parser
@classmethod
def _create_transformation_from_args(cls, args, model_wrapper):
"""Create `Transformation` based on provided `args` and
`model_wrapper`."""
transformation_name = args.transformation
if ARGS_SPLIT_TOKEN in transformation_name:
transformation_name, params = transformation_name.split(ARGS_SPLIT_TOKEN)
if transformation_name in WHITE_BOX_TRANSFORMATION_CLASS_NAMES:
transformation = eval(
f"{WHITE_BOX_TRANSFORMATION_CLASS_NAMES[transformation_name]}(model_wrapper.model, {params})"
)
elif transformation_name in BLACK_BOX_TRANSFORMATION_CLASS_NAMES:
transformation = eval(
f"{BLACK_BOX_TRANSFORMATION_CLASS_NAMES[transformation_name]}({params})"
)
else:
raise ValueError(
f"Error: unsupported transformation {transformation_name}"
)
else:
if transformation_name in WHITE_BOX_TRANSFORMATION_CLASS_NAMES:
transformation = eval(
f"{WHITE_BOX_TRANSFORMATION_CLASS_NAMES[transformation_name]}(model_wrapper.model)"
)
elif transformation_name in BLACK_BOX_TRANSFORMATION_CLASS_NAMES:
transformation = eval(
f"{BLACK_BOX_TRANSFORMATION_CLASS_NAMES[transformation_name]}()"
)
else:
raise ValueError(
f"Error: unsupported transformation {transformation_name}"
)
return transformation
@classmethod
def _create_goal_function_from_args(cls, args, model_wrapper):
"""Create `GoalFunction` based on provided `args` and
`model_wrapper`."""
goal_function = args.goal_function
if ARGS_SPLIT_TOKEN in goal_function:
goal_function_name, params = goal_function.split(ARGS_SPLIT_TOKEN)
if goal_function_name not in GOAL_FUNCTION_CLASS_NAMES:
raise ValueError(
f"Error: unsupported goal_function {goal_function_name}"
)
goal_function = eval(
f"{GOAL_FUNCTION_CLASS_NAMES[goal_function_name]}(model_wrapper, {params})"
)
elif goal_function in GOAL_FUNCTION_CLASS_NAMES:
goal_function = eval(
f"{GOAL_FUNCTION_CLASS_NAMES[goal_function]}(model_wrapper)"
)
else:
raise ValueError(f"Error: unsupported goal_function {goal_function}")
if args.query_budget:
goal_function.query_budget = args.query_budget
goal_function.model_cache_size = args.model_cache_size
goal_function.batch_size = args.model_batch_size
return goal_function
@classmethod
def _create_constraints_from_args(cls, args):
"""Create list of `Constraints` based on provided `args`."""
if not args.constraints:
return []
_constraints = []
for constraint in args.constraints:
if ARGS_SPLIT_TOKEN in constraint:
constraint_name, params = constraint.split(ARGS_SPLIT_TOKEN)
if constraint_name not in CONSTRAINT_CLASS_NAMES:
raise ValueError(f"Error: unsupported constraint {constraint_name}")
_constraints.append(
eval(f"{CONSTRAINT_CLASS_NAMES[constraint_name]}({params})")
)
elif constraint in CONSTRAINT_CLASS_NAMES:
_constraints.append(eval(f"{CONSTRAINT_CLASS_NAMES[constraint]}()"))
else:
raise ValueError(f"Error: unsupported constraint {constraint}")
return _constraints
@classmethod
def _create_attack_from_args(cls, args, model_wrapper):
"""Given ``CommandLineArgs`` and ``ModelWrapper``, return specified
``Attack`` object."""
assert isinstance(
args, cls
), f"Expect args to be of type `{type(cls)}`, but got type `{type(args)}`."
if args.attack_recipe:
if ARGS_SPLIT_TOKEN in args.attack_recipe:
recipe_name, params = args.attack_recipe.split(ARGS_SPLIT_TOKEN)
if recipe_name not in ATTACK_RECIPE_NAMES:
raise ValueError(f"Error: unsupported recipe {recipe_name}")
recipe = eval(
f"{ATTACK_RECIPE_NAMES[recipe_name]}.build(model_wrapper, {params})"
)
elif args.attack_recipe in ATTACK_RECIPE_NAMES:
recipe = eval(
f"{ATTACK_RECIPE_NAMES[args.attack_recipe]}.build(model_wrapper)"
)
else:
raise ValueError(f"Invalid recipe {args.attack_recipe}")
if args.query_budget:
recipe.goal_function.query_budget = args.query_budget
recipe.goal_function.model_cache_size = args.model_cache_size
recipe.constraint_cache_size = args.constraint_cache_size
return recipe
elif args.attack_from_file:
if ARGS_SPLIT_TOKEN in args.attack_from_file:
attack_file, attack_name = args.attack_from_file.split(ARGS_SPLIT_TOKEN)
else:
attack_file, attack_name = args.attack_from_file, "attack"
attack_module = load_module_from_file(attack_file)
if not hasattr(attack_module, attack_name):
raise ValueError(
f"Loaded `{attack_file}` but could not find `{attack_name}`."
)
attack_func = getattr(attack_module, attack_name)
return attack_func(model_wrapper)
else:
goal_function = cls._create_goal_function_from_args(args, model_wrapper)
transformation = cls._create_transformation_from_args(args, model_wrapper)
constraints = cls._create_constraints_from_args(args)
if ARGS_SPLIT_TOKEN in args.search_method:
search_name, params = args.search_method.split(ARGS_SPLIT_TOKEN)
if search_name not in SEARCH_METHOD_CLASS_NAMES:
raise ValueError(f"Error: unsupported search {search_name}")
search_method = eval(
f"{SEARCH_METHOD_CLASS_NAMES[search_name]}({params})"
)
elif args.search_method in SEARCH_METHOD_CLASS_NAMES:
search_method = eval(
f"{SEARCH_METHOD_CLASS_NAMES[args.search_method]}()"
)
else:
raise ValueError(f"Error: unsupported attack {args.search_method}")
return Attack(
goal_function,
constraints,
transformation,
search_method,
constraint_cache_size=args.constraint_cache_size,
)
# This neat trick allows use to reorder the arguments to avoid TypeErrors commonly found when inheriting dataclass.
# https://stackoverflow.com/questions/51575931/class-inheritance-in-python-3-7-dataclasses
@dataclass
class CommandLineAttackArgs(AttackArgs, _CommandLineAttackArgs, DatasetArgs, ModelArgs):
@classmethod
def _add_parser_args(cls, parser):
"""Add listed args to command line parser."""
parser = ModelArgs._add_parser_args(parser)
parser = DatasetArgs._add_parser_args(parser)
parser = _CommandLineAttackArgs._add_parser_args(parser)
parser = AttackArgs._add_parser_args(parser)
return parser