diff --git a/.github/instructions/scenarios.instructions.md b/.github/instructions/scenarios.instructions.md index 0c491da2f..867375f8a 100644 --- a/.github/instructions/scenarios.instructions.md +++ b/.github/instructions/scenarios.instructions.md @@ -11,15 +11,15 @@ Scenarios orchestrate multi-attack security testing campaigns. Each scenario gro All scenarios inherit from `Scenario` (ABC) and must: 1. **Define `VERSION`** as a class constant (increment on breaking changes) -2. **Optionally declare `BASELINE_POLICY`** (defaults to `BaselinePolicy.Enabled` — a baseline `PromptSendingAttack` is prepended and callers can opt out per run via `initialize_async(include_baseline=False)`): - - `BaselinePolicy.Disabled` — baseline supported but off by default (e.g. `Jailbreak`, where templates dominate the run). - - `BaselinePolicy.Forbidden` — baseline is meaningless for this scenario's comparison axis (e.g. `AdversarialBenchmark`, which compares against gold-standard answers). Explicit `include_baseline=True` raises `ValueError`. +2. **Optionally declare `BASELINE_ATTACK_POLICY`** (defaults to `BaselineAttackPolicy.Enabled` — a baseline `PromptSendingAttack` is prepended and callers can opt out per run via `initialize_async(include_baseline=False)`): + - `BaselineAttackPolicy.Disabled` — baseline supported but off by default (e.g. `Jailbreak`, where templates dominate the run). + - `BaselineAttackPolicy.Forbidden` — baseline is meaningless for this scenario's comparison axis (e.g. `AdversarialBenchmark`, which compares against gold-standard answers). Explicit `include_baseline=True` raises `ValueError`. 3. **Implement three abstract methods:** ```python class MyScenario(Scenario): VERSION: int = 1 - BASELINE_POLICY: ClassVar[BaselinePolicy] = BaselinePolicy.Enabled + BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Enabled @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: diff --git a/doc/code/scenarios/0_scenarios.ipynb b/doc/code/scenarios/0_scenarios.ipynb index 82d604b93..b34254098 100644 --- a/doc/code/scenarios/0_scenarios.ipynb +++ b/doc/code/scenarios/0_scenarios.ipynb @@ -83,7 +83,7 @@ " - `max_retries`: Number of retry attempts on failure (default: 0)\n", " - `memory_labels`: Optional labels for tracking (optional)\n", " - `include_baseline`: Whether to prepend a baseline attack (defaults to the scenario type's\n", - " `BASELINE_POLICY`; most scenarios default it on, `Jailbreak` defaults it off)\n", + " `BASELINE_ATTACK_POLICY`; most scenarios default it on, `Jailbreak` defaults it off)\n", "\n", "### Example Structure\n", "\n", @@ -406,11 +406,11 @@ "Every scenario can optionally include a **baseline attack** — a `PromptSendingAttack` that sends\n", "each objective directly to the target without any converters or multi-turn techniques. This is\n", "controlled by the `include_baseline` parameter on `initialize_async`; when omitted, each\n", - "scenario falls back to its own `BASELINE_POLICY` class attribute (most scenarios default\n", + "scenario falls back to its own `BASELINE_ATTACK_POLICY` class attribute (most scenarios default\n", "it on; `Jailbreak` defaults it off). See\n", "[Common Scenario Parameters](./1_common_scenario_parameters.ipynb) for a worked example.\n", "\n", - "Custom scenarios should choose their `BASELINE_POLICY` based on whether an unmodified\n", + "Custom scenarios should choose their `BASELINE_ATTACK_POLICY` based on whether an unmodified\n", "prompt is a meaningful comparator for the scenario's strategies:\n", "\n", "- **`Enabled`** — the baseline is prepended by default and the caller can opt out. Use when an\n", diff --git a/doc/code/scenarios/0_scenarios.py b/doc/code/scenarios/0_scenarios.py index 9e86edf32..bf04a72f6 100644 --- a/doc/code/scenarios/0_scenarios.py +++ b/doc/code/scenarios/0_scenarios.py @@ -85,7 +85,7 @@ # - `max_retries`: Number of retry attempts on failure (default: 0) # - `memory_labels`: Optional labels for tracking (optional) # - `include_baseline`: Whether to prepend a baseline attack (defaults to the scenario type's -# `BASELINE_POLICY`; most scenarios default it on, `Jailbreak` defaults it off) +# `BASELINE_ATTACK_POLICY`; most scenarios default it on, `Jailbreak` defaults it off) # # ### Example Structure # @@ -176,11 +176,11 @@ def _build_display_group(self, *, technique_name: str, seed_group_name: str) -> # Every scenario can optionally include a **baseline attack** — a `PromptSendingAttack` that sends # each objective directly to the target without any converters or multi-turn techniques. This is # controlled by the `include_baseline` parameter on `initialize_async`; when omitted, each -# scenario falls back to its own `BASELINE_POLICY` class attribute (most scenarios default +# scenario falls back to its own `BASELINE_ATTACK_POLICY` class attribute (most scenarios default # it on; `Jailbreak` defaults it off). See # [Common Scenario Parameters](./1_common_scenario_parameters.ipynb) for a worked example. # -# Custom scenarios should choose their `BASELINE_POLICY` based on whether an unmodified +# Custom scenarios should choose their `BASELINE_ATTACK_POLICY` based on whether an unmodified # prompt is a meaningful comparator for the scenario's strategies: # # - **`Enabled`** — the baseline is prepended by default and the caller can opt out. Use when an diff --git a/pyrit/scenario/__init__.py b/pyrit/scenario/__init__.py index b66539543..02d725c58 100644 --- a/pyrit/scenario/__init__.py +++ b/pyrit/scenario/__init__.py @@ -21,7 +21,7 @@ AtomicAttack, AttackTechnique, AttackTechniqueFactory, - BaselinePolicy, + BaselineAttackPolicy, DatasetConfiguration, Scenario, ScenarioCompositeStrategy, @@ -51,7 +51,7 @@ "AtomicAttack", "AttackTechnique", "AttackTechniqueFactory", - "BaselinePolicy", + "BaselineAttackPolicy", "DatasetConfiguration", "Parameter", "Scenario", diff --git a/pyrit/scenario/core/__init__.py b/pyrit/scenario/core/__init__.py index 89c8935da..fede5a1da 100644 --- a/pyrit/scenario/core/__init__.py +++ b/pyrit/scenario/core/__init__.py @@ -8,7 +8,7 @@ from pyrit.scenario.core.attack_technique import AttackTechnique from pyrit.scenario.core.attack_technique_factory import AttackTechniqueFactory, ScorerOverridePolicy from pyrit.scenario.core.dataset_configuration import EXPLICIT_SEED_GROUPS_KEY, DatasetConfiguration -from pyrit.scenario.core.scenario import BaselinePolicy, Scenario +from pyrit.scenario.core.scenario import BaselineAttackPolicy, Scenario from pyrit.scenario.core.scenario_strategy import ScenarioCompositeStrategy, ScenarioStrategy from pyrit.scenario.core.scenario_target_defaults import get_default_adversarial_target, get_default_scorer_target from pyrit.scenario.core.scenario_techniques import ( @@ -20,7 +20,7 @@ "AtomicAttack", "AttackTechnique", "AttackTechniqueFactory", - "BaselinePolicy", + "BaselineAttackPolicy", "DatasetConfiguration", "EXPLICIT_SEED_GROUPS_KEY", "SCENARIO_TECHNIQUES", diff --git a/pyrit/scenario/core/scenario.py b/pyrit/scenario/core/scenario.py index b82f16af9..c93e18e28 100644 --- a/pyrit/scenario/core/scenario.py +++ b/pyrit/scenario/core/scenario.py @@ -55,13 +55,13 @@ logger = logging.getLogger(__name__) -class BaselinePolicy(Enum): +class BaselineAttackPolicy(Enum): """ Declares how a scenario type treats the default baseline atomic attack. The baseline is a plain ``PromptSendingAttack`` that sends each objective unmodified, used as a comparison point against the scenario's strategies. Each scenario class - declares its policy via ``Scenario.BASELINE_POLICY``; callers can still override + declares its policy via ``Scenario.BASELINE_ATTACK_POLICY``; callers can still override at runtime via ``initialize_async(include_baseline=...)`` for the ``Enabled`` and ``Disabled`` states. """ @@ -145,7 +145,7 @@ class Scenario(ABC): #: ``initialize_async`` and overridable per run via ``include_baseline`` for the #: ``Enabled`` and ``Disabled`` states; ``Forbidden`` is a hard constraint and a #: caller-supplied ``include_baseline=True`` raises ``ValueError``. - BASELINE_POLICY: ClassVar[BaselinePolicy] = BaselinePolicy.Enabled + BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Enabled @classmethod def _get_additional_scoring_questions(cls) -> Sequence[Path]: @@ -621,14 +621,14 @@ async def initialize_async( include_baseline (bool | None): Whether to prepend a baseline atomic attack that sends all objectives without modifications, allowing comparison between unmodified prompts and the scenario's strategies. If None (the default), the scenario type's - ``BASELINE_POLICY`` class attribute decides: ``Enabled`` includes it, + ``BASELINE_ATTACK_POLICY`` class attribute decides: ``Enabled`` includes it, ``Disabled`` omits it, and ``Forbidden`` always omits it (and rejects an - explicit ``True``). Passing ``True`` to a scenario whose ``BASELINE_POLICY`` + explicit ``True``). Passing ``True`` to a scenario whose ``BASELINE_ATTACK_POLICY`` is ``Forbidden`` raises ``ValueError``. Raises: ValueError: If no objective_target is provided, or if ``include_baseline=True`` is passed - to a scenario whose ``BASELINE_POLICY`` is ``Forbidden``. + to a scenario whose ``BASELINE_ATTACK_POLICY`` is ``Forbidden``. """ # Validate required parameters if objective_target is None: @@ -657,15 +657,15 @@ async def initialize_async( # scenario type never silently inherits a True default; explicit-True on a forbidden # type is a hard error rather than a silent ignore. For the Enabled / Disabled states, # a None runtime value defers to the policy. - if self.BASELINE_POLICY is BaselinePolicy.Forbidden: + if self.BASELINE_ATTACK_POLICY is BaselineAttackPolicy.Forbidden: if include_baseline is True: raise ValueError( f"{type(self).__name__} does not support a default baseline " - f"(BASELINE_POLICY = Forbidden); pass include_baseline=False or omit the argument." + f"(BASELINE_ATTACK_POLICY = Forbidden); pass include_baseline=False or omit the argument." ) include_baseline = False elif include_baseline is None: - include_baseline = self.BASELINE_POLICY is BaselinePolicy.Enabled + include_baseline = self.BASELINE_ATTACK_POLICY is BaselineAttackPolicy.Enabled self._include_baseline = include_baseline diff --git a/pyrit/scenario/scenarios/benchmark/adversarial.py b/pyrit/scenario/scenarios/benchmark/adversarial.py index 57024e79a..dfec12839 100644 --- a/pyrit/scenario/scenarios/benchmark/adversarial.py +++ b/pyrit/scenario/scenarios/benchmark/adversarial.py @@ -15,7 +15,7 @@ from pyrit.registry.tag_query import TagQuery from pyrit.scenario.core.atomic_attack import AtomicAttack from pyrit.scenario.core.dataset_configuration import DatasetConfiguration -from pyrit.scenario.core.scenario import BaselinePolicy, Scenario +from pyrit.scenario.core.scenario import BaselineAttackPolicy, Scenario from pyrit.scenario.core.scenario_techniques import SCENARIO_TECHNIQUES if TYPE_CHECKING: @@ -37,7 +37,7 @@ class AdversarialBenchmark(Scenario): #: AdversarialBenchmark compares attack-success rates across adversarial models; a baseline #: attack would be model-independent and contribute no signal to the comparison. - BASELINE_POLICY: ClassVar[BaselinePolicy] = BaselinePolicy.Forbidden + BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Forbidden @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: diff --git a/tests/unit/scenario/test_adversarial.py b/tests/unit/scenario/test_adversarial.py index 4f4ccf7fe..5914a40ba 100644 --- a/tests/unit/scenario/test_adversarial.py +++ b/tests/unit/scenario/test_adversarial.py @@ -22,7 +22,7 @@ ) from pyrit.prompt_target import PromptTarget, TargetCapabilities, TargetConfiguration from pyrit.registry.object_registries.attack_technique_registry import AttackTechniqueRegistry -from pyrit.scenario.core import AtomicAttack, BaselinePolicy +from pyrit.scenario.core import AtomicAttack, BaselineAttackPolicy from pyrit.scenario.core.dataset_configuration import DatasetConfiguration from pyrit.scenario.core.scenario_techniques import SCENARIO_TECHNIQUES from pyrit.scenario.scenarios.benchmark.adversarial import AdversarialBenchmark @@ -466,7 +466,7 @@ async def test_baseline_excluded(self, mock_objective_target, single_adversarial mock_objective_target=mock_objective_target, adversarial_models=single_adversarial_model, ) - assert type(scenario).BASELINE_POLICY is BaselinePolicy.Forbidden + assert type(scenario).BASELINE_ATTACK_POLICY is BaselineAttackPolicy.Forbidden assert not any(a.atomic_attack_name == "baseline" for a in scenario._atomic_attacks) async def test_baseline_explicit_true_raises(self, mock_objective_target, single_adversarial_model): diff --git a/tests/unit/scenario/test_baseline_deprecation.py b/tests/unit/scenario/test_baseline_deprecation.py index 5faf9a6d7..837c32b65 100644 --- a/tests/unit/scenario/test_baseline_deprecation.py +++ b/tests/unit/scenario/test_baseline_deprecation.py @@ -16,7 +16,7 @@ from pyrit.identifiers import ComponentIdentifier from pyrit.scenario import DatasetConfiguration -from pyrit.scenario.core import BaselinePolicy, Scenario, ScenarioStrategy +from pyrit.scenario.core import BaselineAttackPolicy, Scenario, ScenarioStrategy from pyrit.score import TrueFalseScorer _TEST_SCORER_ID = ComponentIdentifier(class_name="MockScorer", class_module="tests.unit.scenarios") @@ -34,7 +34,7 @@ def get_aggregate_tags(cls) -> set[str]: class _LegacyScenario(Scenario): """Minimal Scenario stand-in for exercising the deprecated baseline kwargs.""" - BASELINE_POLICY: ClassVar[BaselinePolicy] = BaselinePolicy.Enabled + BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Enabled def __init__(self, **kwargs): kwargs.setdefault("strategy_class", _LegacyStrategy) @@ -99,7 +99,7 @@ def test_base_kwarg_omitted_emits_no_warning(self): assert scenario._legacy_include_baseline is None async def test_legacy_value_drives_initialize_when_runtime_kwarg_omitted(self, mock_objective_target): - """Constructor-time False suppresses the baseline that BASELINE_POLICY=Enabled would add.""" + """Constructor-time False suppresses the baseline that BASELINE_ATTACK_POLICY=Enabled would add.""" with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) scenario = _LegacyScenario(include_default_baseline=False) diff --git a/tests/unit/scenario/test_jailbreak.py b/tests/unit/scenario/test_jailbreak.py index b30d38614..27fe5c663 100644 --- a/tests/unit/scenario/test_jailbreak.py +++ b/tests/unit/scenario/test_jailbreak.py @@ -16,7 +16,7 @@ from pyrit.identifiers import ComponentIdentifier from pyrit.models import SeedGroup, SeedObjective from pyrit.prompt_target import PromptTarget -from pyrit.scenario.core import BaselinePolicy +from pyrit.scenario.core import BaselineAttackPolicy from pyrit.scenario.scenarios.airt.jailbreak import Jailbreak, JailbreakStrategy from pyrit.score.true_false.true_false_inverter_scorer import TrueFalseInverterScorer @@ -203,14 +203,14 @@ async def test_init_raises_exception_when_no_datasets_available(self, mock_objec with pytest.raises(ValueError, match="DatasetConfiguration has no seed_groups"): await scenario.initialize_async(objective_target=mock_objective_target) - def test_class_inherits_default_baseline_policy(self): + def test_class_inherits_default_baseline_attack_policy(self): """Jailbreak inherits the base default (Enabled) — baseline included by default.""" - assert Jailbreak.BASELINE_POLICY is BaselinePolicy.Enabled + assert Jailbreak.BASELINE_ATTACK_POLICY is BaselineAttackPolicy.Enabled async def test_default_initialize_includes_baseline( self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups ): - """initialize_async without include_baseline honors BASELINE_POLICY=Enabled.""" + """initialize_async without include_baseline honors BASELINE_ATTACK_POLICY=Enabled.""" with patch.object(Jailbreak, "_resolve_seed_groups", return_value=mock_memory_seed_groups): scenario = Jailbreak(objective_scorer=mock_objective_scorer) await scenario.initialize_async(objective_target=mock_objective_target) diff --git a/tests/unit/scenario/test_leakage_scenario.py b/tests/unit/scenario/test_leakage_scenario.py index 9c6fa823a..c1b5659e7 100644 --- a/tests/unit/scenario/test_leakage_scenario.py +++ b/tests/unit/scenario/test_leakage_scenario.py @@ -14,7 +14,7 @@ from pyrit.prompt_target import PromptTarget from pyrit.scenario import DatasetConfiguration from pyrit.scenario.airt import Leakage, LeakageStrategy -from pyrit.scenario.core import BaselinePolicy +from pyrit.scenario.core import BaselineAttackPolicy from pyrit.score import TrueFalseCompositeScorer @@ -105,7 +105,7 @@ def test_default_scorer_uses_leakage_yaml(self): def test_init_supports_default_baseline(self): """Leakage opts into the parent's default baseline.""" - assert Leakage.BASELINE_POLICY is BaselinePolicy.Enabled + assert Leakage.BASELINE_ATTACK_POLICY is BaselineAttackPolicy.Enabled @pytest.mark.usefixtures(*FIXTURES) diff --git a/tests/unit/scenario/test_scenario.py b/tests/unit/scenario/test_scenario.py index e7042183d..b15447b01 100644 --- a/tests/unit/scenario/test_scenario.py +++ b/tests/unit/scenario/test_scenario.py @@ -13,7 +13,7 @@ from pyrit.memory import CentralMemory from pyrit.models import AttackOutcome, AttackResult from pyrit.scenario import DatasetConfiguration, ScenarioIdentifier, ScenarioResult -from pyrit.scenario.core import AtomicAttack, BaselinePolicy, Scenario, ScenarioStrategy +from pyrit.scenario.core import AtomicAttack, BaselineAttackPolicy, Scenario, ScenarioStrategy from pyrit.score import Scorer # Reusable test scorer identifier @@ -100,7 +100,7 @@ class ConcreteScenario(Scenario): # Tests using this fixture should default to no baseline; set the class policy to Forbidden # so we don't have to thread include_baseline=False through every initialize_async call. - BASELINE_POLICY: ClassVar[BaselinePolicy] = BaselinePolicy.Forbidden + BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Forbidden def __init__(self, atomic_attacks_to_return=None, **kwargs): # Add required strategy_class if not provided diff --git a/tests/unit/scenario/test_scenario_parameters.py b/tests/unit/scenario/test_scenario_parameters.py index 5289b61ee..9c8b6fe6c 100644 --- a/tests/unit/scenario/test_scenario_parameters.py +++ b/tests/unit/scenario/test_scenario_parameters.py @@ -11,7 +11,7 @@ from pyrit.common import Parameter from pyrit.identifiers import ComponentIdentifier from pyrit.scenario import DatasetConfiguration -from pyrit.scenario.core import BaselinePolicy, Scenario, ScenarioStrategy +from pyrit.scenario.core import BaselineAttackPolicy, Scenario, ScenarioStrategy from pyrit.score import Scorer _TEST_SCORER_ID = ComponentIdentifier(class_name="MockScorer", class_module="tests.unit.scenarios") @@ -35,7 +35,7 @@ def get_aggregate_tags(cls) -> set[str]: class _ParamTestScenario(Scenario): # No baseline in tests so atomic_attacks observations stay deterministic. - BASELINE_POLICY: ClassVar[BaselinePolicy] = BaselinePolicy.Forbidden + BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Forbidden @classmethod def get_strategy_class(cls): diff --git a/tests/unit/scenario/test_scenario_partial_results.py b/tests/unit/scenario/test_scenario_partial_results.py index a18625dc8..86496d68c 100644 --- a/tests/unit/scenario/test_scenario_partial_results.py +++ b/tests/unit/scenario/test_scenario_partial_results.py @@ -13,7 +13,7 @@ from pyrit.memory import CentralMemory from pyrit.models import AttackOutcome, AttackResult from pyrit.scenario import DatasetConfiguration, ScenarioResult -from pyrit.scenario.core import AtomicAttack, BaselinePolicy, Scenario, ScenarioStrategy +from pyrit.scenario.core import AtomicAttack, BaselineAttackPolicy, Scenario, ScenarioStrategy def _mock_scorer_id(name: str = "MockScorer") -> ComponentIdentifier: @@ -74,7 +74,7 @@ def filter_objectives(*, remaining_objectives): class ConcreteScenario(Scenario): """Concrete implementation of Scenario for testing.""" - BASELINE_POLICY: ClassVar[BaselinePolicy] = BaselinePolicy.Forbidden + BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Forbidden def __init__(self, *, atomic_attacks_to_return=None, objective_scorer=None, **kwargs): # Get strategy_class from kwargs or use default diff --git a/tests/unit/scenario/test_scenario_retry.py b/tests/unit/scenario/test_scenario_retry.py index 836503ff5..e193b58ee 100644 --- a/tests/unit/scenario/test_scenario_retry.py +++ b/tests/unit/scenario/test_scenario_retry.py @@ -13,7 +13,7 @@ from pyrit.memory import CentralMemory from pyrit.models import AttackOutcome, AttackResult from pyrit.scenario import DatasetConfiguration, ScenarioResult -from pyrit.scenario.core import AtomicAttack, BaselinePolicy, Scenario, ScenarioStrategy +from pyrit.scenario.core import AtomicAttack, BaselineAttackPolicy, Scenario, ScenarioStrategy # Test constants TEST_ATTACK_TYPE = "TestAttack" @@ -137,7 +137,7 @@ def create_mock_atomic_attack(name: str, objectives: list[str], run_async_mock: class ConcreteScenario(Scenario): """Concrete implementation of Scenario for testing.""" - BASELINE_POLICY: ClassVar[BaselinePolicy] = BaselinePolicy.Forbidden + BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Forbidden def __init__(self, atomic_attacks_to_return=None, objective_scorer=None, **kwargs): # Get strategy_class from kwargs or use default