MAINT: Adding End to End Tests for all scenarios

Makefile

@@ -5,6 +5,7 @@ PYMODULE:=pyrit
 TESTS:=tests
 UNIT_TESTS:=tests/unit
 INTEGRATION_TESTS:=tests/integration
+END_TO_END_TESTS:=tests/end_to_end
 
 all: pre-commit
 
@@ -32,5 +33,8 @@ unit-test-cov-xml:
 integration-test:
 	$(CMD) pytest $(INTEGRATION_TESTS) --cov=$(PYMODULE) $(INTEGRATION_TESTS) --cov-report xml --junitxml=junit/test-results.xml --doctest-modules
 
+end-to-end-test:
+	$(CMD) pytest $(END_TO_END_TESTS) -v --junitxml=junit/test-results.xml
+
 #clean:
 #	git clean -Xdf # Delete all files in .gitignore

build_scripts/env_local_integration_test

@@ -5,17 +5,23 @@
 # This will override the .env value
 OPENAI_CHAT_ENDPOINT=${AZURE_OPENAI_INTEGRATION_TEST_ENDPOINT}
 OPENAI_CHAT_KEY=${AZURE_OPENAI_INTEGRATION_TEST_KEY}
-OPENAI_CHAT_MODEL="gpt-4o"
+OPENAI_CHAT_MODEL=${AZURE_OPENAI_INTEGRATION_TEST_MODEL}
 
 OPENAI_IMAGE_ENDPOINT=${OPENAI_IMAGE_ENDPOINT2}
 OPENAI_IMAGE_API_KEY=${OPENAI_IMAGE_API_KEY2}
+OPENAI_IMAGE_MODEL=${OPENAI_IMAGE_MODEL2}
 
 OPENAI_TTS_ENDPOINT=${OPENAI_TTS_ENDPOINT2}
 OPENAI_TTS_KEY=${OPENAI_TTS_KEY2}
 
 AZURE_SQL_DB_CONNECTION_STRING=${AZURE_SQL_DB_CONNECTION_STRING_TEST}
 AZURE_STORAGE_ACCOUNT_DB_DATA_CONTAINER_URL=${AZURE_STORAGE_ACCOUNT_DB_DATA_CONTAINER_URL_TEST}
 
+# E2E scenario test variables (used by openai_objective_target initializer)
+DEFAULT_OPENAI_FRONTEND_ENDPOINT=${AZURE_OPENAI_INTEGRATION_TEST_ENDPOINT}
+DEFAULT_OPENAI_FRONTEND_KEY=${AZURE_OPENAI_INTEGRATION_TEST_KEY}
+DEFAULT_OPENAI_FRONTEND_MODEL=${AZURE_OPENAI_INTEGRATION_TEST_MODEL}
+
 GLOBAL_MEMORY_LABELS={"username": "integration-test", "op_name": "integration-test"}
 
 ##############

end-to-end-tests.yml

@@ -0,0 +1,121 @@
+# Runs end-to-end scenario tests using pyrit_scan CLI
+
+trigger: none  # Disable automatic CI triggers
+
+schedules:
+- cron: "0 7 * * *"  # 7 AM UTC = 11 PM PST (UTC-8) / Midnight PDT (UTC-7)
+  displayName: Nightly E2E Tests at 11 PM PST
+  branches:
+    include:
+    - main
+  always: true  # Run even if there are no code changes
+
+jobs:
+- job: EndToEndTests
+  displayName: "Run end-to-end scenario tests"
+  timeoutInMinutes: 360   # Allows the job to run up to 6 hours
+  pool:
+    vmImage: ubuntu-latest
+  steps:
+  - checkout: self
+    fetchDepth: 1
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '3.12'
+      addToPath: true
+  - task: AzureKeyVault@2
+    displayName: Azure Key Vault - retrieve .env file secret
+    inputs:
+      azureSubscription: 'integration-test-service-connection'
+      KeyVaultName: 'pyrit-environment'
+      SecretsFilter: 'env-global'
+      RunAsPreJob: false
+  - bash: |
+      python -c "
+      import os;
+      secret = os.environ.get('PYRIT_TEST_SECRET');
+      if not secret:
+        raise ValueError('PYRIT_TEST_SECRET is not set');
+      with open('.env', 'w') as file:
+        file.write(secret)"
+    env:
+      PYRIT_TEST_SECRET: $(env-global)
+    name: create_env_file
+  - bash: |
+      cp build_scripts/env_local_integration_test .env.local
+    displayName: "Create .env.local from example"
+  - bash: pip install --upgrade setuptools pip packaging
+    name: upgrade_pip_and_setuptools_before_installing_PyRIT
+  - bash: sudo apt-get install python3-tk
+    name: install_tkinter
+  - bash: |
+      set -e
+      # Detect Ubuntu version
+      UBUNTU_VERSION=$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)
+      SUPPORTED_VERSIONS="18.04 20.04 22.04 24.04 24.10"
+
+      if ! [[ "$SUPPORTED_VERSIONS" == *"$UBUNTU_VERSION"* ]]; then
+        echo "Ubuntu $UBUNTU_VERSION is not currently supported."
+        exit 1
+      fi
+
+      # Download the package to configure the Microsoft repo
+      curl -sSL -O https://packages.microsoft.com/config/ubuntu/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)/packages-microsoft-prod.deb
+      # Install the package
+      sudo dpkg -i packages-microsoft-prod.deb
+      # Delete the file
+      rm packages-microsoft-prod.deb
+
+      # Install the driver
+      sudo apt-get update
+      sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18
+
+      echo "Microsoft ODBC Driver 18 installed successfully."
+    displayName: 'Install ODBC Driver 18 for SQL Server'
+  - bash: pip install .[dev,all] -v --no-cache-dir
+    name: install_PyRIT
+  - bash: df -all -h
+    name: disk_space_check
+# This step ensures that end-to-end tests are run outside of the PyRIT repository to test that .env files are accessed correctly.
+  - bash: |
+      PyRIT_DIR=$(pwd)
+      NEW_DIR="e2e_test_directory"
+      cd ..
+      mkdir -p $NEW_DIR/tests
+      cp $PyRIT_DIR/.env $NEW_DIR
+      cp $PyRIT_DIR/.env.local $NEW_DIR
+      cp -r $PyRIT_DIR/doc $NEW_DIR
+      cp -r $PyRIT_DIR/assets $NEW_DIR
+      cp -r $PyRIT_DIR/tests/end_to_end $NEW_DIR/tests
+      cd $NEW_DIR
+    displayName: "Create and switch to E2E test directory"
+  - task: AzureCLI@2
+    displayName: "Authenticate with service principal, cache access tokens, and run E2E tests"
+    inputs:
+      azureSubscription: 'integration-test-service-connection'
+      scriptType: 'bash'
+      scriptLocation: 'inlineScript'
+      inlineScript: |
+        # Prefetch token for Cognitive Services before ID token expires (60-90 minute validity)
+        az account get-access-token --scope https://cognitiveservices.azure.com/.default --output none
+        echo "Cognitive Services access token cached successfully."
+
+        # Prefetch token for Azure ML / Foundry model endpoints
+        az account get-access-token --scope https://ml.azure.com/.default --output none
+        echo "Azure ML/Foundry access token cached successfully."
+
+        # Prefetch token for Azure SQL Database
+        az account get-access-token --scope https://database.windows.net/.default --output none
+        echo "Azure SQL Database access token cached successfully."
+
+        # Run end-to-end tests
+        make end-to-end-test
+  - bash: |
+      rm -f .env
+    name: clean_up_env_files
+    condition: always()
+  - task: PublishTestResults@2
+    condition: always()
+    inputs:
+      testResultsFormat: 'JUnit'
+      testResultsFiles: 'junit/test-results.xml'

tests/end_to_end/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""End-to-end tests for PyRIT scenarios."""

tests/end_to_end/test_scenarios.py

@@ -0,0 +1,55 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""
+End-to-end tests for PyRIT scenarios using pyrit_scan CLI.
+
+These tests dynamically discover all available scenarios and run each one
+using the pyrit_scan command with standard initializers.
+"""
+
+import pytest
+
+from pyrit.cli.pyrit_scan import main as pyrit_scan_main
+from pyrit.cli.scenario_registry import ScenarioRegistry
+
+
+def get_all_scenarios():
+    """
+    Dynamically discover all available scenarios from the scenario registry.
+
+    Returns:
+        List[str]: Sorted list of scenario names.
+    """
+    registry = ScenarioRegistry()
+    return registry.get_scenario_names()
+
+
+@pytest.mark.timeout(7200)  # 2 hour timeout per scenario
+@pytest.mark.parametrize("scenario_name", get_all_scenarios())
+def test_scenario_with_pyrit_scan(scenario_name):
+    """
+    Test each scenario runs successfully using pyrit_scan with standard initializers.
+
+    Args:
+        scenario_name: Name of the scenario to test (dynamically discovered).
+    """
+    try:
+        result = pyrit_scan_main(
+            [
+                scenario_name,
+                "--initializers",
+                "openai_objective_target",
+                "load_default_datasets",
+                "--database",
+                "InMemory",
+                "--log-level",
+                "WARNING",
+            ]
+        )
+
+        assert result == 0, f"Scenario '{scenario_name}' failed with exit code {result}"
+
+    except Exception as e:
+        # Re-raise with scenario context while preserving full traceback
+        raise AssertionError(f"Scenario '{scenario_name}' raised an exception") from e