diff --git a/Makefile b/Makefile index ddfefb5977..3ccc56f1e4 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ PYMODULE:=pyrit TESTS:=tests UNIT_TESTS:=tests/unit INTEGRATION_TESTS:=tests/integration +END_TO_END_TESTS:=tests/end_to_end all: pre-commit @@ -32,5 +33,8 @@ unit-test-cov-xml: integration-test: $(CMD) pytest $(INTEGRATION_TESTS) --cov=$(PYMODULE) $(INTEGRATION_TESTS) --cov-report xml --junitxml=junit/test-results.xml --doctest-modules +end-to-end-test: + $(CMD) pytest $(END_TO_END_TESTS) -v --junitxml=junit/test-results.xml + #clean: # git clean -Xdf # Delete all files in .gitignore diff --git a/build_scripts/env_local_integration_test b/build_scripts/env_local_integration_test index eeba793f78..8d709f4aad 100644 --- a/build_scripts/env_local_integration_test +++ b/build_scripts/env_local_integration_test @@ -5,10 +5,11 @@ # This will override the .env value OPENAI_CHAT_ENDPOINT=${AZURE_OPENAI_INTEGRATION_TEST_ENDPOINT} OPENAI_CHAT_KEY=${AZURE_OPENAI_INTEGRATION_TEST_KEY} -OPENAI_CHAT_MODEL="gpt-4o" +OPENAI_CHAT_MODEL=${AZURE_OPENAI_INTEGRATION_TEST_MODEL} OPENAI_IMAGE_ENDPOINT=${OPENAI_IMAGE_ENDPOINT2} OPENAI_IMAGE_API_KEY=${OPENAI_IMAGE_API_KEY2} +OPENAI_IMAGE_MODEL=${OPENAI_IMAGE_MODEL2} OPENAI_TTS_ENDPOINT=${OPENAI_TTS_ENDPOINT2} OPENAI_TTS_KEY=${OPENAI_TTS_KEY2} @@ -16,6 +17,11 @@ OPENAI_TTS_KEY=${OPENAI_TTS_KEY2} AZURE_SQL_DB_CONNECTION_STRING=${AZURE_SQL_DB_CONNECTION_STRING_TEST} AZURE_STORAGE_ACCOUNT_DB_DATA_CONTAINER_URL=${AZURE_STORAGE_ACCOUNT_DB_DATA_CONTAINER_URL_TEST} +# E2E scenario test variables (used by openai_objective_target initializer) +DEFAULT_OPENAI_FRONTEND_ENDPOINT=${AZURE_OPENAI_INTEGRATION_TEST_ENDPOINT} +DEFAULT_OPENAI_FRONTEND_KEY=${AZURE_OPENAI_INTEGRATION_TEST_KEY} +DEFAULT_OPENAI_FRONTEND_MODEL=${AZURE_OPENAI_INTEGRATION_TEST_MODEL} + GLOBAL_MEMORY_LABELS={"username": "integration-test", "op_name": "integration-test"} ############## diff --git a/end-to-end-tests.yml b/end-to-end-tests.yml new file mode 100644 index 0000000000..48fe9d7573 --- /dev/null +++ b/end-to-end-tests.yml @@ -0,0 +1,121 @@ +# Runs end-to-end scenario tests using pyrit_scan CLI + +trigger: none # Disable automatic CI triggers + +schedules: +- cron: "0 7 * * *" # 7 AM UTC = 11 PM PST (UTC-8) / Midnight PDT (UTC-7) + displayName: Nightly E2E Tests at 11 PM PST + branches: + include: + - main + always: true # Run even if there are no code changes + +jobs: +- job: EndToEndTests + displayName: "Run end-to-end scenario tests" + timeoutInMinutes: 360 # Allows the job to run up to 6 hours + pool: + vmImage: ubuntu-latest + steps: + - checkout: self + fetchDepth: 1 + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.12' + addToPath: true + - task: AzureKeyVault@2 + displayName: Azure Key Vault - retrieve .env file secret + inputs: + azureSubscription: 'integration-test-service-connection' + KeyVaultName: 'pyrit-environment' + SecretsFilter: 'env-global' + RunAsPreJob: false + - bash: | + python -c " + import os; + secret = os.environ.get('PYRIT_TEST_SECRET'); + if not secret: + raise ValueError('PYRIT_TEST_SECRET is not set'); + with open('.env', 'w') as file: + file.write(secret)" + env: + PYRIT_TEST_SECRET: $(env-global) + name: create_env_file + - bash: | + cp build_scripts/env_local_integration_test .env.local + displayName: "Create .env.local from example" + - bash: pip install --upgrade setuptools pip packaging + name: upgrade_pip_and_setuptools_before_installing_PyRIT + - bash: sudo apt-get install python3-tk + name: install_tkinter + - bash: | + set -e + # Detect Ubuntu version + UBUNTU_VERSION=$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2) + SUPPORTED_VERSIONS="18.04 20.04 22.04 24.04 24.10" + + if ! [[ "$SUPPORTED_VERSIONS" == *"$UBUNTU_VERSION"* ]]; then + echo "Ubuntu $UBUNTU_VERSION is not currently supported." + exit 1 + fi + + # Download the package to configure the Microsoft repo + curl -sSL -O https://packages.microsoft.com/config/ubuntu/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)/packages-microsoft-prod.deb + # Install the package + sudo dpkg -i packages-microsoft-prod.deb + # Delete the file + rm packages-microsoft-prod.deb + + # Install the driver + sudo apt-get update + sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 + + echo "Microsoft ODBC Driver 18 installed successfully." + displayName: 'Install ODBC Driver 18 for SQL Server' + - bash: pip install .[dev,all] -v --no-cache-dir + name: install_PyRIT + - bash: df -all -h + name: disk_space_check +# This step ensures that end-to-end tests are run outside of the PyRIT repository to test that .env files are accessed correctly. + - bash: | + PyRIT_DIR=$(pwd) + NEW_DIR="e2e_test_directory" + cd .. + mkdir -p $NEW_DIR/tests + cp $PyRIT_DIR/.env $NEW_DIR + cp $PyRIT_DIR/.env.local $NEW_DIR + cp -r $PyRIT_DIR/doc $NEW_DIR + cp -r $PyRIT_DIR/assets $NEW_DIR + cp -r $PyRIT_DIR/tests/end_to_end $NEW_DIR/tests + cd $NEW_DIR + displayName: "Create and switch to E2E test directory" + - task: AzureCLI@2 + displayName: "Authenticate with service principal, cache access tokens, and run E2E tests" + inputs: + azureSubscription: 'integration-test-service-connection' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + # Prefetch token for Cognitive Services before ID token expires (60-90 minute validity) + az account get-access-token --scope https://cognitiveservices.azure.com/.default --output none + echo "Cognitive Services access token cached successfully." + + # Prefetch token for Azure ML / Foundry model endpoints + az account get-access-token --scope https://ml.azure.com/.default --output none + echo "Azure ML/Foundry access token cached successfully." + + # Prefetch token for Azure SQL Database + az account get-access-token --scope https://database.windows.net/.default --output none + echo "Azure SQL Database access token cached successfully." + + # Run end-to-end tests + make end-to-end-test + - bash: | + rm -f .env + name: clean_up_env_files + condition: always() + - task: PublishTestResults@2 + condition: always() + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: 'junit/test-results.xml' diff --git a/tests/end_to_end/__init__.py b/tests/end_to_end/__init__.py new file mode 100644 index 0000000000..d5397e007a --- /dev/null +++ b/tests/end_to_end/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""End-to-end tests for PyRIT scenarios.""" diff --git a/tests/end_to_end/test_scenarios.py b/tests/end_to_end/test_scenarios.py new file mode 100644 index 0000000000..675fc75396 --- /dev/null +++ b/tests/end_to_end/test_scenarios.py @@ -0,0 +1,55 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +End-to-end tests for PyRIT scenarios using pyrit_scan CLI. + +These tests dynamically discover all available scenarios and run each one +using the pyrit_scan command with standard initializers. +""" + +import pytest + +from pyrit.cli.pyrit_scan import main as pyrit_scan_main +from pyrit.cli.scenario_registry import ScenarioRegistry + + +def get_all_scenarios(): + """ + Dynamically discover all available scenarios from the scenario registry. + + Returns: + List[str]: Sorted list of scenario names. + """ + registry = ScenarioRegistry() + return registry.get_scenario_names() + + +@pytest.mark.timeout(7200) # 2 hour timeout per scenario +@pytest.mark.parametrize("scenario_name", get_all_scenarios()) +def test_scenario_with_pyrit_scan(scenario_name): + """ + Test each scenario runs successfully using pyrit_scan with standard initializers. + + Args: + scenario_name: Name of the scenario to test (dynamically discovered). + """ + try: + result = pyrit_scan_main( + [ + scenario_name, + "--initializers", + "openai_objective_target", + "load_default_datasets", + "--database", + "InMemory", + "--log-level", + "WARNING", + ] + ) + + assert result == 0, f"Scenario '{scenario_name}' failed with exit code {result}" + + except Exception as e: + # Re-raise with scenario context while preserving full traceback + raise AssertionError(f"Scenario '{scenario_name}' raised an exception") from e