Skip to content

Commit 6abab63

Browse files
rlundeen2rlundeen2
authored
MAINT: Adding End to End Tests for all scenarios (#1239)
1 parent cb79a41 commit 6abab63

5 files changed

Lines changed: 191 additions & 1 deletion

File tree

‎Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ PYMODULE:=pyrit
55
TESTS:=tests
66
UNIT_TESTS:=tests/unit
77
INTEGRATION_TESTS:=tests/integration
8+
END_TO_END_TESTS:=tests/end_to_end
89

910
all: pre-commit
1011

@@ -32,5 +33,8 @@ unit-test-cov-xml:
3233
integration-test:
3334
$(CMD) pytest $(INTEGRATION_TESTS) --cov=$(PYMODULE) $(INTEGRATION_TESTS) --cov-report xml --junitxml=junit/test-results.xml --doctest-modules
3435

36+
end-to-end-test:
37+
$(CMD) pytest $(END_TO_END_TESTS) -v --junitxml=junit/test-results.xml
38+
3539
#clean:
3640
# git clean -Xdf # Delete all files in .gitignore

‎build_scripts/env_local_integration_test‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,23 @@
55
# This will override the .env value
66
OPENAI_CHAT_ENDPOINT=${AZURE_OPENAI_INTEGRATION_TEST_ENDPOINT}
77
OPENAI_CHAT_KEY=${AZURE_OPENAI_INTEGRATION_TEST_KEY}
8-
OPENAI_CHAT_MODEL="gpt-4o"
8+
OPENAI_CHAT_MODEL=${AZURE_OPENAI_INTEGRATION_TEST_MODEL}
99

1010
OPENAI_IMAGE_ENDPOINT=${OPENAI_IMAGE_ENDPOINT2}
1111
OPENAI_IMAGE_API_KEY=${OPENAI_IMAGE_API_KEY2}
12+
OPENAI_IMAGE_MODEL=${OPENAI_IMAGE_MODEL2}
1213

1314
OPENAI_TTS_ENDPOINT=${OPENAI_TTS_ENDPOINT2}
1415
OPENAI_TTS_KEY=${OPENAI_TTS_KEY2}
1516

1617
AZURE_SQL_DB_CONNECTION_STRING=${AZURE_SQL_DB_CONNECTION_STRING_TEST}
1718
AZURE_STORAGE_ACCOUNT_DB_DATA_CONTAINER_URL=${AZURE_STORAGE_ACCOUNT_DB_DATA_CONTAINER_URL_TEST}
1819

20+
# E2E scenario test variables (used by openai_objective_target initializer)
21+
DEFAULT_OPENAI_FRONTEND_ENDPOINT=${AZURE_OPENAI_INTEGRATION_TEST_ENDPOINT}
22+
DEFAULT_OPENAI_FRONTEND_KEY=${AZURE_OPENAI_INTEGRATION_TEST_KEY}
23+
DEFAULT_OPENAI_FRONTEND_MODEL=${AZURE_OPENAI_INTEGRATION_TEST_MODEL}
24+
1925
GLOBAL_MEMORY_LABELS={"username": "integration-test", "op_name": "integration-test"}
2026

2127
##############

‎end-to-end-tests.yml‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
# Runs end-to-end scenario tests using pyrit_scan CLI
2+
3+
trigger: none # Disable automatic CI triggers
4+
5+
schedules:
6+
- cron: "0 7 * * *" # 7 AM UTC = 11 PM PST (UTC-8) / Midnight PDT (UTC-7)
7+
displayName: Nightly E2E Tests at 11 PM PST
8+
branches:
9+
include:
10+
- main
11+
always: true # Run even if there are no code changes
12+
13+
jobs:
14+
- job: EndToEndTests
15+
displayName: "Run end-to-end scenario tests"
16+
timeoutInMinutes: 360 # Allows the job to run up to 6 hours
17+
pool:
18+
vmImage: ubuntu-latest
19+
steps:
20+
- checkout: self
21+
fetchDepth: 1
22+
- task: UsePythonVersion@0
23+
inputs:
24+
versionSpec: '3.12'
25+
addToPath: true
26+
- task: AzureKeyVault@2
27+
displayName: Azure Key Vault - retrieve .env file secret
28+
inputs:
29+
azureSubscription: 'integration-test-service-connection'
30+
KeyVaultName: 'pyrit-environment'
31+
SecretsFilter: 'env-global'
32+
RunAsPreJob: false
33+
- bash: |
34+
python -c "
35+
import os;
36+
secret = os.environ.get('PYRIT_TEST_SECRET');
37+
if not secret:
38+
raise ValueError('PYRIT_TEST_SECRET is not set');
39+
with open('.env', 'w') as file:
40+
file.write(secret)"
41+
env:
42+
PYRIT_TEST_SECRET: $(env-global)
43+
name: create_env_file
44+
- bash: |
45+
cp build_scripts/env_local_integration_test .env.local
46+
displayName: "Create .env.local from example"
47+
- bash: pip install --upgrade setuptools pip packaging
48+
name: upgrade_pip_and_setuptools_before_installing_PyRIT
49+
- bash: sudo apt-get install python3-tk
50+
name: install_tkinter
51+
- bash: |
52+
set -e
53+
# Detect Ubuntu version
54+
UBUNTU_VERSION=$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)
55+
SUPPORTED_VERSIONS="18.04 20.04 22.04 24.04 24.10"
56+
57+
if ! [[ "$SUPPORTED_VERSIONS" == *"$UBUNTU_VERSION"* ]]; then
58+
echo "Ubuntu $UBUNTU_VERSION is not currently supported."
59+
exit 1
60+
fi
61+
62+
# Download the package to configure the Microsoft repo
63+
curl -sSL -O https://packages.microsoft.com/config/ubuntu/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)/packages-microsoft-prod.deb
64+
# Install the package
65+
sudo dpkg -i packages-microsoft-prod.deb
66+
# Delete the file
67+
rm packages-microsoft-prod.deb
68+
69+
# Install the driver
70+
sudo apt-get update
71+
sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18
72+
73+
echo "Microsoft ODBC Driver 18 installed successfully."
74+
displayName: 'Install ODBC Driver 18 for SQL Server'
75+
- bash: pip install .[dev,all] -v --no-cache-dir
76+
name: install_PyRIT
77+
- bash: df -all -h
78+
name: disk_space_check
79+
# This step ensures that end-to-end tests are run outside of the PyRIT repository to test that .env files are accessed correctly.
80+
- bash: |
81+
PyRIT_DIR=$(pwd)
82+
NEW_DIR="e2e_test_directory"
83+
cd ..
84+
mkdir -p $NEW_DIR/tests
85+
cp $PyRIT_DIR/.env $NEW_DIR
86+
cp $PyRIT_DIR/.env.local $NEW_DIR
87+
cp -r $PyRIT_DIR/doc $NEW_DIR
88+
cp -r $PyRIT_DIR/assets $NEW_DIR
89+
cp -r $PyRIT_DIR/tests/end_to_end $NEW_DIR/tests
90+
cd $NEW_DIR
91+
displayName: "Create and switch to E2E test directory"
92+
- task: AzureCLI@2
93+
displayName: "Authenticate with service principal, cache access tokens, and run E2E tests"
94+
inputs:
95+
azureSubscription: 'integration-test-service-connection'
96+
scriptType: 'bash'
97+
scriptLocation: 'inlineScript'
98+
inlineScript: |
99+
# Prefetch token for Cognitive Services before ID token expires (60-90 minute validity)
100+
az account get-access-token --scope https://cognitiveservices.azure.com/.default --output none
101+
echo "Cognitive Services access token cached successfully."
102+
103+
# Prefetch token for Azure ML / Foundry model endpoints
104+
az account get-access-token --scope https://ml.azure.com/.default --output none
105+
echo "Azure ML/Foundry access token cached successfully."
106+
107+
# Prefetch token for Azure SQL Database
108+
az account get-access-token --scope https://database.windows.net/.default --output none
109+
echo "Azure SQL Database access token cached successfully."
110+
111+
# Run end-to-end tests
112+
make end-to-end-test
113+
- bash: |
114+
rm -f .env
115+
name: clean_up_env_files
116+
condition: always()
117+
- task: PublishTestResults@2
118+
condition: always()
119+
inputs:
120+
testResultsFormat: 'JUnit'
121+
testResultsFiles: 'junit/test-results.xml'

‎tests/end_to_end/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Copyright (c) Microsoft Corporation.
2+
# Licensed under the MIT license.
3+
4+
"""End-to-end tests for PyRIT scenarios."""

‎tests/end_to_end/test_scenarios.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# Copyright (c) Microsoft Corporation.
2+
# Licensed under the MIT license.
3+
4+
"""
5+
End-to-end tests for PyRIT scenarios using pyrit_scan CLI.
6+
7+
These tests dynamically discover all available scenarios and run each one
8+
using the pyrit_scan command with standard initializers.
9+
"""
10+
11+
import pytest
12+
13+
from pyrit.cli.pyrit_scan import main as pyrit_scan_main
14+
from pyrit.cli.scenario_registry import ScenarioRegistry
15+
16+
17+
def get_all_scenarios():
18+
"""
19+
Dynamically discover all available scenarios from the scenario registry.
20+
21+
Returns:
22+
List[str]: Sorted list of scenario names.
23+
"""
24+
registry = ScenarioRegistry()
25+
return registry.get_scenario_names()
26+
27+
28+
@pytest.mark.timeout(7200) # 2 hour timeout per scenario
29+
@pytest.mark.parametrize("scenario_name", get_all_scenarios())
30+
def test_scenario_with_pyrit_scan(scenario_name):
31+
"""
32+
Test each scenario runs successfully using pyrit_scan with standard initializers.
33+
34+
Args:
35+
scenario_name: Name of the scenario to test (dynamically discovered).
36+
"""
37+
try:
38+
result = pyrit_scan_main(
39+
[
40+
scenario_name,
41+
"--initializers",
42+
"openai_objective_target",
43+
"load_default_datasets",
44+
"--database",
45+
"InMemory",
46+
"--log-level",
47+
"WARNING",
48+
]
49+
)
50+
51+
assert result == 0, f"Scenario '{scenario_name}' failed with exit code {result}"
52+
53+
except Exception as e:
54+
# Re-raise with scenario context while preserving full traceback
55+
raise AssertionError(f"Scenario '{scenario_name}' raised an exception") from e

0 commit comments

Comments
 (0)