From 22dd740f3e319633d4ff27ccd002401186474dd3 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 19 Jun 2026 16:47:14 -0700 Subject: [PATCH 01/21] Add sparse checkouts to signing jobs, skip checkout for consolidation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../templates/stages/archetype-sdk-client.yml | 176 ++++++++++++++++-- 1 file changed, 162 insertions(+), 14 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 109f9e6bd012..5851921a04b1 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -147,17 +147,165 @@ extends: - template: /eng/pipelines/templates/variables/globals.yml - template: /eng/pipelines/templates/variables/image.yml - - template: archetype-python-release.yml - parameters: - DependsOn: "Build" - ServiceDirectory: ${{ parameters.ServiceDirectory }} - Artifacts: ${{ parameters.Artifacts }} - ${{ if eq(parameters.ServiceDirectory, 'template') }}: - TestPipeline: true - ArtifactName: packages_extended - DocArtifact: documentation - TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }} - TargetDocRepoName: ${{ parameters.TargetDocRepoName }} - DevFeedName: ${{ parameters.DevFeedName }} - PublicFeed: ${{ parameters.PublicFeed }} - PublicPublishEnvironment: ${{ parameters.PublicPublishEnvironment }} + - ${{ if eq(parameters.ServiceDirectory, 'storage') }}: + - stage: Sign_Binaries + displayName: Sign Extension Wheels + dependsOn: Build + condition: succeeded() + jobs: + - job: Sign_macOS + displayName: Sign macOS Wheels + pool: + name: $(WINDOWSPOOL) + image: $(WINDOWSVMIMAGE) + os: windows + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - task: UsePythonVersion@0 + displayName: "Use Python $(PythonVersion)" + inputs: + versionSpec: $(PythonVersion) + + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: packages_mac + targetPath: $(Build.ArtifactStagingDirectory)/packages_mac + + - pwsh: | + python eng/scripts/wheel_signing/extract_sign_inputs.py ` + --platform mac ` + --wheels-dir "$(Build.ArtifactStagingDirectory)/packages_mac" ` + --work-dir "$(Build.ArtifactStagingDirectory)/mac-sign-work" ` + --sign-input-zip "$(Build.ArtifactStagingDirectory)/mac-sign-input.zip" + displayName: Extract mac wheel binaries + + - template: pipelines/steps/azd-cli-mac-signing.yml@azure-sdk-build-tools + parameters: + MacPath: "$(Build.ArtifactStagingDirectory)" + MacPattern: "mac-sign-input.zip" + Notarize: false + + - pwsh: | + python eng/scripts/wheel_signing/repackage_signed_wheels.py ` + --platform mac ` + --work-dir "$(Build.ArtifactStagingDirectory)/mac-sign-work" ` + --signed-input-zip "$(Build.ArtifactStagingDirectory)/mac-sign-input.zip" ` + --output-wheels-dir "$(Build.ArtifactStagingDirectory)/mac-wheels-signed" + displayName: Repackage mac wheels + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: '$(Build.ArtifactStagingDirectory)/mac-wheels-signed' + ArtifactName: 'packages_mac_signed' + + - job: Sign_Windows + displayName: Sign Windows Wheels + pool: + name: $(WINDOWSPOOL) + image: $(WINDOWSVMIMAGE) + os: windows + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - task: UsePythonVersion@0 + displayName: "Use Python $(PythonVersion)" + inputs: + versionSpec: $(PythonVersion) + + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: packages_windows + targetPath: $(Build.ArtifactStagingDirectory)/packages_windows + + - pwsh: | + python eng/scripts/wheel_signing/extract_sign_inputs.py ` + --platform windows ` + --wheels-dir "$(Build.ArtifactStagingDirectory)/packages_windows" ` + --work-dir "$(Build.ArtifactStagingDirectory)/win-sign-work" ` + --sign-input-dir "$(Build.ArtifactStagingDirectory)/win-sign-input" + displayName: Extract windows wheel binaries + + - template: pipelines/steps/azd-cli-win-signing.yml@azure-sdk-build-tools + parameters: + WinPath: "$(Build.ArtifactStagingDirectory)/win-sign-input" + WinPattern: '*.pyd' + + - pwsh: | + python eng/scripts/wheel_signing/repackage_signed_wheels.py ` + --platform windows ` + --work-dir "$(Build.ArtifactStagingDirectory)/win-sign-work" ` + --signed-input-dir "$(Build.ArtifactStagingDirectory)/win-sign-input" ` + --output-wheels-dir "$(Build.ArtifactStagingDirectory)/win-wheels-signed" + displayName: Repackage windows wheels + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: '$(Build.ArtifactStagingDirectory)/win-wheels-signed' + ArtifactName: 'packages_win_signed' + + - job: Consolidate_Wheels + displayName: Consolidate Wheels + dependsOn: + - Sign_macOS + - Sign_Windows + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + steps: + - checkout: none + + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: packages_linux + targetPath: $(Build.ArtifactStagingDirectory)/packages_linux + + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: packages_mac_signed + targetPath: $(Build.ArtifactStagingDirectory)/packages_mac + + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: packages_win_signed + targetPath: $(Build.ArtifactStagingDirectory)/packages_windows + + - pwsh: | + $allWheelsDir = "$(Build.ArtifactStagingDirectory)/all_wheels" + New-Item -ItemType Directory -Path $allWheelsDir -Force | Out-Null + + Get-ChildItem "$(Build.ArtifactStagingDirectory)/packages_linux" -Recurse -Filter "*.whl" | + Copy-Item -Destination $allWheelsDir -ErrorAction SilentlyContinue + + Get-ChildItem "$(Build.ArtifactStagingDirectory)/packages_mac" -Recurse -Filter "*.whl" | + Copy-Item -Destination $allWheelsDir -ErrorAction SilentlyContinue + + Get-ChildItem "$(Build.ArtifactStagingDirectory)/packages_windows" -Recurse -Filter "*.whl" | + Copy-Item -Destination $allWheelsDir -ErrorAction SilentlyContinue + displayName: Consolidate signed wheels + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: '$(Build.ArtifactStagingDirectory)/all_wheels' + ArtifactName: 'packages_all_signed' + + variables: + - template: /eng/pipelines/templates/variables/globals.yml + - template: /eng/pipelines/templates/variables/image.yml + + # Release stage intentionally disabled for first cut while build/signing are validated. + # - template: archetype-python-release.yml + # parameters: + # DependsOn: "Sign_Wheels" + # ServiceDirectory: ${{ parameters.ServiceDirectory }} + # Artifacts: ${{ parameters.Artifacts }} + # ${{ if eq(parameters.ServiceDirectory, 'template') }}: + # TestPipeline: true + # ArtifactName: packages_all_signed + # DocArtifact: documentation + # TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }} + # TargetDocRepoName: ${{ parameters.TargetDocRepoName }} + # DevFeedName: ${{ parameters.DevFeedName }} + # PublicFeed: ${{ parameters.PublicFeed }} + # PublicPublishEnvironment: ${{ parameters.PublicPublishEnvironment }} From 5165090c0f3728ed32804af8daa4ad30c2779d48 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 22 Jun 2026 12:06:28 -0700 Subject: [PATCH 02/21] Artifact creation, DevOps release --- .../stages/archetype-python-release.yml | 36 ++++++++++--------- .../templates/stages/archetype-sdk-client.yml | 14 ++++++-- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml index 6632d6f6bdd3..7b1b8ebb22f3 100644 --- a/eng/pipelines/templates/stages/archetype-python-release.yml +++ b/eng/pipelines/templates/stages/archetype-python-release.yml @@ -152,8 +152,8 @@ stages: artifactName: release_artifact targetPath: $(Pipeline.Workspace)/release_artifact - input: pipelineArtifact - artifactName: packages_extended - targetPath: $(Pipeline.Workspace)/packages_extended + artifactName: ${{parameters.ArtifactName}} + targetPath: $(Pipeline.Workspace)/${{parameters.ArtifactName}} pool: image: ubuntu-24.04 @@ -179,21 +179,23 @@ stages: python -m pip install -r $(Pipeline.Workspace)/release_artifact/release_requirements.txt displayName: Install Release Dependencies - - ${{ if eq(parameters.PublicFeed, 'PyPi') }}: - - pwsh: | - $esrpDirectory = "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{artifact.name}}" - New-Item -ItemType Directory -Force -Path $esrpDirectory - - Get-ChildItem -Path "$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}" ` - | Where-Object { ($_.Name -like "*.tar.gz" -or $_.Name -like "*.whl") } ` - | Copy-Item -Destination $esrpDirectory - - Get-ChildItem $esrpDirectory - displayName: Isolate files for ESRP Publish - - - template: /eng/pipelines/templates/steps/esrp-publish.yml - parameters: - targetFolder: "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{artifact.name}}" + # Live PyPI publication via ESRP is intentionally disabled. + # Keep the non-PyPI/PublicFeed and DevFeed publication paths below. + # - ${{ if eq(parameters.PublicFeed, 'PyPi') }}: + # - pwsh: | + # $esrpDirectory = "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{artifact.name}}" + # New-Item -ItemType Directory -Force -Path $esrpDirectory + # + # Get-ChildItem -Path "$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}" ` + # | Where-Object { ($_.Name -like "*.tar.gz" -or $_.Name -like "*.whl") } ` + # | Copy-Item -Destination $esrpDirectory + # + # Get-ChildItem $esrpDirectory + # displayName: Isolate files for ESRP Publish + # + # - template: /eng/pipelines/templates/steps/esrp-publish.yml + # parameters: + # targetFolder: "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{artifact.name}}" - ${{ if ne(parameters.PublicFeed, 'PyPi') }}: - task: TwineAuthenticate@0 diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 5851921a04b1..ef1409f51cd5 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -244,8 +244,8 @@ extends: ArtifactPath: '$(Build.ArtifactStagingDirectory)/win-wheels-signed' ArtifactName: 'packages_win_signed' - - job: Consolidate_Wheels - displayName: Consolidate Wheels + - job: Build_Release_Artifact + displayName: Build release artifact dependsOn: - Sign_macOS - Sign_Windows @@ -261,6 +261,11 @@ extends: artifactName: packages_linux targetPath: $(Build.ArtifactStagingDirectory)/packages_linux + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: packages_extended + targetPath: $(Build.ArtifactStagingDirectory)/packages_extended + - task: DownloadPipelineArtifact@2 inputs: artifactName: packages_mac_signed @@ -278,12 +283,15 @@ extends: Get-ChildItem "$(Build.ArtifactStagingDirectory)/packages_linux" -Recurse -Filter "*.whl" | Copy-Item -Destination $allWheelsDir -ErrorAction SilentlyContinue + Get-ChildItem "$(Build.ArtifactStagingDirectory)/packages_extended" -Recurse -Filter "*.tar.gz" | + Copy-Item -Destination $allWheelsDir -ErrorAction SilentlyContinue + Get-ChildItem "$(Build.ArtifactStagingDirectory)/packages_mac" -Recurse -Filter "*.whl" | Copy-Item -Destination $allWheelsDir -ErrorAction SilentlyContinue Get-ChildItem "$(Build.ArtifactStagingDirectory)/packages_windows" -Recurse -Filter "*.whl" | Copy-Item -Destination $allWheelsDir -ErrorAction SilentlyContinue - displayName: Consolidate signed wheels + displayName: Assemble release artifact - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: From ad28f6dfae870e746701636a432441885b20e7b6 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 22 Jun 2026 12:08:10 -0700 Subject: [PATCH 03/21] Uncomment --- .../templates/stages/archetype-sdk-client.yml | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index ef1409f51cd5..3616b755f6a6 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -302,18 +302,17 @@ extends: - template: /eng/pipelines/templates/variables/globals.yml - template: /eng/pipelines/templates/variables/image.yml - # Release stage intentionally disabled for first cut while build/signing are validated. - # - template: archetype-python-release.yml - # parameters: - # DependsOn: "Sign_Wheels" - # ServiceDirectory: ${{ parameters.ServiceDirectory }} - # Artifacts: ${{ parameters.Artifacts }} - # ${{ if eq(parameters.ServiceDirectory, 'template') }}: - # TestPipeline: true - # ArtifactName: packages_all_signed - # DocArtifact: documentation - # TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }} - # TargetDocRepoName: ${{ parameters.TargetDocRepoName }} - # DevFeedName: ${{ parameters.DevFeedName }} - # PublicFeed: ${{ parameters.PublicFeed }} - # PublicPublishEnvironment: ${{ parameters.PublicPublishEnvironment }} + - template: archetype-python-release.yml + parameters: + DependsOn: "Sign_Wheels" + ServiceDirectory: ${{ parameters.ServiceDirectory }} + Artifacts: ${{ parameters.Artifacts }} + ${{ if eq(parameters.ServiceDirectory, 'template') }}: + TestPipeline: true + ArtifactName: packages_all_signed + DocArtifact: documentation + TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }} + TargetDocRepoName: ${{ parameters.TargetDocRepoName }} + DevFeedName: ${{ parameters.DevFeedName }} + PublicFeed: ${{ parameters.PublicFeed }} + PublicPublishEnvironment: ${{ parameters.PublicPublishEnvironment }} From 3dada1a9b32c893a25cab831d0396b9dbfed089e Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 22 Jun 2026 12:14:52 -0700 Subject: [PATCH 04/21] Wire storage releases to signed artifacts Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../templates/stages/archetype-sdk-client.yml | 64 +++++++++++++++---- 1 file changed, 53 insertions(+), 11 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 3616b755f6a6..6ba0a1b9c9ef 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -277,25 +277,63 @@ extends: targetPath: $(Build.ArtifactStagingDirectory)/packages_windows - pwsh: | - $allWheelsDir = "$(Build.ArtifactStagingDirectory)/all_wheels" - New-Item -ItemType Directory -Path $allWheelsDir -Force | Out-Null + $releaseArtifactDir = "$(Build.ArtifactStagingDirectory)/release_packages" + $linuxPackagesDir = "$(Build.ArtifactStagingDirectory)/packages_linux" + $extendedPackagesDir = "$(Build.ArtifactStagingDirectory)/packages_extended" + New-Item -ItemType Directory -Path $releaseArtifactDir -Force | Out-Null - Get-ChildItem "$(Build.ArtifactStagingDirectory)/packages_linux" -Recurse -Filter "*.whl" | - Copy-Item -Destination $allWheelsDir -ErrorAction SilentlyContinue + Get-ChildItem $extendedPackagesDir | + Copy-Item -Destination $releaseArtifactDir -Recurse -Force - Get-ChildItem "$(Build.ArtifactStagingDirectory)/packages_extended" -Recurse -Filter "*.tar.gz" | - Copy-Item -Destination $allWheelsDir -ErrorAction SilentlyContinue + function Copy-ArtifactPreservingRelativePath { + param( + [string]$artifactPath, + [string]$sourceRoot + ) + + $sourceDirectory = Split-Path $artifactPath -Parent + $relativeDirectory = $sourceDirectory.Substring($sourceRoot.Length).TrimStart('\', '/') + $targetDirectory = if ([string]::IsNullOrEmpty($relativeDirectory)) { + $releaseArtifactDir + } else { + Join-Path $releaseArtifactDir $relativeDirectory + } + + New-Item -ItemType Directory -Path $targetDirectory -Force | Out-Null + Copy-Item -Path $artifactPath -Destination $targetDirectory -Force + } + + function Copy-SignedWheelToPackageFolder { + param([string]$wheelPath) + + $wheelName = Split-Path $wheelPath -Leaf + if ($wheelName -notmatch '^(?.+?)-(?\d[^-]*)-') { + throw "Unable to determine the package folder for signed wheel '$wheelName'." + } + + $packageFolder = $matches['package'].Replace('_', '-') + $targetFolder = Join-Path $releaseArtifactDir $packageFolder + + if (-not (Test-Path $targetFolder)) { + throw "Expected package folder '$targetFolder' for signed wheel '$wheelName'." + } + + Copy-Item -Path $wheelPath -Destination $targetFolder -Force + } + + Get-ChildItem $linuxPackagesDir -Recurse -Filter "*.whl" | + ForEach-Object { Copy-ArtifactPreservingRelativePath $_.FullName $linuxPackagesDir } Get-ChildItem "$(Build.ArtifactStagingDirectory)/packages_mac" -Recurse -Filter "*.whl" | - Copy-Item -Destination $allWheelsDir -ErrorAction SilentlyContinue + ForEach-Object { Copy-SignedWheelToPackageFolder $_.FullName } Get-ChildItem "$(Build.ArtifactStagingDirectory)/packages_windows" -Recurse -Filter "*.whl" | - Copy-Item -Destination $allWheelsDir -ErrorAction SilentlyContinue + ForEach-Object { Copy-SignedWheelToPackageFolder $_.FullName } displayName: Assemble release artifact - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: - ArtifactPath: '$(Build.ArtifactStagingDirectory)/all_wheels' + ArtifactPath: '$(Build.ArtifactStagingDirectory)/release_packages' ArtifactName: 'packages_all_signed' variables: @@ -304,12 +342,16 @@ extends: - template: archetype-python-release.yml parameters: - DependsOn: "Sign_Wheels" + ${{ if eq(parameters.ServiceDirectory, 'storage') }}: + DependsOn: Sign_Binaries + ArtifactName: packages_all_signed + ${{ if ne(parameters.ServiceDirectory, 'storage') }}: + DependsOn: Build + ArtifactName: packages_extended ServiceDirectory: ${{ parameters.ServiceDirectory }} Artifacts: ${{ parameters.Artifacts }} ${{ if eq(parameters.ServiceDirectory, 'template') }}: TestPipeline: true - ArtifactName: packages_all_signed DocArtifact: documentation TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }} TargetDocRepoName: ${{ parameters.TargetDocRepoName }} From c7eef03af6fcdb53824e8f51505756ab5e505d6f Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 22 Jun 2026 12:16:44 -0700 Subject: [PATCH 05/21] Only build azure-storage-extensions --- sdk/storage/ci.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index a05078529031..6863c1d224d5 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -43,16 +43,16 @@ extends: # must be scanned EnableCompiledCodeql: true Artifacts: - - name: azure-storage-blob - safeName: azurestorageblob - - name: azure-storage-blob-changefeed - safeName: azurestorageblobchangefeed - - name: azure-storage-file-share - safeName: azurestoragefileshare - - name: azure-storage-file-datalake - safeName: azurestoragefiledatalake - - name: azure-storage-queue - safeName: azurestoragequeue + # - name: azure-storage-blob + # safeName: azurestorageblob + # - name: azure-storage-blob-changefeed + # safeName: azurestorageblobchangefeed + # - name: azure-storage-file-share + # safeName: azurestoragefileshare + # - name: azure-storage-file-datalake + # safeName: azurestoragefiledatalake + # - name: azure-storage-queue + # safeName: azurestoragequeue - name: azure-storage-extensions safeName: azurestorageextensions triggeringPaths: @@ -63,11 +63,11 @@ extends: # Pure C-based storage extension package, not generating docs at this moment. skipPublishDocGithubIo: true skipPublishDocMs: true - - name: azure-mgmt-storage - safeName: azuremgmtstorage - - name: azure-mgmt-storagecache - safeName: azuremgmtstoragecache - - name: azure-mgmt-storagesync - safeName: azuremgmtstoragesync - - name: azure-mgmt-storageimportexport - safeName: azuremgmtstorageimportexport + # - name: azure-mgmt-storage + # safeName: azuremgmtstorage + # - name: azure-mgmt-storagecache + # safeName: azuremgmtstoragecache + # - name: azure-mgmt-storagesync + # safeName: azuremgmtstoragesync + # - name: azure-mgmt-storageimportexport + # safeName: azuremgmtstorageimportexport From 4ee0c97342bb30146bafc66b48b206eb33feeb08 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 23 Jun 2026 11:59:57 -0700 Subject: [PATCH 06/21] BuildTargetingString --- sdk/storage/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index 6863c1d224d5..64f567dff463 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -37,6 +37,7 @@ extends: ${{ if eq(parameters.ReleaseToDevOpsOnly, 'true') }}: PublicFeed: 'public/storage-staging' ServiceDirectory: storage + BuildTargetingString: azure-storage-extensions TestProxy: true TestTimeoutInMinutes: 120 # Enable Compiled CodeQL because azure-storage-extensions has C code that From 1d6f736d64a0fa9693b2d9959e8cc10d2a5a206f Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 23 Jun 2026 14:32:01 -0700 Subject: [PATCH 07/21] Restore signing extraction and repackage scripts --- .../templates/stages/archetype-sdk-client.yml | 13 +- .../wheel_signing/extract_sign_inputs.py | 176 ++++++++++++++++ .../wheel_signing/repackage_signed_wheels.py | 199 ++++++++++++++++++ 3 files changed, 382 insertions(+), 6 deletions(-) create mode 100644 eng/scripts/wheel_signing/extract_sign_inputs.py create mode 100644 eng/scripts/wheel_signing/repackage_signed_wheels.py diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 6ba0a1b9c9ef..6e95e9a98ad0 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -156,9 +156,9 @@ extends: - job: Sign_macOS displayName: Sign macOS Wheels pool: - name: $(WINDOWSPOOL) - image: $(WINDOWSVMIMAGE) - os: windows + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux steps: - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -173,6 +173,7 @@ extends: targetPath: $(Build.ArtifactStagingDirectory)/packages_mac - pwsh: | + Get-ChildItem python eng/scripts/wheel_signing/extract_sign_inputs.py ` --platform mac ` --wheels-dir "$(Build.ArtifactStagingDirectory)/packages_mac" ` @@ -202,9 +203,9 @@ extends: - job: Sign_Windows displayName: Sign Windows Wheels pool: - name: $(WINDOWSPOOL) - image: $(WINDOWSVMIMAGE) - os: windows + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux steps: - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml diff --git a/eng/scripts/wheel_signing/extract_sign_inputs.py b/eng/scripts/wheel_signing/extract_sign_inputs.py new file mode 100644 index 000000000000..3f8b9cfd8235 --- /dev/null +++ b/eng/scripts/wheel_signing/extract_sign_inputs.py @@ -0,0 +1,176 @@ +import argparse +import json +import shutil +import zipfile +from pathlib import Path +from typing import Dict, List + + +SIGNABLE_SUFFIXES = {".so", ".dylib", ".dll", ".pyd"} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Extract wheel files, collect signable binaries, and generate " + "signing payloads plus a manifest for wheel repackaging." + ) + ) + parser.add_argument("--platform", choices=["mac", "windows"], required=True) + parser.add_argument("--wheels-dir", required=True, help="Directory containing input wheel files.") + parser.add_argument("--work-dir", required=True, help="Working directory for unpacked wheels and manifest.") + parser.add_argument( + "--sign-input-zip", + default=None, + help="Output zip containing binaries to sign (required for --platform mac).", + ) + parser.add_argument( + "--sign-input-dir", + default=None, + help="Output folder containing binaries to sign (required for --platform windows).", + ) + return parser.parse_args() + + +def validate_args(args: argparse.Namespace) -> None: + if args.platform == "mac": + if not args.sign_input_zip: + raise ValueError("--sign-input-zip is required for --platform mac.") + if args.sign_input_dir: + raise ValueError("--sign-input-dir is not valid for --platform mac.") + else: + if not args.sign_input_dir: + raise ValueError("--sign-input-dir is required for --platform windows.") + if args.sign_input_zip: + raise ValueError("--sign-input-zip is not valid for --platform windows.") + + +def reset_dir(path: Path) -> None: + if path.exists(): + shutil.rmtree(path) + path.mkdir(parents=True, exist_ok=True) + + +def collect_wheels(wheels_dir: Path) -> List[Path]: + if not wheels_dir.is_dir(): + raise FileNotFoundError(f"Wheel directory not found: {wheels_dir}") + return sorted(wheels_dir.rglob("*.whl")) + + +def collect_signable_files(unpacked_wheel_dir: Path) -> List[Path]: + files = [] + for path in sorted(unpacked_wheel_dir.rglob("*")): + if path.is_file() and path.suffix.lower() in SIGNABLE_SUFFIXES: + files.append(path) + return files + + +def write_manifest(manifest_path: Path, manifest_data: Dict) -> None: + manifest_path.parent.mkdir(parents=True, exist_ok=True) + with manifest_path.open("w", encoding="utf-8") as handle: + json.dump(manifest_data, handle, indent=2, sort_keys=True) + handle.write("\n") + + +def create_zip_from_dir(source_dir: Path, output_zip: Path) -> None: + output_zip.parent.mkdir(parents=True, exist_ok=True) + if output_zip.exists(): + output_zip.unlink() + + with zipfile.ZipFile(output_zip, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for file_path in sorted(source_dir.rglob("*")): + if file_path.is_file(): + archive.write(file_path, file_path.relative_to(source_dir).as_posix()) + + +def main() -> None: + args = parse_args() + validate_args(args) + + wheels_dir = Path(args.wheels_dir).resolve() + work_dir = Path(args.work_dir).resolve() + unpack_root = work_dir / "unpacked" + manifest_path = work_dir / "signing-manifest.json" + + if args.platform == "windows": + payload_dir = Path(args.sign_input_dir).resolve() + else: + payload_dir = work_dir / "mac-sign-input" + + reset_dir(work_dir) + reset_dir(unpack_root) + reset_dir(payload_dir) + + wheels = collect_wheels(wheels_dir) + print(f"Platform: {args.platform}") + print(f"Input wheels dir: {wheels_dir}") + print(f"Work dir: {work_dir}") + + manifest: Dict[str, object] = { + "platform": args.platform, + "wheels": [], + "entries": [], + } + + payload_index = 0 + + for wheel_path in wheels: + unpack_dir_name = wheel_path.name[:-4] + unpacked_wheel_dir = unpack_root / unpack_dir_name + unpacked_wheel_dir.mkdir(parents=True, exist_ok=True) + print(f"[EXTRACT] wheel={wheel_path.name} unpack_dir={unpacked_wheel_dir}") + + with zipfile.ZipFile(wheel_path, "r") as archive: + archive.extractall(unpacked_wheel_dir) + + manifest["wheels"].append( + { + "wheel_filename": wheel_path.name, + "unpack_dir": unpack_dir_name, + } + ) + + signable_files = collect_signable_files(unpacked_wheel_dir) + print(f"[EXTRACT] wheel={wheel_path.name} signable_count={len(signable_files)}") + for signable_file in signable_files: + relative_path = signable_file.relative_to(unpacked_wheel_dir).as_posix() + payload_name = f"{payload_index:05d}__{signable_file.name}" + payload_index += 1 + + payload_path = payload_dir / payload_name + shutil.copy2(signable_file, payload_path) + print( + "[MAP_EXTRACT] " + f"payload={payload_name} " + f"source_wheel={wheel_path.name} " + f"source_relative_path={relative_path} " + f"source_file={signable_file} " + f"payload_file={payload_path}" + ) + + manifest["entries"].append( + { + "wheel_filename": wheel_path.name, + "unpack_dir": unpack_dir_name, + "relative_path": relative_path, + "payload_name": payload_name, + } + ) + + write_manifest(manifest_path, manifest) + + if args.platform == "mac": + sign_zip = Path(args.sign_input_zip).resolve() + create_zip_from_dir(payload_dir, sign_zip) + + print(f"Wheels processed: {len(wheels)}") + print(f"Signable binaries collected: {len(manifest['entries'])}") + print(f"Manifest: {manifest_path}") + if args.platform == "mac": + print(f"Sign payload zip: {Path(args.sign_input_zip).resolve()}") + else: + print(f"Sign payload dir: {payload_dir}") + + +if __name__ == "__main__": + main() diff --git a/eng/scripts/wheel_signing/repackage_signed_wheels.py b/eng/scripts/wheel_signing/repackage_signed_wheels.py new file mode 100644 index 000000000000..292700db44b3 --- /dev/null +++ b/eng/scripts/wheel_signing/repackage_signed_wheels.py @@ -0,0 +1,199 @@ +import argparse +import base64 +import csv +import hashlib +import json +import shutil +import tempfile +import zipfile +from pathlib import Path +from typing import Dict, List + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Reinsert signed binaries into unpacked wheel trees and rebuild wheel files " + "with original filenames." + ) + ) + parser.add_argument("--platform", choices=["mac", "windows"], required=True) + parser.add_argument("--work-dir", required=True, help="Working directory created by extract_sign_inputs.py.") + parser.add_argument( + "--signed-input-zip", + default=None, + help="Signed binary zip payload (required for --platform mac).", + ) + parser.add_argument( + "--signed-input-dir", + default=None, + help="Signed binary folder payload (required for --platform windows).", + ) + parser.add_argument("--output-wheels-dir", required=True, help="Directory where rebuilt wheel files are written.") + return parser.parse_args() + + +def validate_args(args: argparse.Namespace) -> None: + if args.platform == "mac": + if not args.signed_input_zip: + raise ValueError("--signed-input-zip is required for --platform mac.") + if args.signed_input_dir: + raise ValueError("--signed-input-dir is not valid for --platform mac.") + else: + if not args.signed_input_dir: + raise ValueError("--signed-input-dir is required for --platform windows.") + if args.signed_input_zip: + raise ValueError("--signed-input-zip is not valid for --platform windows.") + + +def load_manifest(manifest_path: Path) -> Dict: + if not manifest_path.is_file(): + raise FileNotFoundError(f"Manifest not found: {manifest_path}") + with manifest_path.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def reset_dir(path: Path) -> None: + if path.exists(): + shutil.rmtree(path) + path.mkdir(parents=True, exist_ok=True) + + +def get_digest_and_size(file_path: Path) -> List[str]: + content = file_path.read_bytes() + digest = base64.urlsafe_b64encode(hashlib.sha256(content).digest()).decode("ascii").rstrip("=") + return [f"sha256={digest}", str(len(content))] + + +def find_record_path(unpacked_wheel_dir: Path) -> Path: + candidates = sorted(unpacked_wheel_dir.glob("*.dist-info/RECORD")) + if len(candidates) != 1: + raise RuntimeError( + f"Expected exactly one RECORD file under {unpacked_wheel_dir}, found {len(candidates)}." + ) + return candidates[0] + + +def rewrite_record(unpacked_wheel_dir: Path) -> None: + record_path = find_record_path(unpacked_wheel_dir) + record_rel = record_path.relative_to(unpacked_wheel_dir).as_posix() + + rows: List[List[str]] = [] + for file_path in sorted(unpacked_wheel_dir.rglob("*")): + if not file_path.is_file(): + continue + rel = file_path.relative_to(unpacked_wheel_dir).as_posix() + if rel == record_rel: + continue + digest, size = get_digest_and_size(file_path) + rows.append([rel, digest, size]) + + rows.append([record_rel, "", ""]) + + with record_path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.writer(handle, lineterminator="\n") + writer.writerows(rows) + + +def build_wheel(unpacked_wheel_dir: Path, output_wheel_path: Path) -> None: + output_wheel_path.parent.mkdir(parents=True, exist_ok=True) + if output_wheel_path.exists(): + output_wheel_path.unlink() + + rewrite_record(unpacked_wheel_dir) + + with zipfile.ZipFile(output_wheel_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for file_path in sorted(unpacked_wheel_dir.rglob("*")): + if file_path.is_file(): + archive.write(file_path, file_path.relative_to(unpacked_wheel_dir).as_posix()) + + +def main() -> None: + args = parse_args() + validate_args(args) + + work_dir = Path(args.work_dir).resolve() + unpack_root = work_dir / "unpacked" + manifest_path = work_dir / "signing-manifest.json" + output_wheels_dir = Path(args.output_wheels_dir).resolve() + + manifest = load_manifest(manifest_path) + if manifest.get("platform") != args.platform: + raise RuntimeError( + f"Manifest platform '{manifest.get('platform')}' does not match argument platform '{args.platform}'." + ) + + if not unpack_root.is_dir(): + raise FileNotFoundError(f"Unpacked wheel directory not found: {unpack_root}") + + print(f"Platform: {args.platform}") + print(f"Work dir: {work_dir}") + print(f"Manifest: {manifest_path}") + + with tempfile.TemporaryDirectory(prefix="signed-binaries-") as tmpdir: + if args.platform == "mac": + signed_payload_dir = Path(tmpdir) / "signed-payload" + signed_payload_dir.mkdir(parents=True, exist_ok=True) + signed_zip = Path(args.signed_input_zip).resolve() + if not signed_zip.is_file(): + raise FileNotFoundError(f"Signed payload zip not found: {signed_zip}") + with zipfile.ZipFile(signed_zip, "r") as archive: + archive.extractall(signed_payload_dir) + print(f"Signed payload zip: {signed_zip}") + else: + signed_payload_dir = Path(args.signed_input_dir).resolve() + if not signed_payload_dir.is_dir(): + raise FileNotFoundError(f"Signed payload directory not found: {signed_payload_dir}") + print(f"Signed payload dir: {signed_payload_dir}") + + for entry in manifest.get("entries", []): + wheel_filename = entry["wheel_filename"] + unpack_dir = entry["unpack_dir"] + relative_path = entry["relative_path"] + payload_name = entry["payload_name"] + + source_signed_binary = signed_payload_dir / payload_name + target_binary = unpack_root / unpack_dir / relative_path + + if not source_signed_binary.is_file(): + raise FileNotFoundError(f"Signed binary missing: {source_signed_binary}") + if not target_binary.is_file(): + raise FileNotFoundError(f"Target binary missing in unpacked wheel: {target_binary}") + + shutil.copy2(source_signed_binary, target_binary) + print( + "[MAP_REPACKAGE] " + f"payload={payload_name} " + f"target_wheel={wheel_filename} " + f"target_relative_path={relative_path} " + f"signed_source={source_signed_binary} " + f"target_file={target_binary}" + ) + + reset_dir(output_wheels_dir) + + rebuilt_count = 0 + for wheel_info in manifest.get("wheels", []): + wheel_filename = wheel_info["wheel_filename"] + unpack_dir = wheel_info["unpack_dir"] + + unpacked_wheel_dir = unpack_root / unpack_dir + if not unpacked_wheel_dir.is_dir(): + raise FileNotFoundError(f"Unpacked wheel directory missing: {unpacked_wheel_dir}") + + output_wheel_path = output_wheels_dir / wheel_filename + print( + "[REBUILD] " + f"wheel={wheel_filename} " + f"unpacked_dir={unpacked_wheel_dir} " + f"output_wheel={output_wheel_path}" + ) + build_wheel(unpacked_wheel_dir, output_wheel_path) + rebuilt_count += 1 + + print(f"Wheels rebuilt: {rebuilt_count}") + print(f"Output wheels dir: {output_wheels_dir}") + + +if __name__ == "__main__": + main() From 2efae67c76135ac798bed5ea598f81b9eab36f18 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 21 Jul 2026 12:24:21 -0700 Subject: [PATCH 08/21] Declare azure-sdk-build-tools repo resource (tag-pinned) Restores the azure-sdk-build-tools repository resource referenced by the mac/win signing templates (@azure-sdk-build-tools), pinned to tag azure-sdk-build-tools_20260702.2 to match azure-dev. This seeds a ref line that the tools-repo-versioning auto-updater will keep current. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8225ba73-f36e-4078-b21a-a7a166d261f9 --- eng/pipelines/templates/stages/1es-redirect.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/pipelines/templates/stages/1es-redirect.yml b/eng/pipelines/templates/stages/1es-redirect.yml index 6878cdb48af0..693786c6d4f2 100644 --- a/eng/pipelines/templates/stages/1es-redirect.yml +++ b/eng/pipelines/templates/stages/1es-redirect.yml @@ -8,6 +8,10 @@ resources: type: git name: 1ESPipelineTemplates/1ESPipelineTemplates ref: refs/tags/canary + - repository: azure-sdk-build-tools + type: git + name: internal/azure-sdk-build-tools + ref: refs/tags/azure-sdk-build-tools_20260702.2 parameters: - name: stages From b50cf64796d5b218cea1e18da52669dbeea7fdd8 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 21 Jul 2026 12:39:04 -0700 Subject: [PATCH 09/21] Re-enable ESRP PyPI publish for signed release Restores the ESRP publish path (isolate + esrp-publish.yml) gated on PublicFeed == 'PyPi'. Reads from packages_all_signed/, so the signed wheels/sdist are what get published. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8225ba73-f36e-4078-b21a-a7a166d261f9 --- .../stages/archetype-python-release.yml | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml index 7b1b8ebb22f3..cde40b71146b 100644 --- a/eng/pipelines/templates/stages/archetype-python-release.yml +++ b/eng/pipelines/templates/stages/archetype-python-release.yml @@ -179,23 +179,21 @@ stages: python -m pip install -r $(Pipeline.Workspace)/release_artifact/release_requirements.txt displayName: Install Release Dependencies - # Live PyPI publication via ESRP is intentionally disabled. - # Keep the non-PyPI/PublicFeed and DevFeed publication paths below. - # - ${{ if eq(parameters.PublicFeed, 'PyPi') }}: - # - pwsh: | - # $esrpDirectory = "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{artifact.name}}" - # New-Item -ItemType Directory -Force -Path $esrpDirectory - # - # Get-ChildItem -Path "$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}" ` - # | Where-Object { ($_.Name -like "*.tar.gz" -or $_.Name -like "*.whl") } ` - # | Copy-Item -Destination $esrpDirectory - # - # Get-ChildItem $esrpDirectory - # displayName: Isolate files for ESRP Publish - # - # - template: /eng/pipelines/templates/steps/esrp-publish.yml - # parameters: - # targetFolder: "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{artifact.name}}" + - ${{ if eq(parameters.PublicFeed, 'PyPi') }}: + - pwsh: | + $esrpDirectory = "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{artifact.name}}" + New-Item -ItemType Directory -Force -Path $esrpDirectory + + Get-ChildItem -Path "$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}" ` + | Where-Object { ($_.Name -like "*.tar.gz" -or $_.Name -like "*.whl") } ` + | Copy-Item -Destination $esrpDirectory + + Get-ChildItem $esrpDirectory + displayName: Isolate files for ESRP Publish + + - template: /eng/pipelines/templates/steps/esrp-publish.yml + parameters: + targetFolder: "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{artifact.name}}" - ${{ if ne(parameters.PublicFeed, 'PyPi') }}: - task: TwineAuthenticate@0 From ec2c2ef6e8263c09609ab6c96a4eadafb9a03995 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 21 Jul 2026 13:37:32 -0700 Subject: [PATCH 10/21] Signing extraction, prevent accidental publish --- .../templates/stages/archetype-python-release.yml | 7 ++++--- eng/scripts/wheel_signing/extract_sign_inputs.py | 14 ++++++++------ .../wheel_signing/repackage_signed_wheels.py | 6 +++--- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml index cde40b71146b..2931d72c9eb9 100644 --- a/eng/pipelines/templates/stages/archetype-python-release.yml +++ b/eng/pipelines/templates/stages/archetype-python-release.yml @@ -191,9 +191,10 @@ stages: Get-ChildItem $esrpDirectory displayName: Isolate files for ESRP Publish - - template: /eng/pipelines/templates/steps/esrp-publish.yml - parameters: - targetFolder: "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{artifact.name}}" + # TEMP: disabled to avoid accidental ESRP publish during testing — re-enable before release + # - template: /eng/pipelines/templates/steps/esrp-publish.yml + # parameters: + # targetFolder: "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{artifact.name}}" - ${{ if ne(parameters.PublicFeed, 'PyPi') }}: - task: TwineAuthenticate@0 diff --git a/eng/scripts/wheel_signing/extract_sign_inputs.py b/eng/scripts/wheel_signing/extract_sign_inputs.py index 3f8b9cfd8235..045dc4ea1c5c 100644 --- a/eng/scripts/wheel_signing/extract_sign_inputs.py +++ b/eng/scripts/wheel_signing/extract_sign_inputs.py @@ -134,18 +134,20 @@ def main() -> None: print(f"[EXTRACT] wheel={wheel_path.name} signable_count={len(signable_files)}") for signable_file in signable_files: relative_path = signable_file.relative_to(unpacked_wheel_dir).as_posix() - payload_name = f"{payload_index:05d}__{signable_file.name}" + payload_subdir = f"{payload_index:05d}" payload_index += 1 - payload_path = payload_dir / payload_name - shutil.copy2(signable_file, payload_path) + payload_path = f"{payload_subdir}/{signable_file.name}" + payload_file = payload_dir / payload_subdir / signable_file.name + payload_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(signable_file, payload_file) print( "[MAP_EXTRACT] " - f"payload={payload_name} " + f"payload={payload_path} " f"source_wheel={wheel_path.name} " f"source_relative_path={relative_path} " f"source_file={signable_file} " - f"payload_file={payload_path}" + f"payload_file={payload_file}" ) manifest["entries"].append( @@ -153,7 +155,7 @@ def main() -> None: "wheel_filename": wheel_path.name, "unpack_dir": unpack_dir_name, "relative_path": relative_path, - "payload_name": payload_name, + "payload_path": payload_path, } ) diff --git a/eng/scripts/wheel_signing/repackage_signed_wheels.py b/eng/scripts/wheel_signing/repackage_signed_wheels.py index 292700db44b3..0fa25172ca3d 100644 --- a/eng/scripts/wheel_signing/repackage_signed_wheels.py +++ b/eng/scripts/wheel_signing/repackage_signed_wheels.py @@ -150,9 +150,9 @@ def main() -> None: wheel_filename = entry["wheel_filename"] unpack_dir = entry["unpack_dir"] relative_path = entry["relative_path"] - payload_name = entry["payload_name"] + payload_path = entry["payload_path"] - source_signed_binary = signed_payload_dir / payload_name + source_signed_binary = signed_payload_dir / payload_path target_binary = unpack_root / unpack_dir / relative_path if not source_signed_binary.is_file(): @@ -163,7 +163,7 @@ def main() -> None: shutil.copy2(source_signed_binary, target_binary) print( "[MAP_REPACKAGE] " - f"payload={payload_name} " + f"payload={payload_path} " f"target_wheel={wheel_filename} " f"target_relative_path={relative_path} " f"signed_source={source_signed_binary} " From 5cb97be8ddfe25c07ebf9c3820062c92398528b2 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 21 Jul 2026 15:13:05 -0700 Subject: [PATCH 11/21] max parallel --- eng/pipelines/templates/steps/build-test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index 6597389cd56d..6c8725ab7174 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -94,7 +94,8 @@ steps: --mark_arg="$markArg" --service="${{ parameters.ServiceDirectory }}" --checks="${{ parameters.CheckEnv }}" - --injected-packages="${{ parameters.InjectedPackages }}"; + --injected-packages="${{ parameters.InjectedPackages }}" + --max-parallel=1; Write-Host "Last exit code: $LASTEXITCODE"; exit $LASTEXITCODE; @@ -116,7 +117,8 @@ steps: --mark_arg="$markArg" ` --service="${{ parameters.ServiceDirectory }}" ` --checks="${{ parameters.CheckEnv }}" ` - --injected-packages="${{ parameters.InjectedPackages }}"; + --injected-packages="${{ parameters.InjectedPackages }}" ` + --max-parallel=1; exit $LASTEXITCODE; env: ${{ parameters.EnvVars }} displayName: Run Tests From a950e92b0eb83135852bd3ee8313be9266375214 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 22 Jul 2026 10:54:53 -0700 Subject: [PATCH 12/21] Disable codeql for compiled languages on macos --- eng/pipelines/templates/jobs/ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 6bda192940e2..3ed6f229bde9 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -116,6 +116,13 @@ jobs: vmImage: $(MACVMIMAGE) os: macOS + templateContext: + sdl: + codeql: + compiled: + enabled: false + justificationForDisabling: "Compiled language support is not available on macOS ARM64. See: https://eng.ms/docs/coreai/devdiv/one-engineering-system-1es/1es-docs/codeql/troubleshooting/onboarding/language-compiled#arm64-cpu-on-macos" + steps: - template: /eng/pipelines/templates/steps/build-package-artifacts.yml parameters: From dc10b346db7577cb70088f4149d4e295a97396db Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 22 Jul 2026 11:25:23 -0700 Subject: [PATCH 13/21] CodeQL: cpp and python --- eng/pipelines/templates/stages/1es-redirect.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/1es-redirect.yml b/eng/pipelines/templates/stages/1es-redirect.yml index 693786c6d4f2..aa93db842d01 100644 --- a/eng/pipelines/templates/stages/1es-redirect.yml +++ b/eng/pipelines/templates/stages/1es-redirect.yml @@ -69,7 +69,7 @@ extends: ${{ if eq(parameters.EnableCompiledCodeql, true) }}: # "cpp" covers both C and C++ code. Language is specified because # checkout happens after the injected "CodeQL Initialize" step - language: cpp + language: cpp,python compiled: enabled: true ${{ else }}: From c7077f23ffc9433166214c7f22c3087c76db3e34 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 22 Jul 2026 15:28:19 -0700 Subject: [PATCH 14/21] Fix Windows binary signing no-op: recurse into payload subfolders ESRP Code Signing used minimatch pattern '*.pyd' (non-recursive) against win-sign-input/, but extract_sign_inputs.py places .pyd in numbered subfolders (00000/crc64.pyd). The glob matched 0 of 4 files, the task reported success signing nothing, and repackage reinserted unsigned .pyd. Mac was unaffected because it zips the payload dir and signs the zip. Change WinPattern to '**/*.pyd' so ESRP recurses into the subfolders, matching the mac path's recursive behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8225ba73-f36e-4078-b21a-a7a166d261f9 --- eng/pipelines/templates/stages/archetype-sdk-client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 6e95e9a98ad0..ed90d133e796 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -230,7 +230,7 @@ extends: - template: pipelines/steps/azd-cli-win-signing.yml@azure-sdk-build-tools parameters: WinPath: "$(Build.ArtifactStagingDirectory)/win-sign-input" - WinPattern: '*.pyd' + WinPattern: '**/*.pyd' - pwsh: | python eng/scripts/wheel_signing/repackage_signed_wheels.py ` From 58f98a17a62b8d982082005877f498d7ac94a8bc Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 23 Jul 2026 09:53:38 -0700 Subject: [PATCH 15/21] Re-enable ESRP publish for release Uncomment the esrp-publish.yml template call that was temporarily disabled during signing-pipeline testing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8225ba73-f36e-4078-b21a-a7a166d261f9 --- .../templates/stages/archetype-python-release.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml index 2931d72c9eb9..cde40b71146b 100644 --- a/eng/pipelines/templates/stages/archetype-python-release.yml +++ b/eng/pipelines/templates/stages/archetype-python-release.yml @@ -191,10 +191,9 @@ stages: Get-ChildItem $esrpDirectory displayName: Isolate files for ESRP Publish - # TEMP: disabled to avoid accidental ESRP publish during testing — re-enable before release - # - template: /eng/pipelines/templates/steps/esrp-publish.yml - # parameters: - # targetFolder: "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{artifact.name}}" + - template: /eng/pipelines/templates/steps/esrp-publish.yml + parameters: + targetFolder: "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{artifact.name}}" - ${{ if ne(parameters.PublicFeed, 'PyPi') }}: - task: TwineAuthenticate@0 From 4e2cc6577cccf044fd13d11375eb542616c33430 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 23 Jul 2026 10:03:52 -0700 Subject: [PATCH 16/21] Disable CodeQL compiled scanning on macos test jobs --- eng/pipelines/templates/jobs/ci.tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 4599c11c7661..4b24f9e6c7ba 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -74,6 +74,14 @@ jobs: os: ${{ parameters.OSName }} templateContext: + # Compiled CodeQL is auto-injected by 1ES but is unsupported on macOS ARM64 + # (and the test jobs do not compile the extension). Mirror the Build_MacOS carve-out. + ${{ if eq(parameters.OSName, 'macOS') }}: + sdl: + codeql: + compiled: + enabled: false + justificationForDisabling: "Compiled language support is not available on macOS ARM64. See: https://eng.ms/docs/coreai/devdiv/one-engineering-system-1es/1es-docs/codeql/troubleshooting/onboarding/language-compiled#arm64-cpu-on-macos" # See eng/common/pipelines/templates/steps/upload-llm-artifacts.yml for corresponding file copy step outputs: - output: pipelineArtifact From 7df30466d924ea5c8bcc8444dfa7610fe1a6145d Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Tue, 28 Jul 2026 12:26:21 -0400 Subject: [PATCH 17/21] Changed release date --- sdk/storage/azure-storage-extensions/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-extensions/CHANGELOG.md b/sdk/storage/azure-storage-extensions/CHANGELOG.md index 39fe92420296..849493e40c4e 100644 --- a/sdk/storage/azure-storage-extensions/CHANGELOG.md +++ b/sdk/storage/azure-storage-extensions/CHANGELOG.md @@ -1,5 +1,5 @@ # Release History -## 0.1.0 (Unreleased) +## 0.1.0 (2026-07-28) Initial release. From aa9f94c0b8b8a60af44ecc13061cd6443b40763f Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Tue, 28 Jul 2026 13:57:07 -0400 Subject: [PATCH 18/21] Features added section --- sdk/storage/azure-storage-extensions/CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-extensions/CHANGELOG.md b/sdk/storage/azure-storage-extensions/CHANGELOG.md index 849493e40c4e..9cf51f6b5715 100644 --- a/sdk/storage/azure-storage-extensions/CHANGELOG.md +++ b/sdk/storage/azure-storage-extensions/CHANGELOG.md @@ -2,4 +2,5 @@ ## 0.1.0 (2026-07-28) -Initial release. +### Features Added +- Initial release. From 860ac5e6db74cff11352a801814f0781efdc8e20 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 10 Sep 2026 13:45:31 -0700 Subject: [PATCH 19/21] Route signed-binary packages through a per-artifact release path Packages that ship compiled binaries need their wheels signed by ESRP before release, but the previous approach keyed signing off ServiceDirectory, which dragged every storage package onto the signed path. Make it a per-artifact opt-in instead. An artifact sets `signBinaries: true` in its service ci.yml and gets a dedicated Sign_ stage plus a Release_ stage fed from the resulting packages__signed artifact. Every other artifact keeps releasing from packages_extended off the build stage exactly as before, so a service can mix both kinds of packages. The signed artifact mirrors the layout of packages_extended, so the release jobs work against it unchanged. To keep those jobs single-sourced they move verbatim into release-artifact.yml, which parameterizes only the artifact name and the upstream stage. packages_extended itself is untouched. Signing runs on release and scheduled builds and never on PR builds: the public project cannot reach internal/azure-sdk-build-tools, and ESRP must not be handed unreviewed code. Scheduled runs are included because the Integration stage now publishes signed alpha packages to the dev feed for opted-in artifacts. The signing stage deliberately ignores Skip.Release and SetDevVersion. Those belong on the consumers, which already carry them; checking them here would propagate through Integration's dependency and silently break the nightly and manual dev version alpha publishes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9079636e-8478-4697-ae82-538136cd41b5 --- .../stages/archetype-python-release.yml | 474 +++--------------- .../templates/stages/archetype-sdk-client.yml | 202 +------- .../templates/stages/release-artifact.yml | 426 ++++++++++++++++ .../templates/stages/sign-binaries.yml | 232 +++++++++ eng/pipelines/templates/steps/build-test.yml | 6 +- .../templates/steps/publish-alpha-package.yml | 47 ++ sdk/storage/ci.yml | 40 +- 7 files changed, 798 insertions(+), 629 deletions(-) create mode 100644 eng/pipelines/templates/stages/release-artifact.yml create mode 100644 eng/pipelines/templates/stages/sign-binaries.yml create mode 100644 eng/pipelines/templates/steps/publish-alpha-package.yml diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml index 5357beb17f49..7f58eaa8880f 100644 --- a/eng/pipelines/templates/stages/archetype-python-release.yml +++ b/eng/pipelines/templates/stages/archetype-python-release.yml @@ -13,6 +13,24 @@ parameters: PackageSourceOverride: "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" stages: + # Binary signing stages, emitted per artifact that opted in with `signBinaries: true`. + # + # INVARIANT: this compile-time gate must remain a superset of every gate that references a + # Sign_* stage, namely the release gate below and the Integration stage's dependsOn. If they + # drift so a Sign_* stage is referenced but not emitted, the pipeline fails to compile on an + # unresolvable dependsOn. "internal and not a PR build" is the loosest of the three, so it is + # repeated verbatim in all of them, and the narrower "which run reasons actually sign" logic + # lives in the stage's runtime condition instead of being duplicated here. + # + # Signing never runs on PR builds: the public project cannot reach internal/azure-sdk-build-tools, + # and ESRP must not be handed unreviewed code. + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ each artifact in parameters.Artifacts }}: + - ${{ if eq(artifact.signBinaries, true) }}: + - template: /eng/pipelines/templates/stages/sign-binaries.yml + parameters: + Artifact: ${{ artifact }} + DependsOn: ${{ parameters.DependsOn }} # Release stages are compiled for: # * internal manual runs (existing behavior), and # * internal post-merge CI on 'main' (auto-release). For auto-release, only the packages changed by a @@ -46,389 +64,40 @@ stages: EnableUvAuth: true - ${{ each artifact in parameters.Artifacts }}: - - stage: 'Release_${{artifact.safename}}' - displayName: 'Release: ${{artifact.name}}' - dependsOn: - - ${{ parameters.DependsOn }} - - ${{ if and(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: - - AutoReleasePrepare - variables: - - template: /eng/pipelines/templates/variables/image.yml - - template: /eng/common/pipelines/templates/variables/api-review-break-glass.yml - # Auto-release CI: only release when the shared prepare stage flagged this artifact as changed. - # Manual runs: every declared artifact remains eligible (existing behavior). - ${{ if and(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: - condition: and(succeeded(), eq(dependencies.AutoReleasePrepare.outputs['ResolveAutoReleasePackages.resolve.ReleaseArtifact_${{ artifact.safename }}'], 'true'), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr')) + - template: /eng/pipelines/templates/stages/release-artifact.yml + parameters: + Artifact: ${{ artifact }} + # Packages that ship compiled binaries release from the artifact rebuilt by their + # Sign_ stage. Everything else keeps releasing from the build output, so a + # service can mix both kinds of packages freely. + ${{ if eq(artifact.signBinaries, true) }}: + ArtifactName: packages_${{ artifact.safeName }}_signed + DependsOn: Sign_${{ artifact.safeName }} ${{ else }}: - condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr')) - jobs: - - job: TagRepository - displayName: "Create release tag" - condition: and(succeeded(), ne(variables['Skip.TagRepository'], 'true')) - - pool: - image: ubuntu-24.04 - name: azsdk-pool - os: linux - - steps: - - checkout: self - - - download: current - artifact: ${{parameters.ArtifactName}} - timeoutInMinutes: 5 - - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.12' - - - template: /eng/pipelines/templates/steps/auth-dev-feed.yml - parameters: - DevFeedName: ${{ parameters.DevFeedName }} - - - template: /eng/common/pipelines/templates/steps/retain-run.yml - - - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml - parameters: - PackageName: "azure-template" - ServiceDirectory: "template" - TestPipeline: ${{ parameters.TestPipeline }} - - - template: /eng/common/pipelines/templates/steps/verify-changelog.yml - parameters: - PackageName: ${{artifact.name}} - ServiceName: ${{parameters.ServiceDirectory}} - ForRelease: true - - - template: /eng/common/pipelines/templates/steps/verify-restapi-spec-location.yml - parameters: - PackageName: ${{artifact.name}} - ServiceDirectory: ${{parameters.ServiceDirectory}} - ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}} - - - script: | - python -m pip install "./eng/tools/azure-sdk-tools" - displayName: Install tool dependencies - - - task: PythonScript@0 - displayName: Verify Dependency Presence - condition: and(succeeded(), ne(variables['Skip.VerifyDependencies'], 'true')) - inputs: - scriptPath: 'scripts/devops_tasks/verify_dependencies_present.py' - arguments: '--package-name ${{ artifact.name }} --service ${{ parameters.ServiceDirectory }}' - - - task: PythonScript@0 - displayName: Verify CI enabled - condition: succeeded() - inputs: - scriptPath: 'scripts/devops_tasks/verify_ci_enabled.py' - arguments: '--package-name ${{ artifact.name }} --service ${{ parameters.ServiceDirectory }}' - - - pwsh: | - Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}} - workingDirectory: $(Pipeline.Workspace) - displayName: Output Visible Artifacts - - - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml - parameters: - ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}} - PackageRepository: PyPI - ReleaseSha: $(Build.SourceVersion) - RepoId: Azure/azure-sdk-for-python - WorkingDirectory: $(System.DefaultWorkingDirectory) - AuthToken: '' - - - ${{if ne(artifact.skipPublishPackage, 'true')}}: - - deployment: PublishPackage - displayName: "Publish to ${{ parameters.PublicFeed }}" - condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true')) - # Auto-release runs after merge on main and must not wait on the package-publish approval gate. - ${{ if and(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: - environment: none - ${{ else }}: - environment: ${{ parameters.PublicPublishEnvironment }} - dependsOn: TagRepository - - templateContext: - type: releaseJob - isProduction: true - inputs: - - input: pipelineArtifact - artifactName: release_artifact - targetPath: $(Pipeline.Workspace)/release_artifact - - input: pipelineArtifact - artifactName: packages_extended - targetPath: $(Pipeline.Workspace)/packages_extended - - pool: - image: ubuntu-24.04 - name: azsdk-pool - os: linux - - strategy: - runOnce: - deploy: - steps: - - template: /eng/pipelines/templates/steps/auth-dev-feed.yml - parameters: - DevFeedName: ${{ parameters.DevFeedName }} - EnableTwineAuth: false - EnablePipAuth: true - EnableUvAuth: false - - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.10' - - - script: | - python -m pip install -r $(Pipeline.Workspace)/release_artifact/release_requirements.txt - displayName: Install Release Dependencies - - - ${{ if eq(parameters.PublicFeed, 'PyPi') }}: - - pwsh: | - $esrpDirectory = "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{artifact.name}}" - New-Item -ItemType Directory -Force -Path $esrpDirectory - - Get-ChildItem -Path "$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}" ` - | Where-Object { ($_.Name -like "*.tar.gz" -or $_.Name -like "*.whl") } ` - | Copy-Item -Destination $esrpDirectory - - Get-ChildItem $esrpDirectory - displayName: Isolate files for ESRP Publish - - - template: /eng/pipelines/templates/steps/esrp-publish.yml - parameters: - targetFolder: "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{artifact.name}}" - - - ${{ if ne(parameters.PublicFeed, 'PyPi') }}: - - task: TwineAuthenticate@0 - displayName: 'Authenticate to feed: ${{parameters.PublicFeed}}' - inputs: - artifactFeeds: ${{parameters.PublicFeed}} - - - script: | - set -e - twine upload --repository ${{parameters.PublicFeed}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.whl - echo "Uploaded whl to devops feed" - twine upload --repository ${{parameters.PublicFeed}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.tar.gz - echo "Uploaded sdist to devops feed" - displayName: 'Publish package to feed: ${{parameters.PublicFeed}}' - - - task: TwineAuthenticate@0 - displayName: 'Authenticate to feed: ${{parameters.DevFeedName}}' - inputs: - artifactFeeds: ${{parameters.DevFeedName}} - - - script: | - set -e - twine upload --repository ${{parameters.DevFeedName}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.whl - echo "Uploaded whl to devops feed" - twine upload --repository ${{parameters.DevFeedName}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.tar.gz - echo "Uploaded sdist to devops feed" - displayName: 'Publish package to feed: ${{parameters.DevFeedName}}' - - - job: MarkPackageReleaseCompletion - displayName: "Mark package release completion" - dependsOn: PublishPackage - - pool: - image: ubuntu-24.04 - name: azsdk-pool - os: linux - - steps: - - checkout: self - - - download: current - artifact: ${{parameters.ArtifactName}} - timeoutInMinutes: 5 - - - template: /eng/pipelines/templates/steps/auth-dev-feed.yml - parameters: - DevFeedName: ${{ parameters.DevFeedName }} - EnableTwineAuth: false - EnablePipAuth: true - EnableUvAuth: false - - - template: /eng/common/pipelines/templates/steps/mark-release-completion.yml - parameters: - ConfigFileDir: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo' - PackageArtifactName: ${{artifact.name}} - - # Management-plane-only workaround: skip marking packages that do not generate an APIView revision. - - pwsh: | - $packageInfoPath = '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{ artifact.name }}.json' - $packageInfo = Get-Content -Raw -Path $packageInfoPath | ConvertFrom-Json - $isManagementPackage = $packageInfo.SdkType -eq 'mgmt' - Write-Host "##vso[task.setvariable variable=IsManagementPackage]$isManagementPackage" - displayName: Check package SDK type - - - task: AzureCLI@2 - displayName: Mark Package Released - condition: >- - and( - succeeded(), - ne(variables['IsManagementPackage'], 'true'), - not( - and( - eq(variables['Skip.MarkPackageReleased'], 'true'), - eq(variables['IsRequesterAuthorizedToSkipApiReview'], 'true') - ) - ) - ) - inputs: - azureSubscription: "ADO to ARH Service Connection" - scriptType: pscore - scriptLocation: scriptPath - scriptPath: $(Build.SourcesDirectory)/eng/common/scripts/Mark-PackageReleased.ps1 - arguments: > - -PackageInfoFiles @('$(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{ artifact.name }}.json') - -RepoOwner 'Azure' - -AzSdkExePath '$(AZSDK)' - workingDirectory: $(Pipeline.Workspace) - - - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}: - - job: PublishGitHubIODocs - displayName: Publish Docs to GitHubIO Blob Storage - condition: >- - and( - succeeded(), - ne(variables['Skip.PublishDocs'], 'true'), - ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr') - ) - dependsOn: PublishPackage - - pool: - name: azsdk-pool - image: windows-2022 - os: windows - - steps: - - checkout: self - - - download: current - artifact: ${{parameters.ArtifactName}} - timeoutInMinutes: 5 - - - download: current - artifact: ${{parameters.DocArtifact}} - timeoutInMinutes: 5 - - - pwsh: | - if (Test-Path "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}") { - Get-ChildItem -Recurse "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}" - } - else { - New-Item -ItemType Directory -Force -Path "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}" - } - workingDirectory: $(Pipeline.Workspace) - displayName: Output Visible Artifacts - - - template: /eng/common/pipelines/templates/steps/publish-blobs.yml - parameters: - FolderForUpload: '$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}' - TargetLanguage: 'python' - ArtifactLocation: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}' - - - ${{if ne(artifact.skipPublishDocMs, 'true')}}: - - job: PublishDocs - displayName: Docs.MS Release - condition: >- - and( - succeeded(), - ne(variables['Skip.PublishDocs'], 'true'), - ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr') - ) - dependsOn: PublishPackage - - pool: - image: ubuntu-24.04 - name: azsdk-pool - os: linux - - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - sdk/**/*.md - - .github/CODEOWNERS - - - download: current - - # py2docfx requires Python >= 3.12.x, match docs pipeline version specification - - task: UsePythonVersion@0 - displayName: 'Use Python 3.12.x' - inputs: - versionSpec: '3.12.x' - - - template: /eng/pipelines/templates/steps/auth-dev-feed.yml - parameters: - DevFeedName: ${{ parameters.DevFeedName }} - - - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml - - - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml - parameters: - PackageInfoLocations: - - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json - WorkingDirectory: $(System.DefaultWorkingDirectory) - TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} - TargetDocRepoName: ${{parameters.TargetDocRepoName}} - Language: 'python' - SparseCheckoutPaths: - - docs-ref-services/ - - metadata/ - - - job: UpdatePackageVersion - displayName: "Update Package Version" - condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) - dependsOn: PublishPackage - - pool: - image: ubuntu-24.04 - name: azsdk-pool - os: linux - - steps: - - checkout: self - - task: UsePythonVersion@0 - - - template: /eng/pipelines/templates/steps/auth-dev-feed.yml - parameters: - DevFeedName: ${{ parameters.DevFeedName }} - - - script: | - python -m pip install "./eng/tools/azure-sdk-tools" - displayName: Install versioning tool dependencies - - - pwsh: | - sdk_increment_version --package-name ${{ artifact.name }} --service ${{ parameters.ServiceDirectory }} - if (Test-Path component-detection-pip-report.json) { - Write-Host "Deleting component-detection-pip-report.json" - rm component-detection-pip-report.json - } - displayName: Increment package version - - - template: /eng/common/pipelines/templates/steps/create-pull-request.yml - parameters: - RepoName: azure-sdk-for-python - PRBranchName: increment-package-version-${{ parameters.ServiceDirectory }}-$(Build.BuildId) - CommitMsg: "Increment package version after release of ${{ artifact.name }}" - PRTitle: "Increment version for ${{ parameters.ServiceDirectory }} releases" - CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' - AuthToken: '' - - - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: - - template: /eng/pipelines/templates/jobs/smoke.tests.yml - parameters: - Daily: false - ArtifactName: ${{ parameters.ArtifactName }} - Artifact: ${{ artifact }} - DevFeedName: ${{ parameters.DevFeedName }} + ArtifactName: ${{ parameters.ArtifactName }} + DependsOn: ${{ parameters.DependsOn }} + ServiceDirectory: ${{ parameters.ServiceDirectory }} + TestPipeline: ${{ parameters.TestPipeline }} + DocArtifact: ${{ parameters.DocArtifact }} + DevFeedName: ${{ parameters.DevFeedName }} + PublicFeed: ${{ parameters.PublicFeed }} + PublicPublishEnvironment: ${{ parameters.PublicPublishEnvironment }} + TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }} + TargetDocRepoName: ${{ parameters.TargetDocRepoName }} - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - stage: Integration - dependsOn: ${{parameters.DependsOn}} + # Signing stages are dependencies purely for ordering, so their signed artifacts exist + # before this stage downloads them. The condition still keys off the build stage only, + # exactly as before, so a signing failure degrades to skipping that one package's alpha + # publish rather than taking down the dev feed publish for the whole service. + # The gate below must stay identical to the signing emission gate at the top of this file. + dependsOn: + - ${{ parameters.DependsOn }} + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - ${{ each artifact in parameters.Artifacts }}: + - ${{ if eq(artifact.signBinaries, true) }}: + - Sign_${{ artifact.safeName }} condition: succeededOrFailed('${{parameters.DependsOn}}') jobs: - job: PublishPackages @@ -443,6 +112,16 @@ stages: artifact: ${{parameters.ArtifactName}} timeoutInMinutes: 5 + # Signed artifacts are downloaded best effort. If a Sign_* stage failed or was + # skipped the artifact is absent, and only that package's alpha publish is skipped. + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - ${{ each artifact in parameters.Artifacts }}: + - ${{ if eq(artifact.signBinaries, true) }}: + - download: current + artifact: packages_${{ artifact.safeName }}_signed + timeoutInMinutes: 5 + continueOnError: true + - task: UsePythonVersion@0 - template: ../steps/auth-dev-feed.yml @@ -456,33 +135,16 @@ stages: - ${{ each artifact in parameters.Artifacts }}: - ${{if ne(artifact.skipPublishDevFeed, 'true')}}: - - - pwsh: | - # If BuildTargetingString is set, check whether this artifact matches any of the - # (possibly comma-separated) glob patterns before attempting to publish. - # This handles scoped builds where only a subset of packages are built. - $targetingString = $env:BUILDTARGETINGSTRING - if ($targetingString) { - $globs = $targetingString -split "," - $isTargeted = $globs | Where-Object { "${{artifact.name}}" -like $_.Trim() } - if (-not $isTargeted) { - Write-Host "Package '${{artifact.name}}' does not match BuildTargetingString '$targetingString'. Skipping integration publish." - exit 0 - } - } - - $fileCount = (Get-ChildItem $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}} | ? {$_.Name -match "-[0-9]*.[0-9]*.[0-9]*a[0-9]*" } | Measure-Object).Count - - if ($fileCount -eq 0) { - Write-Host "No alpha packages for ${{artifact.name}} to publish." - exit 0 - } - - twine upload --repository $(DevFeedName) --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*-*a*.whl - echo "Uploaded whl to devops feed $(DevFeedName)" - twine upload --repository $(DevFeedName) --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*-*a*.tar.gz - echo "Uploaded sdist to devops feed $(DevFeedName)" - displayName: 'Publish ${{artifact.name}} alpha package' + - template: /eng/pipelines/templates/steps/publish-alpha-package.yml + parameters: + Artifact: ${{ artifact }} + # Packages that ship compiled binaries publish the wheels rebuilt by + # their Sign_ stage; everything else publishes straight from + # the build output, so a service can mix both kinds of packages freely. + ${{ if and(eq(artifact.signBinaries, true), ne(variables['Build.Reason'], 'PullRequest')) }}: + PackagePath: $(Pipeline.Workspace)/packages_${{ artifact.safeName }}_signed/${{ artifact.name }} + ${{ else }}: + PackagePath: $(Pipeline.Workspace)/${{ parameters.ArtifactName }}/${{ artifact.name }} - job: PublishDocsToNightlyBranch dependsOn: PublishPackages diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index f91eaf44c176..28b8ded76654 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -221,212 +221,14 @@ extends: - template: /eng/pipelines/templates/variables/image.yml - template: /eng/common/pipelines/templates/variables/api-review-break-glass.yml - - ${{ if eq(parameters.ServiceDirectory, 'storage') }}: - - stage: Sign_Binaries - displayName: Sign Extension Wheels - dependsOn: Build - condition: succeeded() - jobs: - - job: Sign_macOS - displayName: Sign macOS Wheels - pool: - name: $(LINUXPOOL) - image: $(LINUXVMIMAGE) - os: linux - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - - - task: UsePythonVersion@0 - displayName: "Use Python $(PythonVersion)" - inputs: - versionSpec: $(PythonVersion) - - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: packages_mac - targetPath: $(Build.ArtifactStagingDirectory)/packages_mac - - - pwsh: | - Get-ChildItem - python eng/scripts/wheel_signing/extract_sign_inputs.py ` - --platform mac ` - --wheels-dir "$(Build.ArtifactStagingDirectory)/packages_mac" ` - --work-dir "$(Build.ArtifactStagingDirectory)/mac-sign-work" ` - --sign-input-zip "$(Build.ArtifactStagingDirectory)/mac-sign-input.zip" - displayName: Extract mac wheel binaries - - - template: pipelines/steps/azd-cli-mac-signing.yml@azure-sdk-build-tools - parameters: - MacPath: "$(Build.ArtifactStagingDirectory)" - MacPattern: "mac-sign-input.zip" - Notarize: false - - - pwsh: | - python eng/scripts/wheel_signing/repackage_signed_wheels.py ` - --platform mac ` - --work-dir "$(Build.ArtifactStagingDirectory)/mac-sign-work" ` - --signed-input-zip "$(Build.ArtifactStagingDirectory)/mac-sign-input.zip" ` - --output-wheels-dir "$(Build.ArtifactStagingDirectory)/mac-wheels-signed" - displayName: Repackage mac wheels - - - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml - parameters: - ArtifactPath: '$(Build.ArtifactStagingDirectory)/mac-wheels-signed' - ArtifactName: 'packages_mac_signed' - - - job: Sign_Windows - displayName: Sign Windows Wheels - pool: - name: $(LINUXPOOL) - image: $(LINUXVMIMAGE) - os: linux - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - - - task: UsePythonVersion@0 - displayName: "Use Python $(PythonVersion)" - inputs: - versionSpec: $(PythonVersion) - - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: packages_windows - targetPath: $(Build.ArtifactStagingDirectory)/packages_windows - - - pwsh: | - python eng/scripts/wheel_signing/extract_sign_inputs.py ` - --platform windows ` - --wheels-dir "$(Build.ArtifactStagingDirectory)/packages_windows" ` - --work-dir "$(Build.ArtifactStagingDirectory)/win-sign-work" ` - --sign-input-dir "$(Build.ArtifactStagingDirectory)/win-sign-input" - displayName: Extract windows wheel binaries - - - template: pipelines/steps/azd-cli-win-signing.yml@azure-sdk-build-tools - parameters: - WinPath: "$(Build.ArtifactStagingDirectory)/win-sign-input" - WinPattern: '**/*.pyd' - - - pwsh: | - python eng/scripts/wheel_signing/repackage_signed_wheels.py ` - --platform windows ` - --work-dir "$(Build.ArtifactStagingDirectory)/win-sign-work" ` - --signed-input-dir "$(Build.ArtifactStagingDirectory)/win-sign-input" ` - --output-wheels-dir "$(Build.ArtifactStagingDirectory)/win-wheels-signed" - displayName: Repackage windows wheels - - - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml - parameters: - ArtifactPath: '$(Build.ArtifactStagingDirectory)/win-wheels-signed' - ArtifactName: 'packages_win_signed' - - - job: Build_Release_Artifact - displayName: Build release artifact - dependsOn: - - Sign_macOS - - Sign_Windows - pool: - name: $(LINUXPOOL) - image: $(LINUXVMIMAGE) - os: linux - steps: - - checkout: none - - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: packages_linux - targetPath: $(Build.ArtifactStagingDirectory)/packages_linux - - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: packages_extended - targetPath: $(Build.ArtifactStagingDirectory)/packages_extended - - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: packages_mac_signed - targetPath: $(Build.ArtifactStagingDirectory)/packages_mac - - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: packages_win_signed - targetPath: $(Build.ArtifactStagingDirectory)/packages_windows - - - pwsh: | - $releaseArtifactDir = "$(Build.ArtifactStagingDirectory)/release_packages" - $linuxPackagesDir = "$(Build.ArtifactStagingDirectory)/packages_linux" - $extendedPackagesDir = "$(Build.ArtifactStagingDirectory)/packages_extended" - New-Item -ItemType Directory -Path $releaseArtifactDir -Force | Out-Null - - Get-ChildItem $extendedPackagesDir | - Copy-Item -Destination $releaseArtifactDir -Recurse -Force - - function Copy-ArtifactPreservingRelativePath { - param( - [string]$artifactPath, - [string]$sourceRoot - ) - - $sourceDirectory = Split-Path $artifactPath -Parent - $relativeDirectory = $sourceDirectory.Substring($sourceRoot.Length).TrimStart('\', '/') - $targetDirectory = if ([string]::IsNullOrEmpty($relativeDirectory)) { - $releaseArtifactDir - } else { - Join-Path $releaseArtifactDir $relativeDirectory - } - - New-Item -ItemType Directory -Path $targetDirectory -Force | Out-Null - Copy-Item -Path $artifactPath -Destination $targetDirectory -Force - } - - function Copy-SignedWheelToPackageFolder { - param([string]$wheelPath) - - $wheelName = Split-Path $wheelPath -Leaf - if ($wheelName -notmatch '^(?.+?)-(?\d[^-]*)-') { - throw "Unable to determine the package folder for signed wheel '$wheelName'." - } - - $packageFolder = $matches['package'].Replace('_', '-') - $targetFolder = Join-Path $releaseArtifactDir $packageFolder - - if (-not (Test-Path $targetFolder)) { - throw "Expected package folder '$targetFolder' for signed wheel '$wheelName'." - } - - Copy-Item -Path $wheelPath -Destination $targetFolder -Force - } - - Get-ChildItem $linuxPackagesDir -Recurse -Filter "*.whl" | - ForEach-Object { Copy-ArtifactPreservingRelativePath $_.FullName $linuxPackagesDir } - - Get-ChildItem "$(Build.ArtifactStagingDirectory)/packages_mac" -Recurse -Filter "*.whl" | - ForEach-Object { Copy-SignedWheelToPackageFolder $_.FullName } - - Get-ChildItem "$(Build.ArtifactStagingDirectory)/packages_windows" -Recurse -Filter "*.whl" | - ForEach-Object { Copy-SignedWheelToPackageFolder $_.FullName } - displayName: Assemble release artifact - - - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml - parameters: - ArtifactPath: '$(Build.ArtifactStagingDirectory)/release_packages' - ArtifactName: 'packages_all_signed' - - variables: - - template: /eng/pipelines/templates/variables/globals.yml - - template: /eng/pipelines/templates/variables/image.yml - - template: archetype-python-release.yml parameters: - ${{ if eq(parameters.ServiceDirectory, 'storage') }}: - DependsOn: Sign_Binaries - ArtifactName: packages_all_signed - ${{ if ne(parameters.ServiceDirectory, 'storage') }}: - DependsOn: Build - ArtifactName: packages_extended + DependsOn: "Build" ServiceDirectory: ${{ parameters.ServiceDirectory }} Artifacts: ${{ parameters.Artifacts }} ${{ if eq(parameters.ServiceDirectory, 'template') }}: TestPipeline: true + ArtifactName: packages_extended DocArtifact: documentation TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }} TargetDocRepoName: ${{ parameters.TargetDocRepoName }} diff --git a/eng/pipelines/templates/stages/release-artifact.yml b/eng/pipelines/templates/stages/release-artifact.yml new file mode 100644 index 000000000000..1a678c644692 --- /dev/null +++ b/eng/pipelines/templates/stages/release-artifact.yml @@ -0,0 +1,426 @@ +# Emits the `Release_` stage for a single artifact. +# +# Extracted verbatim from archetype-python-release.yml so the release jobs keep exactly one +# definition. Artifacts that ship compiled binaries release from a different pipeline artifact +# (`packages__signed`) and depend on a different upstream stage (`Sign_`) +# than everything else. Parameterizing just those two values here avoids forking the release +# job definitions into signed and unsigned copies that would inevitably drift. +# +# Callers resolve ArtifactName and DependsOn per artifact; every job below is otherwise +# unchanged from the shared release workflow. + +parameters: + - name: Artifact + type: object + # Pipeline artifact this package releases from. `packages_extended` for ordinary packages, + # `packages__signed` for packages that opted in with `signBinaries: true`. + - name: ArtifactName + type: string + # Upstream stage. The build stage for ordinary packages, `Sign_` when signing. + - name: DependsOn + type: string + - name: ServiceDirectory + type: string + default: 'not-specified' + - name: TestPipeline + type: boolean + default: false + - name: DocArtifact + type: string + default: 'documentation' + - name: DevFeedName + type: string + default: 'public/azure-sdk-for-python' + - name: PublicFeed + type: string + default: PyPi + - name: PublicPublishEnvironment + type: string + default: package-publish + - name: TargetDocRepoOwner + type: string + default: '' + - name: TargetDocRepoName + type: string + default: '' + +stages: + - stage: 'Release_${{parameters.Artifact.safename}}' + displayName: 'Release: ${{parameters.Artifact.name}}' + dependsOn: + - ${{ parameters.DependsOn }} + - ${{ if and(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: + - AutoReleasePrepare + variables: + - template: /eng/pipelines/templates/variables/image.yml + - template: /eng/common/pipelines/templates/variables/api-review-break-glass.yml + # Auto-release CI: only release when the shared prepare stage flagged this artifact as changed. + # Manual runs: every declared artifact remains eligible (existing behavior). + ${{ if and(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: + condition: and(succeeded(), eq(dependencies.AutoReleasePrepare.outputs['ResolveAutoReleasePackages.resolve.ReleaseArtifact_${{ parameters.Artifact.safename }}'], 'true'), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr')) + ${{ else }}: + condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr')) + jobs: + - job: TagRepository + displayName: "Create release tag" + condition: and(succeeded(), ne(variables['Skip.TagRepository'], 'true')) + + pool: + image: ubuntu-24.04 + name: azsdk-pool + os: linux + + steps: + - checkout: self + + - download: current + artifact: ${{parameters.ArtifactName}} + timeoutInMinutes: 5 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.12' + + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + + - template: /eng/common/pipelines/templates/steps/retain-run.yml + + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + parameters: + PackageName: "azure-template" + ServiceDirectory: "template" + TestPipeline: ${{ parameters.TestPipeline }} + + - template: /eng/common/pipelines/templates/steps/verify-changelog.yml + parameters: + PackageName: ${{parameters.Artifact.name}} + ServiceName: ${{parameters.ServiceDirectory}} + ForRelease: true + + - template: /eng/common/pipelines/templates/steps/verify-restapi-spec-location.yml + parameters: + PackageName: ${{parameters.Artifact.name}} + ServiceDirectory: ${{parameters.ServiceDirectory}} + ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}} + + - script: | + python -m pip install "./eng/tools/azure-sdk-tools" + displayName: Install tool dependencies + + - task: PythonScript@0 + displayName: Verify Dependency Presence + condition: and(succeeded(), ne(variables['Skip.VerifyDependencies'], 'true')) + inputs: + scriptPath: 'scripts/devops_tasks/verify_dependencies_present.py' + arguments: '--package-name ${{ parameters.Artifact.name }} --service ${{ parameters.ServiceDirectory }}' + + - task: PythonScript@0 + displayName: Verify CI enabled + condition: succeeded() + inputs: + scriptPath: 'scripts/devops_tasks/verify_ci_enabled.py' + arguments: '--package-name ${{ parameters.Artifact.name }} --service ${{ parameters.ServiceDirectory }}' + + - pwsh: | + Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}} + workingDirectory: $(Pipeline.Workspace) + displayName: Output Visible Artifacts + + - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml + parameters: + ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}} + PackageRepository: PyPI + ReleaseSha: $(Build.SourceVersion) + RepoId: Azure/azure-sdk-for-python + WorkingDirectory: $(System.DefaultWorkingDirectory) + AuthToken: '' + + - ${{if ne(parameters.Artifact.skipPublishPackage, 'true')}}: + - deployment: PublishPackage + displayName: "Publish to ${{ parameters.PublicFeed }}" + condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true')) + # Auto-release runs after merge on main and must not wait on the package-publish approval gate. + ${{ if and(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: + environment: none + ${{ else }}: + environment: ${{ parameters.PublicPublishEnvironment }} + dependsOn: TagRepository + + templateContext: + type: releaseJob + isProduction: true + inputs: + - input: pipelineArtifact + artifactName: release_artifact + targetPath: $(Pipeline.Workspace)/release_artifact + - input: pipelineArtifact + artifactName: packages_extended + targetPath: $(Pipeline.Workspace)/packages_extended + + pool: + image: ubuntu-24.04 + name: azsdk-pool + os: linux + + strategy: + runOnce: + deploy: + steps: + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + EnableTwineAuth: false + EnablePipAuth: true + EnableUvAuth: false + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.10' + + - script: | + python -m pip install -r $(Pipeline.Workspace)/release_artifact/release_requirements.txt + displayName: Install Release Dependencies + + - ${{ if eq(parameters.PublicFeed, 'PyPi') }}: + - pwsh: | + $esrpDirectory = "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}" + New-Item -ItemType Directory -Force -Path $esrpDirectory + + Get-ChildItem -Path "$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}" ` + | Where-Object { ($_.Name -like "*.tar.gz" -or $_.Name -like "*.whl") } ` + | Copy-Item -Destination $esrpDirectory + + Get-ChildItem $esrpDirectory + displayName: Isolate files for ESRP Publish + + - template: /eng/pipelines/templates/steps/esrp-publish.yml + parameters: + targetFolder: "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}" + + - ${{ if ne(parameters.PublicFeed, 'PyPi') }}: + - task: TwineAuthenticate@0 + displayName: 'Authenticate to feed: ${{parameters.PublicFeed}}' + inputs: + artifactFeeds: ${{parameters.PublicFeed}} + + - script: | + set -e + twine upload --repository ${{parameters.PublicFeed}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}/*.whl + echo "Uploaded whl to devops feed" + twine upload --repository ${{parameters.PublicFeed}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}/*.tar.gz + echo "Uploaded sdist to devops feed" + displayName: 'Publish package to feed: ${{parameters.PublicFeed}}' + + - task: TwineAuthenticate@0 + displayName: 'Authenticate to feed: ${{parameters.DevFeedName}}' + inputs: + artifactFeeds: ${{parameters.DevFeedName}} + + - script: | + set -e + twine upload --repository ${{parameters.DevFeedName}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}/*.whl + echo "Uploaded whl to devops feed" + twine upload --repository ${{parameters.DevFeedName}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}/*.tar.gz + echo "Uploaded sdist to devops feed" + displayName: 'Publish package to feed: ${{parameters.DevFeedName}}' + + - job: MarkPackageReleaseCompletion + displayName: "Mark package release completion" + dependsOn: PublishPackage + + pool: + image: ubuntu-24.04 + name: azsdk-pool + os: linux + + steps: + - checkout: self + + - download: current + artifact: ${{parameters.ArtifactName}} + timeoutInMinutes: 5 + + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + EnableTwineAuth: false + EnablePipAuth: true + EnableUvAuth: false + + - template: /eng/common/pipelines/templates/steps/mark-release-completion.yml + parameters: + ConfigFileDir: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo' + PackageArtifactName: ${{parameters.Artifact.name}} + + # Management-plane-only workaround: skip marking packages that do not generate an APIView revision. + - pwsh: | + $packageInfoPath = '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{ parameters.Artifact.name }}.json' + $packageInfo = Get-Content -Raw -Path $packageInfoPath | ConvertFrom-Json + $isManagementPackage = $packageInfo.SdkType -eq 'mgmt' + Write-Host "##vso[task.setvariable variable=IsManagementPackage]$isManagementPackage" + displayName: Check package SDK type + + - task: AzureCLI@2 + displayName: Mark Package Released + condition: >- + and( + succeeded(), + ne(variables['IsManagementPackage'], 'true'), + not( + and( + eq(variables['Skip.MarkPackageReleased'], 'true'), + eq(variables['IsRequesterAuthorizedToSkipApiReview'], 'true') + ) + ) + ) + inputs: + azureSubscription: "ADO to ARH Service Connection" + scriptType: pscore + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/scripts/Mark-PackageReleased.ps1 + arguments: > + -PackageInfoFiles @('$(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{ parameters.Artifact.name }}.json') + -RepoOwner 'Azure' + -AzSdkExePath '$(AZSDK)' + workingDirectory: $(Pipeline.Workspace) + + - ${{if ne(parameters.Artifact.skipPublishDocGithubIo, 'true')}}: + - job: PublishGitHubIODocs + displayName: Publish Docs to GitHubIO Blob Storage + condition: >- + and( + succeeded(), + ne(variables['Skip.PublishDocs'], 'true'), + ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr') + ) + dependsOn: PublishPackage + + pool: + name: azsdk-pool + image: windows-2022 + os: windows + + steps: + - checkout: self + + - download: current + artifact: ${{parameters.ArtifactName}} + timeoutInMinutes: 5 + + - download: current + artifact: ${{parameters.DocArtifact}} + timeoutInMinutes: 5 + + - pwsh: | + if (Test-Path "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{parameters.Artifact.name}}") { + Get-ChildItem -Recurse "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{parameters.Artifact.name}}" + } + else { + New-Item -ItemType Directory -Force -Path "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{parameters.Artifact.name}}" + } + workingDirectory: $(Pipeline.Workspace) + displayName: Output Visible Artifacts + + - template: /eng/common/pipelines/templates/steps/publish-blobs.yml + parameters: + FolderForUpload: '$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{parameters.Artifact.name}}' + TargetLanguage: 'python' + ArtifactLocation: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}' + + - ${{if ne(parameters.Artifact.skipPublishDocMs, 'true')}}: + - job: PublishDocs + displayName: Docs.MS Release + condition: >- + and( + succeeded(), + ne(variables['Skip.PublishDocs'], 'true'), + ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr') + ) + dependsOn: PublishPackage + + pool: + image: ubuntu-24.04 + name: azsdk-pool + os: linux + + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - sdk/**/*.md + - .github/CODEOWNERS + + - download: current + + # py2docfx requires Python >= 3.12.x, match docs pipeline version specification + - task: UsePythonVersion@0 + displayName: 'Use Python 3.12.x' + inputs: + versionSpec: '3.12.x' + + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml + parameters: + PackageInfoLocations: + - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{parameters.Artifact.name}}.json + WorkingDirectory: $(System.DefaultWorkingDirectory) + TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} + TargetDocRepoName: ${{parameters.TargetDocRepoName}} + Language: 'python' + SparseCheckoutPaths: + - docs-ref-services/ + - metadata/ + + - job: UpdatePackageVersion + displayName: "Update Package Version" + condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) + dependsOn: PublishPackage + + pool: + image: ubuntu-24.04 + name: azsdk-pool + os: linux + + steps: + - checkout: self + - task: UsePythonVersion@0 + + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + + - script: | + python -m pip install "./eng/tools/azure-sdk-tools" + displayName: Install versioning tool dependencies + + - pwsh: | + sdk_increment_version --package-name ${{ parameters.Artifact.name }} --service ${{ parameters.ServiceDirectory }} + if (Test-Path component-detection-pip-report.json) { + Write-Host "Deleting component-detection-pip-report.json" + rm component-detection-pip-report.json + } + displayName: Increment package version + + - template: /eng/common/pipelines/templates/steps/create-pull-request.yml + parameters: + RepoName: azure-sdk-for-python + PRBranchName: increment-package-version-${{ parameters.ServiceDirectory }}-$(Build.BuildId) + CommitMsg: "Increment package version after release of ${{ parameters.Artifact.name }}" + PRTitle: "Increment version for ${{ parameters.ServiceDirectory }} releases" + CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' + AuthToken: '' + + - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: + - template: /eng/pipelines/templates/jobs/smoke.tests.yml + parameters: + Daily: false + ArtifactName: ${{ parameters.ArtifactName }} + Artifact: ${{ parameters.Artifact }} + DevFeedName: ${{ parameters.DevFeedName }} diff --git a/eng/pipelines/templates/stages/sign-binaries.yml b/eng/pipelines/templates/stages/sign-binaries.yml new file mode 100644 index 000000000000..a6d87de8ea76 --- /dev/null +++ b/eng/pipelines/templates/stages/sign-binaries.yml @@ -0,0 +1,232 @@ +# Emits a single `Sign_` stage for one artifact that ships compiled binaries. +# +# Packages opt in from their service ci.yml by setting `signBinaries: true` on the artifact. +# This stage is emitted by archetype-python-release.yml, which owns the gating that decides +# when signing runs (release + scheduled builds, never PR builds). +# +# The mac and windows wheels built by the Build stage contain unsigned binaries. This stage +# unpacks only this artifact's wheels, sends the binaries through ESRP, rebuilds the wheels, +# and republishes them as `packages__signed`. +# +# The published artifact deliberately mirrors the layout of `packages_extended`: +# +# packages__signed/ +# PackageInfo/.json +# /*.whl, *.tar.gz, apistub tokens +# +# so that every downstream release job works against it unchanged, just by swapping the +# artifact name. `packages_extended` itself is left untouched and still contains the +# unsigned wheels for this package alongside every other package in the service. + +parameters: + - name: Artifact + type: object + - name: DependsOn + type: string + default: Build + +stages: + - stage: Sign_${{ parameters.Artifact.safeName }} + displayName: 'Sign: ${{ parameters.Artifact.name }}' + dependsOn: ${{ parameters.DependsOn }} + # Intentionally does not check Skip.Release or SetDevVersion. Signing must run whenever + # *either* consumer might need the signed wheels: the Release_ stage or the + # Integration dev feed publish. Skip logic belongs on those consumers, which already carry + # it. Adding it here would propagate through Integration's dependency and silently break + # the nightly and manual dev version alpha publishes. + # + # Run reasons are filtered here rather than at compile time so the emission gate in + # archetype-python-release.yml can stay short enough to repeat verbatim wherever a + # Sign_* stage is referenced. Release builds are manual runs and auto-release CI on main; + # scheduled builds sign the alpha packages that the Integration stage publishes. + condition: >- + and( + succeeded(), + ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr'), + or( + in(variables['Build.Reason'], 'Manual', '', 'Schedule'), + and( + eq(variables['Build.Reason'], 'IndividualCI'), + eq(variables['Build.SourceBranch'], 'refs/heads/main') + ) + ) + ) + + variables: + - template: /eng/pipelines/templates/variables/globals.yml + - template: /eng/pipelines/templates/variables/image.yml + + jobs: + - job: Sign_macOS + displayName: Sign macOS Wheels + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + steps: + # The default sparse checkout always includes /eng, which is all these jobs need. + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - task: UsePythonVersion@0 + displayName: "Use Python $(PythonVersion)" + inputs: + versionSpec: $(PythonVersion) + + - task: DownloadPipelineArtifact@2 + displayName: Download unsigned mac wheels + inputs: + artifactName: packages_mac + # Only this artifact's wheels. Other packages in the service are untouched. + itemPattern: '${{ parameters.Artifact.name }}/**' + targetPath: $(Build.ArtifactStagingDirectory)/packages_mac + + - pwsh: | + python eng/scripts/wheel_signing/extract_sign_inputs.py ` + --platform mac ` + --wheels-dir "$(Build.ArtifactStagingDirectory)/packages_mac/${{ parameters.Artifact.name }}" ` + --work-dir "$(Build.ArtifactStagingDirectory)/mac-sign-work" ` + --sign-input-zip "$(Build.ArtifactStagingDirectory)/mac-sign-input.zip" + displayName: Extract mac wheel binaries + + - template: pipelines/steps/azd-cli-mac-signing.yml@azure-sdk-build-tools + parameters: + MacPath: "$(Build.ArtifactStagingDirectory)" + MacPattern: "mac-sign-input.zip" + Notarize: false + + - pwsh: | + python eng/scripts/wheel_signing/repackage_signed_wheels.py ` + --platform mac ` + --work-dir "$(Build.ArtifactStagingDirectory)/mac-sign-work" ` + --signed-input-zip "$(Build.ArtifactStagingDirectory)/mac-sign-input.zip" ` + --output-wheels-dir "$(Build.ArtifactStagingDirectory)/mac-wheels-signed" + displayName: Repackage mac wheels + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: '$(Build.ArtifactStagingDirectory)/mac-wheels-signed' + ArtifactName: 'packages_${{ parameters.Artifact.safeName }}_mac_signed' + + - job: Sign_Windows + displayName: Sign Windows Wheels + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + steps: + # The default sparse checkout always includes /eng, which is all these jobs need. + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - task: UsePythonVersion@0 + displayName: "Use Python $(PythonVersion)" + inputs: + versionSpec: $(PythonVersion) + + - task: DownloadPipelineArtifact@2 + displayName: Download unsigned windows wheels + inputs: + artifactName: packages_windows + itemPattern: '${{ parameters.Artifact.name }}/**' + targetPath: $(Build.ArtifactStagingDirectory)/packages_windows + + - pwsh: | + python eng/scripts/wheel_signing/extract_sign_inputs.py ` + --platform windows ` + --wheels-dir "$(Build.ArtifactStagingDirectory)/packages_windows/${{ parameters.Artifact.name }}" ` + --work-dir "$(Build.ArtifactStagingDirectory)/win-sign-work" ` + --sign-input-dir "$(Build.ArtifactStagingDirectory)/win-sign-input" + displayName: Extract windows wheel binaries + + - template: pipelines/steps/azd-cli-win-signing.yml@azure-sdk-build-tools + parameters: + WinPath: "$(Build.ArtifactStagingDirectory)/win-sign-input" + WinPattern: '**/*.pyd' + + - pwsh: | + python eng/scripts/wheel_signing/repackage_signed_wheels.py ` + --platform windows ` + --work-dir "$(Build.ArtifactStagingDirectory)/win-sign-work" ` + --signed-input-dir "$(Build.ArtifactStagingDirectory)/win-sign-input" ` + --output-wheels-dir "$(Build.ArtifactStagingDirectory)/win-wheels-signed" + displayName: Repackage windows wheels + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: '$(Build.ArtifactStagingDirectory)/win-wheels-signed' + ArtifactName: 'packages_${{ parameters.Artifact.safeName }}_win_signed' + + - job: Assemble + displayName: Assemble signed artifact + dependsOn: + - Sign_macOS + - Sign_Windows + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + steps: + - checkout: none + + # packages_extended is the merged output of the three platform builds plus the + # apistub tokens and PackageInfo. Take this package's slice of it as the base, + # then overwrite the mac and windows wheels with their signed replacements. + - task: DownloadPipelineArtifact@2 + displayName: Download package files + inputs: + artifactName: packages_extended + itemPattern: | + ${{ parameters.Artifact.name }}/** + PackageInfo/${{ parameters.Artifact.name }}.json + targetPath: $(Build.ArtifactStagingDirectory)/signed + + - task: DownloadPipelineArtifact@2 + displayName: Download signed mac wheels + inputs: + artifactName: packages_${{ parameters.Artifact.safeName }}_mac_signed + targetPath: $(Build.ArtifactStagingDirectory)/mac_signed + + - task: DownloadPipelineArtifact@2 + displayName: Download signed windows wheels + inputs: + artifactName: packages_${{ parameters.Artifact.safeName }}_win_signed + targetPath: $(Build.ArtifactStagingDirectory)/win_signed + + - pwsh: | + $ErrorActionPreference = 'Stop' + + $packageDir = "$(Build.ArtifactStagingDirectory)/signed/${{ parameters.Artifact.name }}" + $packageInfo = "$(Build.ArtifactStagingDirectory)/signed/PackageInfo/${{ parameters.Artifact.name }}.json" + + if (-not (Test-Path $packageDir)) { + throw "Expected package folder '$packageDir' in packages_extended." + } + if (-not (Test-Path $packageInfo)) { + throw "Expected package info file '$packageInfo' in packages_extended." + } + + $signedWheels = @( + Get-ChildItem "$(Build.ArtifactStagingDirectory)/mac_signed" -Recurse -Filter "*.whl" + Get-ChildItem "$(Build.ArtifactStagingDirectory)/win_signed" -Recurse -Filter "*.whl" + ) + + if (-not $signedWheels) { + throw "No signed wheels were produced for ${{ parameters.Artifact.name }}." + } + + foreach ($wheel in $signedWheels) { + $target = Join-Path $packageDir $wheel.Name + if (-not (Test-Path $target)) { + throw "Signed wheel '$($wheel.Name)' has no unsigned counterpart in packages_extended. The signed artifact would not match the build output." + } + Write-Host "Replacing $($wheel.Name) with its signed build." + Copy-Item -Path $wheel.FullName -Destination $target -Force + } + + Write-Host "`nFinal contents of the signed artifact:" + Get-ChildItem -Recurse "$(Build.ArtifactStagingDirectory)/signed" | Select-Object -ExpandProperty FullName + displayName: Overlay signed wheels + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: '$(Build.ArtifactStagingDirectory)/signed' + ArtifactName: 'packages_${{ parameters.Artifact.safeName }}_signed' diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index 6c8725ab7174..6597389cd56d 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -94,8 +94,7 @@ steps: --mark_arg="$markArg" --service="${{ parameters.ServiceDirectory }}" --checks="${{ parameters.CheckEnv }}" - --injected-packages="${{ parameters.InjectedPackages }}" - --max-parallel=1; + --injected-packages="${{ parameters.InjectedPackages }}"; Write-Host "Last exit code: $LASTEXITCODE"; exit $LASTEXITCODE; @@ -117,8 +116,7 @@ steps: --mark_arg="$markArg" ` --service="${{ parameters.ServiceDirectory }}" ` --checks="${{ parameters.CheckEnv }}" ` - --injected-packages="${{ parameters.InjectedPackages }}" ` - --max-parallel=1; + --injected-packages="${{ parameters.InjectedPackages }}"; exit $LASTEXITCODE; env: ${{ parameters.EnvVars }} displayName: Run Tests diff --git a/eng/pipelines/templates/steps/publish-alpha-package.yml b/eng/pipelines/templates/steps/publish-alpha-package.yml new file mode 100644 index 000000000000..b30351aa6ae3 --- /dev/null +++ b/eng/pipelines/templates/steps/publish-alpha-package.yml @@ -0,0 +1,47 @@ +# Publishes one package's alpha (daily dev build) wheel and sdist to the dev feed. +# +# Extracted from the Integration stage in archetype-python-release.yml so the package path can +# vary per artifact. Packages built with `signBinaries: true` publish from their signed +# artifact; everything else publishes from `packages_extended`. + +parameters: + - name: Artifact + type: object + # Folder holding this package's built files, already resolved by the caller. + - name: PackagePath + type: string + +steps: + - pwsh: | + # The signed artifact is absent when a Sign_* stage failed or was skipped. Skip this one + # package rather than failing the dev feed publish for every other package in the service. + if (-not (Test-Path "${{ parameters.PackagePath }}")) { + Write-Warning "No build output at '${{ parameters.PackagePath }}'. Skipping alpha publish for ${{ parameters.Artifact.name }}." + exit 0 + } + + # If BuildTargetingString is set, check whether this artifact matches any of the + # (possibly comma-separated) glob patterns before attempting to publish. + # This handles scoped builds where only a subset of packages are built. + $targetingString = $env:BUILDTARGETINGSTRING + if ($targetingString) { + $globs = $targetingString -split "," + $isTargeted = $globs | Where-Object { "${{ parameters.Artifact.name }}" -like $_.Trim() } + if (-not $isTargeted) { + Write-Host "Package '${{ parameters.Artifact.name }}' does not match BuildTargetingString '$targetingString'. Skipping integration publish." + exit 0 + } + } + + $fileCount = (Get-ChildItem ${{ parameters.PackagePath }} | ? {$_.Name -match "-[0-9]*.[0-9]*.[0-9]*a[0-9]*" } | Measure-Object).Count + + if ($fileCount -eq 0) { + Write-Host "No alpha packages for ${{ parameters.Artifact.name }} to publish." + exit 0 + } + + twine upload --repository $(DevFeedName) --config-file $(PYPIRC_PATH) ${{ parameters.PackagePath }}/*-*a*.whl + echo "Uploaded whl to devops feed $(DevFeedName)" + twine upload --repository $(DevFeedName) --config-file $(PYPIRC_PATH) ${{ parameters.PackagePath }}/*-*a*.tar.gz + echo "Uploaded sdist to devops feed $(DevFeedName)" + displayName: 'Publish ${{ parameters.Artifact.name }} alpha package' diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index 64f567dff463..da3d9c40a91c 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -37,25 +37,27 @@ extends: ${{ if eq(parameters.ReleaseToDevOpsOnly, 'true') }}: PublicFeed: 'public/storage-staging' ServiceDirectory: storage - BuildTargetingString: azure-storage-extensions TestProxy: true TestTimeoutInMinutes: 120 # Enable Compiled CodeQL because azure-storage-extensions has C code that # must be scanned EnableCompiledCodeql: true Artifacts: - # - name: azure-storage-blob - # safeName: azurestorageblob - # - name: azure-storage-blob-changefeed - # safeName: azurestorageblobchangefeed - # - name: azure-storage-file-share - # safeName: azurestoragefileshare - # - name: azure-storage-file-datalake - # safeName: azurestoragefiledatalake - # - name: azure-storage-queue - # safeName: azurestoragequeue + - name: azure-storage-blob + safeName: azurestorageblob + - name: azure-storage-blob-changefeed + safeName: azurestorageblobchangefeed + - name: azure-storage-file-share + safeName: azurestoragefileshare + - name: azure-storage-file-datalake + safeName: azurestoragefiledatalake + - name: azure-storage-queue + safeName: azurestoragequeue - name: azure-storage-extensions safeName: azurestorageextensions + # Ships compiled binaries, so its wheels are routed through a dedicated + # Sign_azurestorageextensions stage before release and dev feed publishing. + signBinaries: true triggeringPaths: - /sdk/storage/azure-storage-blob - /sdk/storage/azure-storage-file-datalake @@ -64,11 +66,11 @@ extends: # Pure C-based storage extension package, not generating docs at this moment. skipPublishDocGithubIo: true skipPublishDocMs: true - # - name: azure-mgmt-storage - # safeName: azuremgmtstorage - # - name: azure-mgmt-storagecache - # safeName: azuremgmtstoragecache - # - name: azure-mgmt-storagesync - # safeName: azuremgmtstoragesync - # - name: azure-mgmt-storageimportexport - # safeName: azuremgmtstorageimportexport + - name: azure-mgmt-storage + safeName: azuremgmtstorage + - name: azure-mgmt-storagecache + safeName: azuremgmtstoragecache + - name: azure-mgmt-storagesync + safeName: azuremgmtstoragesync + - name: azure-mgmt-storageimportexport + safeName: azuremgmtstorageimportexport From e3e91f304fd295c498b70c243407d1ff63c78897 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 10 Sep 2026 14:13:31 -0700 Subject: [PATCH 20/21] Surface skipped alpha publishes and pin the signing run reasons Testing the scheduled path directly, rather than simulating it with a queue-time SetDevVersion, showed the signing condition was only ever being satisfied by its 'Manual' term. Two gaps that hid behind that: The 'Schedule' term is coupled to daily-dev-build-variable.yml, which sets SetDevVersion only when Build.Reason is exactly 'Schedule'. Alpha wheels exist only when that is true, so the two must widen together. Record that, along with why the reasons left out of the list need no signing. A missing signed artifact skips one package's alpha publish by design, so a signing failure cannot take down the dev feed publish for a whole service. Raise it as a build issue rather than Write-Warning so that degradation shows up in the run summary instead of only in the task log. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9079636e-8478-4697-ae82-538136cd41b5 --- eng/pipelines/templates/stages/sign-binaries.yml | 8 ++++++++ eng/pipelines/templates/steps/publish-alpha-package.yml | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/sign-binaries.yml b/eng/pipelines/templates/stages/sign-binaries.yml index a6d87de8ea76..498c467aa0fd 100644 --- a/eng/pipelines/templates/stages/sign-binaries.yml +++ b/eng/pipelines/templates/stages/sign-binaries.yml @@ -39,6 +39,14 @@ stages: # archetype-python-release.yml can stay short enough to repeat verbatim wherever a # Sign_* stage is referenced. Release builds are manual runs and auto-release CI on main; # scheduled builds sign the alpha packages that the Integration stage publishes. + # + # The 'Schedule' term is load bearing and coupled to daily-dev-build-variable.yml, which + # sets SetDevVersion=true only when Build.Reason is exactly 'Schedule'. Alpha wheels exist + # only when that is true, so Integration needs signed wheels under exactly the same reasons. + # If that check ever widens (for example to ScheduleForced), widen this list to match or + # scheduled signing silently stops and the alpha publish falls back to skipping the package. + # Reasons absent from this list (BatchedCI, ScheduleForced, IndividualCI off main) produce + # neither alpha wheels nor a release stage, so skipping signing for them is correct. condition: >- and( succeeded(), diff --git a/eng/pipelines/templates/steps/publish-alpha-package.yml b/eng/pipelines/templates/steps/publish-alpha-package.yml index b30351aa6ae3..dc246652c884 100644 --- a/eng/pipelines/templates/steps/publish-alpha-package.yml +++ b/eng/pipelines/templates/steps/publish-alpha-package.yml @@ -15,8 +15,10 @@ steps: - pwsh: | # The signed artifact is absent when a Sign_* stage failed or was skipped. Skip this one # package rather than failing the dev feed publish for every other package in the service. + # Logged as a build issue, not just Write-Warning, so a signing failure degrading to + # "this package was not published" is visible in the run summary instead of buried in logs. if (-not (Test-Path "${{ parameters.PackagePath }}")) { - Write-Warning "No build output at '${{ parameters.PackagePath }}'. Skipping alpha publish for ${{ parameters.Artifact.name }}." + Write-Host "##vso[task.logissue type=warning]No build output at '${{ parameters.PackagePath }}'. Skipping alpha publish for ${{ parameters.Artifact.name }}. If this package sets signBinaries, check whether its Sign stage failed or was skipped." exit 0 } From c6af88bfe3f65539dfc580e5e972c223b89d68cc Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 10 Sep 2026 21:01:09 -0700 Subject: [PATCH 21/21] Derive platform builds from the signBinaries artifact property ENABLE_EXTENSION_BUILD gated the mac and windows builds on a hardcoded azure-storage-extensions name check, so any other package opting into signBinaries got a signing stage with no wheels to sign. Resolve it from the artifact property instead, which Save-Package-Properties already copies into each package info file as ArtifactDetails. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9079636e-8478-4697-ae82-538136cd41b5 --- .../steps/build-package-artifacts.yml | 4 +-- .../steps/resolve-build-platforms.yml | 31 ++++++++++++++----- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 5f026f19ded9..a4bf41dba5d1 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -91,8 +91,8 @@ steps: BuildTargetingString: ${{ parameters.BuildTargetingString }} PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo - # todo, walk the artifacts and ensure that one which includes an extension package is present - # if not, we only need to build on linux. if so, we need to build on all platforms + # Decides whether the mac/windows builds produce anything, based on whether any targeted package + # declares compiled binaries. If none do, only the linux build runs. - template: /eng/pipelines/templates/steps/resolve-build-platforms.yml parameters: PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo diff --git a/eng/pipelines/templates/steps/resolve-build-platforms.yml b/eng/pipelines/templates/steps/resolve-build-platforms.yml index 43db2e725352..2de46faf2585 100644 --- a/eng/pipelines/templates/steps/resolve-build-platforms.yml +++ b/eng/pipelines/templates/steps/resolve-build-platforms.yml @@ -4,15 +4,32 @@ parameters: default: '' steps: - # when we merge pipeline v3, this check will change to examining the targeting string $(TargetingString) - # as the generate-pr-diff call + resolution will be present in resolve-package-targeting.yml. - # until then, we simply check to see if we're targeting storage service directory + # Packages that ship compiled binaries opt in with `signBinaries: true` on their artifact entry in + # the service ci.yml. Save-Package-Properties copies that artifact entry verbatim into each package + # info file as `ArtifactDetails`, so the opt-in is readable here without threading the artifact list + # through every caller. + # + # resolve-package-targeting.yml runs immediately before this and deletes the package info files that + # this run is not targeting, so the folder is already narrowed to the packages being built; on 'auto' + # pull request builds save-package-properties.yml narrows it to the PR diff first. This deliberately + # keeps reading the folder rather than the $(TargetingString) those steps also set, because that + # variable carries only package names and the decision below needs an artifact property. - pwsh: | - $packageProperties = Get-ChildItem -Recurse -Force "${{ parameters.PackagePropertiesFolder }}/*.json" ` - | ForEach-Object { $_.Name.Replace(".json", "") } + $binaryPackages = @() - if ($packageProperties -contains "azure-storage-extensions") { - Write-Host "Targeting storage, enabling extension build." + foreach ($packageInfoPath in (Get-ChildItem -Recurse -Force "${{ parameters.PackagePropertiesFolder }}/*.json")) { + $packageInfo = Get-Content -Raw -Path $packageInfoPath.FullName | ConvertFrom-Json + + if ($packageInfo.ArtifactDetails.signBinaries -eq $true) { + $binaryPackages += $packageInfo.Name + } + } + + if ($binaryPackages) { + Write-Host "Targeting package(s) with compiled binaries ($($binaryPackages -join ', ')), enabling extension build." Write-Host "##vso[task.setvariable variable=ENABLE_EXTENSION_BUILD]true" } + else { + Write-Host "No targeted package declares compiled binaries, building on linux only." + } displayName: Check extension package presence