diff --git a/eng/pipelines/templates/stages/1es-redirect.yml b/eng/pipelines/templates/stages/1es-redirect.yml index e40adb4c54b0..b5ec95d13592 100644 --- a/eng/pipelines/templates/stages/1es-redirect.yml +++ b/eng/pipelines/templates/stages/1es-redirect.yml @@ -8,6 +8,10 @@ resources: type: git name: 1ESPipelineTemplates/1ESPipelineTemplates ref: refs/tags/canary + - repository: azure-sdk-build-tools + type: git + name: internal/azure-sdk-build-tools + ref: refs/tags/azure-sdk-build-tools_20260702.2 parameters: - name: stages diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml index 5357beb17f49..7f58eaa8880f 100644 --- a/eng/pipelines/templates/stages/archetype-python-release.yml +++ b/eng/pipelines/templates/stages/archetype-python-release.yml @@ -13,6 +13,24 @@ parameters: PackageSourceOverride: "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" stages: + # Binary signing stages, emitted per artifact that opted in with `signBinaries: true`. + # + # INVARIANT: this compile-time gate must remain a superset of every gate that references a + # Sign_* stage, namely the release gate below and the Integration stage's dependsOn. If they + # drift so a Sign_* stage is referenced but not emitted, the pipeline fails to compile on an + # unresolvable dependsOn. "internal and not a PR build" is the loosest of the three, so it is + # repeated verbatim in all of them, and the narrower "which run reasons actually sign" logic + # lives in the stage's runtime condition instead of being duplicated here. + # + # Signing never runs on PR builds: the public project cannot reach internal/azure-sdk-build-tools, + # and ESRP must not be handed unreviewed code. + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ each artifact in parameters.Artifacts }}: + - ${{ if eq(artifact.signBinaries, true) }}: + - template: /eng/pipelines/templates/stages/sign-binaries.yml + parameters: + Artifact: ${{ artifact }} + DependsOn: ${{ parameters.DependsOn }} # Release stages are compiled for: # * internal manual runs (existing behavior), and # * internal post-merge CI on 'main' (auto-release). For auto-release, only the packages changed by a @@ -46,389 +64,40 @@ stages: EnableUvAuth: true - ${{ each artifact in parameters.Artifacts }}: - - stage: 'Release_${{artifact.safename}}' - displayName: 'Release: ${{artifact.name}}' - dependsOn: - - ${{ parameters.DependsOn }} - - ${{ if and(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: - - AutoReleasePrepare - variables: - - template: /eng/pipelines/templates/variables/image.yml - - template: /eng/common/pipelines/templates/variables/api-review-break-glass.yml - # Auto-release CI: only release when the shared prepare stage flagged this artifact as changed. - # Manual runs: every declared artifact remains eligible (existing behavior). - ${{ if and(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: - condition: and(succeeded(), eq(dependencies.AutoReleasePrepare.outputs['ResolveAutoReleasePackages.resolve.ReleaseArtifact_${{ artifact.safename }}'], 'true'), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr')) + - template: /eng/pipelines/templates/stages/release-artifact.yml + parameters: + Artifact: ${{ artifact }} + # Packages that ship compiled binaries release from the artifact rebuilt by their + # Sign_ stage. Everything else keeps releasing from the build output, so a + # service can mix both kinds of packages freely. + ${{ if eq(artifact.signBinaries, true) }}: + ArtifactName: packages_${{ artifact.safeName }}_signed + DependsOn: Sign_${{ artifact.safeName }} ${{ else }}: - condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr')) - jobs: - - job: TagRepository - displayName: "Create release tag" - condition: and(succeeded(), ne(variables['Skip.TagRepository'], 'true')) - - pool: - image: ubuntu-24.04 - name: azsdk-pool - os: linux - - steps: - - checkout: self - - - download: current - artifact: ${{parameters.ArtifactName}} - timeoutInMinutes: 5 - - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.12' - - - template: /eng/pipelines/templates/steps/auth-dev-feed.yml - parameters: - DevFeedName: ${{ parameters.DevFeedName }} - - - template: /eng/common/pipelines/templates/steps/retain-run.yml - - - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml - parameters: - PackageName: "azure-template" - ServiceDirectory: "template" - TestPipeline: ${{ parameters.TestPipeline }} - - - template: /eng/common/pipelines/templates/steps/verify-changelog.yml - parameters: - PackageName: ${{artifact.name}} - ServiceName: ${{parameters.ServiceDirectory}} - ForRelease: true - - - template: /eng/common/pipelines/templates/steps/verify-restapi-spec-location.yml - parameters: - PackageName: ${{artifact.name}} - ServiceDirectory: ${{parameters.ServiceDirectory}} - ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}} - - - script: | - python -m pip install "./eng/tools/azure-sdk-tools" - displayName: Install tool dependencies - - - task: PythonScript@0 - displayName: Verify Dependency Presence - condition: and(succeeded(), ne(variables['Skip.VerifyDependencies'], 'true')) - inputs: - scriptPath: 'scripts/devops_tasks/verify_dependencies_present.py' - arguments: '--package-name ${{ artifact.name }} --service ${{ parameters.ServiceDirectory }}' - - - task: PythonScript@0 - displayName: Verify CI enabled - condition: succeeded() - inputs: - scriptPath: 'scripts/devops_tasks/verify_ci_enabled.py' - arguments: '--package-name ${{ artifact.name }} --service ${{ parameters.ServiceDirectory }}' - - - pwsh: | - Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}} - workingDirectory: $(Pipeline.Workspace) - displayName: Output Visible Artifacts - - - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml - parameters: - ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}} - PackageRepository: PyPI - ReleaseSha: $(Build.SourceVersion) - RepoId: Azure/azure-sdk-for-python - WorkingDirectory: $(System.DefaultWorkingDirectory) - AuthToken: '' - - - ${{if ne(artifact.skipPublishPackage, 'true')}}: - - deployment: PublishPackage - displayName: "Publish to ${{ parameters.PublicFeed }}" - condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true')) - # Auto-release runs after merge on main and must not wait on the package-publish approval gate. - ${{ if and(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: - environment: none - ${{ else }}: - environment: ${{ parameters.PublicPublishEnvironment }} - dependsOn: TagRepository - - templateContext: - type: releaseJob - isProduction: true - inputs: - - input: pipelineArtifact - artifactName: release_artifact - targetPath: $(Pipeline.Workspace)/release_artifact - - input: pipelineArtifact - artifactName: packages_extended - targetPath: $(Pipeline.Workspace)/packages_extended - - pool: - image: ubuntu-24.04 - name: azsdk-pool - os: linux - - strategy: - runOnce: - deploy: - steps: - - template: /eng/pipelines/templates/steps/auth-dev-feed.yml - parameters: - DevFeedName: ${{ parameters.DevFeedName }} - EnableTwineAuth: false - EnablePipAuth: true - EnableUvAuth: false - - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.10' - - - script: | - python -m pip install -r $(Pipeline.Workspace)/release_artifact/release_requirements.txt - displayName: Install Release Dependencies - - - ${{ if eq(parameters.PublicFeed, 'PyPi') }}: - - pwsh: | - $esrpDirectory = "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{artifact.name}}" - New-Item -ItemType Directory -Force -Path $esrpDirectory - - Get-ChildItem -Path "$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}" ` - | Where-Object { ($_.Name -like "*.tar.gz" -or $_.Name -like "*.whl") } ` - | Copy-Item -Destination $esrpDirectory - - Get-ChildItem $esrpDirectory - displayName: Isolate files for ESRP Publish - - - template: /eng/pipelines/templates/steps/esrp-publish.yml - parameters: - targetFolder: "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{artifact.name}}" - - - ${{ if ne(parameters.PublicFeed, 'PyPi') }}: - - task: TwineAuthenticate@0 - displayName: 'Authenticate to feed: ${{parameters.PublicFeed}}' - inputs: - artifactFeeds: ${{parameters.PublicFeed}} - - - script: | - set -e - twine upload --repository ${{parameters.PublicFeed}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.whl - echo "Uploaded whl to devops feed" - twine upload --repository ${{parameters.PublicFeed}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.tar.gz - echo "Uploaded sdist to devops feed" - displayName: 'Publish package to feed: ${{parameters.PublicFeed}}' - - - task: TwineAuthenticate@0 - displayName: 'Authenticate to feed: ${{parameters.DevFeedName}}' - inputs: - artifactFeeds: ${{parameters.DevFeedName}} - - - script: | - set -e - twine upload --repository ${{parameters.DevFeedName}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.whl - echo "Uploaded whl to devops feed" - twine upload --repository ${{parameters.DevFeedName}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.tar.gz - echo "Uploaded sdist to devops feed" - displayName: 'Publish package to feed: ${{parameters.DevFeedName}}' - - - job: MarkPackageReleaseCompletion - displayName: "Mark package release completion" - dependsOn: PublishPackage - - pool: - image: ubuntu-24.04 - name: azsdk-pool - os: linux - - steps: - - checkout: self - - - download: current - artifact: ${{parameters.ArtifactName}} - timeoutInMinutes: 5 - - - template: /eng/pipelines/templates/steps/auth-dev-feed.yml - parameters: - DevFeedName: ${{ parameters.DevFeedName }} - EnableTwineAuth: false - EnablePipAuth: true - EnableUvAuth: false - - - template: /eng/common/pipelines/templates/steps/mark-release-completion.yml - parameters: - ConfigFileDir: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo' - PackageArtifactName: ${{artifact.name}} - - # Management-plane-only workaround: skip marking packages that do not generate an APIView revision. - - pwsh: | - $packageInfoPath = '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{ artifact.name }}.json' - $packageInfo = Get-Content -Raw -Path $packageInfoPath | ConvertFrom-Json - $isManagementPackage = $packageInfo.SdkType -eq 'mgmt' - Write-Host "##vso[task.setvariable variable=IsManagementPackage]$isManagementPackage" - displayName: Check package SDK type - - - task: AzureCLI@2 - displayName: Mark Package Released - condition: >- - and( - succeeded(), - ne(variables['IsManagementPackage'], 'true'), - not( - and( - eq(variables['Skip.MarkPackageReleased'], 'true'), - eq(variables['IsRequesterAuthorizedToSkipApiReview'], 'true') - ) - ) - ) - inputs: - azureSubscription: "ADO to ARH Service Connection" - scriptType: pscore - scriptLocation: scriptPath - scriptPath: $(Build.SourcesDirectory)/eng/common/scripts/Mark-PackageReleased.ps1 - arguments: > - -PackageInfoFiles @('$(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{ artifact.name }}.json') - -RepoOwner 'Azure' - -AzSdkExePath '$(AZSDK)' - workingDirectory: $(Pipeline.Workspace) - - - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}: - - job: PublishGitHubIODocs - displayName: Publish Docs to GitHubIO Blob Storage - condition: >- - and( - succeeded(), - ne(variables['Skip.PublishDocs'], 'true'), - ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr') - ) - dependsOn: PublishPackage - - pool: - name: azsdk-pool - image: windows-2022 - os: windows - - steps: - - checkout: self - - - download: current - artifact: ${{parameters.ArtifactName}} - timeoutInMinutes: 5 - - - download: current - artifact: ${{parameters.DocArtifact}} - timeoutInMinutes: 5 - - - pwsh: | - if (Test-Path "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}") { - Get-ChildItem -Recurse "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}" - } - else { - New-Item -ItemType Directory -Force -Path "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}" - } - workingDirectory: $(Pipeline.Workspace) - displayName: Output Visible Artifacts - - - template: /eng/common/pipelines/templates/steps/publish-blobs.yml - parameters: - FolderForUpload: '$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}' - TargetLanguage: 'python' - ArtifactLocation: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}' - - - ${{if ne(artifact.skipPublishDocMs, 'true')}}: - - job: PublishDocs - displayName: Docs.MS Release - condition: >- - and( - succeeded(), - ne(variables['Skip.PublishDocs'], 'true'), - ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr') - ) - dependsOn: PublishPackage - - pool: - image: ubuntu-24.04 - name: azsdk-pool - os: linux - - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - sdk/**/*.md - - .github/CODEOWNERS - - - download: current - - # py2docfx requires Python >= 3.12.x, match docs pipeline version specification - - task: UsePythonVersion@0 - displayName: 'Use Python 3.12.x' - inputs: - versionSpec: '3.12.x' - - - template: /eng/pipelines/templates/steps/auth-dev-feed.yml - parameters: - DevFeedName: ${{ parameters.DevFeedName }} - - - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml - - - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml - parameters: - PackageInfoLocations: - - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json - WorkingDirectory: $(System.DefaultWorkingDirectory) - TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} - TargetDocRepoName: ${{parameters.TargetDocRepoName}} - Language: 'python' - SparseCheckoutPaths: - - docs-ref-services/ - - metadata/ - - - job: UpdatePackageVersion - displayName: "Update Package Version" - condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) - dependsOn: PublishPackage - - pool: - image: ubuntu-24.04 - name: azsdk-pool - os: linux - - steps: - - checkout: self - - task: UsePythonVersion@0 - - - template: /eng/pipelines/templates/steps/auth-dev-feed.yml - parameters: - DevFeedName: ${{ parameters.DevFeedName }} - - - script: | - python -m pip install "./eng/tools/azure-sdk-tools" - displayName: Install versioning tool dependencies - - - pwsh: | - sdk_increment_version --package-name ${{ artifact.name }} --service ${{ parameters.ServiceDirectory }} - if (Test-Path component-detection-pip-report.json) { - Write-Host "Deleting component-detection-pip-report.json" - rm component-detection-pip-report.json - } - displayName: Increment package version - - - template: /eng/common/pipelines/templates/steps/create-pull-request.yml - parameters: - RepoName: azure-sdk-for-python - PRBranchName: increment-package-version-${{ parameters.ServiceDirectory }}-$(Build.BuildId) - CommitMsg: "Increment package version after release of ${{ artifact.name }}" - PRTitle: "Increment version for ${{ parameters.ServiceDirectory }} releases" - CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' - AuthToken: '' - - - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: - - template: /eng/pipelines/templates/jobs/smoke.tests.yml - parameters: - Daily: false - ArtifactName: ${{ parameters.ArtifactName }} - Artifact: ${{ artifact }} - DevFeedName: ${{ parameters.DevFeedName }} + ArtifactName: ${{ parameters.ArtifactName }} + DependsOn: ${{ parameters.DependsOn }} + ServiceDirectory: ${{ parameters.ServiceDirectory }} + TestPipeline: ${{ parameters.TestPipeline }} + DocArtifact: ${{ parameters.DocArtifact }} + DevFeedName: ${{ parameters.DevFeedName }} + PublicFeed: ${{ parameters.PublicFeed }} + PublicPublishEnvironment: ${{ parameters.PublicPublishEnvironment }} + TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }} + TargetDocRepoName: ${{ parameters.TargetDocRepoName }} - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - stage: Integration - dependsOn: ${{parameters.DependsOn}} + # Signing stages are dependencies purely for ordering, so their signed artifacts exist + # before this stage downloads them. The condition still keys off the build stage only, + # exactly as before, so a signing failure degrades to skipping that one package's alpha + # publish rather than taking down the dev feed publish for the whole service. + # The gate below must stay identical to the signing emission gate at the top of this file. + dependsOn: + - ${{ parameters.DependsOn }} + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - ${{ each artifact in parameters.Artifacts }}: + - ${{ if eq(artifact.signBinaries, true) }}: + - Sign_${{ artifact.safeName }} condition: succeededOrFailed('${{parameters.DependsOn}}') jobs: - job: PublishPackages @@ -443,6 +112,16 @@ stages: artifact: ${{parameters.ArtifactName}} timeoutInMinutes: 5 + # Signed artifacts are downloaded best effort. If a Sign_* stage failed or was + # skipped the artifact is absent, and only that package's alpha publish is skipped. + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - ${{ each artifact in parameters.Artifacts }}: + - ${{ if eq(artifact.signBinaries, true) }}: + - download: current + artifact: packages_${{ artifact.safeName }}_signed + timeoutInMinutes: 5 + continueOnError: true + - task: UsePythonVersion@0 - template: ../steps/auth-dev-feed.yml @@ -456,33 +135,16 @@ stages: - ${{ each artifact in parameters.Artifacts }}: - ${{if ne(artifact.skipPublishDevFeed, 'true')}}: - - - pwsh: | - # If BuildTargetingString is set, check whether this artifact matches any of the - # (possibly comma-separated) glob patterns before attempting to publish. - # This handles scoped builds where only a subset of packages are built. - $targetingString = $env:BUILDTARGETINGSTRING - if ($targetingString) { - $globs = $targetingString -split "," - $isTargeted = $globs | Where-Object { "${{artifact.name}}" -like $_.Trim() } - if (-not $isTargeted) { - Write-Host "Package '${{artifact.name}}' does not match BuildTargetingString '$targetingString'. Skipping integration publish." - exit 0 - } - } - - $fileCount = (Get-ChildItem $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}} | ? {$_.Name -match "-[0-9]*.[0-9]*.[0-9]*a[0-9]*" } | Measure-Object).Count - - if ($fileCount -eq 0) { - Write-Host "No alpha packages for ${{artifact.name}} to publish." - exit 0 - } - - twine upload --repository $(DevFeedName) --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*-*a*.whl - echo "Uploaded whl to devops feed $(DevFeedName)" - twine upload --repository $(DevFeedName) --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*-*a*.tar.gz - echo "Uploaded sdist to devops feed $(DevFeedName)" - displayName: 'Publish ${{artifact.name}} alpha package' + - template: /eng/pipelines/templates/steps/publish-alpha-package.yml + parameters: + Artifact: ${{ artifact }} + # Packages that ship compiled binaries publish the wheels rebuilt by + # their Sign_ stage; everything else publishes straight from + # the build output, so a service can mix both kinds of packages freely. + ${{ if and(eq(artifact.signBinaries, true), ne(variables['Build.Reason'], 'PullRequest')) }}: + PackagePath: $(Pipeline.Workspace)/packages_${{ artifact.safeName }}_signed/${{ artifact.name }} + ${{ else }}: + PackagePath: $(Pipeline.Workspace)/${{ parameters.ArtifactName }}/${{ artifact.name }} - job: PublishDocsToNightlyBranch dependsOn: PublishPackages diff --git a/eng/pipelines/templates/stages/release-artifact.yml b/eng/pipelines/templates/stages/release-artifact.yml new file mode 100644 index 000000000000..1a678c644692 --- /dev/null +++ b/eng/pipelines/templates/stages/release-artifact.yml @@ -0,0 +1,426 @@ +# Emits the `Release_` stage for a single artifact. +# +# Extracted verbatim from archetype-python-release.yml so the release jobs keep exactly one +# definition. Artifacts that ship compiled binaries release from a different pipeline artifact +# (`packages__signed`) and depend on a different upstream stage (`Sign_`) +# than everything else. Parameterizing just those two values here avoids forking the release +# job definitions into signed and unsigned copies that would inevitably drift. +# +# Callers resolve ArtifactName and DependsOn per artifact; every job below is otherwise +# unchanged from the shared release workflow. + +parameters: + - name: Artifact + type: object + # Pipeline artifact this package releases from. `packages_extended` for ordinary packages, + # `packages__signed` for packages that opted in with `signBinaries: true`. + - name: ArtifactName + type: string + # Upstream stage. The build stage for ordinary packages, `Sign_` when signing. + - name: DependsOn + type: string + - name: ServiceDirectory + type: string + default: 'not-specified' + - name: TestPipeline + type: boolean + default: false + - name: DocArtifact + type: string + default: 'documentation' + - name: DevFeedName + type: string + default: 'public/azure-sdk-for-python' + - name: PublicFeed + type: string + default: PyPi + - name: PublicPublishEnvironment + type: string + default: package-publish + - name: TargetDocRepoOwner + type: string + default: '' + - name: TargetDocRepoName + type: string + default: '' + +stages: + - stage: 'Release_${{parameters.Artifact.safename}}' + displayName: 'Release: ${{parameters.Artifact.name}}' + dependsOn: + - ${{ parameters.DependsOn }} + - ${{ if and(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: + - AutoReleasePrepare + variables: + - template: /eng/pipelines/templates/variables/image.yml + - template: /eng/common/pipelines/templates/variables/api-review-break-glass.yml + # Auto-release CI: only release when the shared prepare stage flagged this artifact as changed. + # Manual runs: every declared artifact remains eligible (existing behavior). + ${{ if and(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: + condition: and(succeeded(), eq(dependencies.AutoReleasePrepare.outputs['ResolveAutoReleasePackages.resolve.ReleaseArtifact_${{ parameters.Artifact.safename }}'], 'true'), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr')) + ${{ else }}: + condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr')) + jobs: + - job: TagRepository + displayName: "Create release tag" + condition: and(succeeded(), ne(variables['Skip.TagRepository'], 'true')) + + pool: + image: ubuntu-24.04 + name: azsdk-pool + os: linux + + steps: + - checkout: self + + - download: current + artifact: ${{parameters.ArtifactName}} + timeoutInMinutes: 5 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.12' + + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + + - template: /eng/common/pipelines/templates/steps/retain-run.yml + + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + parameters: + PackageName: "azure-template" + ServiceDirectory: "template" + TestPipeline: ${{ parameters.TestPipeline }} + + - template: /eng/common/pipelines/templates/steps/verify-changelog.yml + parameters: + PackageName: ${{parameters.Artifact.name}} + ServiceName: ${{parameters.ServiceDirectory}} + ForRelease: true + + - template: /eng/common/pipelines/templates/steps/verify-restapi-spec-location.yml + parameters: + PackageName: ${{parameters.Artifact.name}} + ServiceDirectory: ${{parameters.ServiceDirectory}} + ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}} + + - script: | + python -m pip install "./eng/tools/azure-sdk-tools" + displayName: Install tool dependencies + + - task: PythonScript@0 + displayName: Verify Dependency Presence + condition: and(succeeded(), ne(variables['Skip.VerifyDependencies'], 'true')) + inputs: + scriptPath: 'scripts/devops_tasks/verify_dependencies_present.py' + arguments: '--package-name ${{ parameters.Artifact.name }} --service ${{ parameters.ServiceDirectory }}' + + - task: PythonScript@0 + displayName: Verify CI enabled + condition: succeeded() + inputs: + scriptPath: 'scripts/devops_tasks/verify_ci_enabled.py' + arguments: '--package-name ${{ parameters.Artifact.name }} --service ${{ parameters.ServiceDirectory }}' + + - pwsh: | + Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}} + workingDirectory: $(Pipeline.Workspace) + displayName: Output Visible Artifacts + + - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml + parameters: + ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}} + PackageRepository: PyPI + ReleaseSha: $(Build.SourceVersion) + RepoId: Azure/azure-sdk-for-python + WorkingDirectory: $(System.DefaultWorkingDirectory) + AuthToken: '' + + - ${{if ne(parameters.Artifact.skipPublishPackage, 'true')}}: + - deployment: PublishPackage + displayName: "Publish to ${{ parameters.PublicFeed }}" + condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true')) + # Auto-release runs after merge on main and must not wait on the package-publish approval gate. + ${{ if and(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: + environment: none + ${{ else }}: + environment: ${{ parameters.PublicPublishEnvironment }} + dependsOn: TagRepository + + templateContext: + type: releaseJob + isProduction: true + inputs: + - input: pipelineArtifact + artifactName: release_artifact + targetPath: $(Pipeline.Workspace)/release_artifact + - input: pipelineArtifact + artifactName: packages_extended + targetPath: $(Pipeline.Workspace)/packages_extended + + pool: + image: ubuntu-24.04 + name: azsdk-pool + os: linux + + strategy: + runOnce: + deploy: + steps: + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + EnableTwineAuth: false + EnablePipAuth: true + EnableUvAuth: false + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.10' + + - script: | + python -m pip install -r $(Pipeline.Workspace)/release_artifact/release_requirements.txt + displayName: Install Release Dependencies + + - ${{ if eq(parameters.PublicFeed, 'PyPi') }}: + - pwsh: | + $esrpDirectory = "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}" + New-Item -ItemType Directory -Force -Path $esrpDirectory + + Get-ChildItem -Path "$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}" ` + | Where-Object { ($_.Name -like "*.tar.gz" -or $_.Name -like "*.whl") } ` + | Copy-Item -Destination $esrpDirectory + + Get-ChildItem $esrpDirectory + displayName: Isolate files for ESRP Publish + + - template: /eng/pipelines/templates/steps/esrp-publish.yml + parameters: + targetFolder: "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}" + + - ${{ if ne(parameters.PublicFeed, 'PyPi') }}: + - task: TwineAuthenticate@0 + displayName: 'Authenticate to feed: ${{parameters.PublicFeed}}' + inputs: + artifactFeeds: ${{parameters.PublicFeed}} + + - script: | + set -e + twine upload --repository ${{parameters.PublicFeed}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}/*.whl + echo "Uploaded whl to devops feed" + twine upload --repository ${{parameters.PublicFeed}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}/*.tar.gz + echo "Uploaded sdist to devops feed" + displayName: 'Publish package to feed: ${{parameters.PublicFeed}}' + + - task: TwineAuthenticate@0 + displayName: 'Authenticate to feed: ${{parameters.DevFeedName}}' + inputs: + artifactFeeds: ${{parameters.DevFeedName}} + + - script: | + set -e + twine upload --repository ${{parameters.DevFeedName}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}/*.whl + echo "Uploaded whl to devops feed" + twine upload --repository ${{parameters.DevFeedName}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}/*.tar.gz + echo "Uploaded sdist to devops feed" + displayName: 'Publish package to feed: ${{parameters.DevFeedName}}' + + - job: MarkPackageReleaseCompletion + displayName: "Mark package release completion" + dependsOn: PublishPackage + + pool: + image: ubuntu-24.04 + name: azsdk-pool + os: linux + + steps: + - checkout: self + + - download: current + artifact: ${{parameters.ArtifactName}} + timeoutInMinutes: 5 + + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + EnableTwineAuth: false + EnablePipAuth: true + EnableUvAuth: false + + - template: /eng/common/pipelines/templates/steps/mark-release-completion.yml + parameters: + ConfigFileDir: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo' + PackageArtifactName: ${{parameters.Artifact.name}} + + # Management-plane-only workaround: skip marking packages that do not generate an APIView revision. + - pwsh: | + $packageInfoPath = '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{ parameters.Artifact.name }}.json' + $packageInfo = Get-Content -Raw -Path $packageInfoPath | ConvertFrom-Json + $isManagementPackage = $packageInfo.SdkType -eq 'mgmt' + Write-Host "##vso[task.setvariable variable=IsManagementPackage]$isManagementPackage" + displayName: Check package SDK type + + - task: AzureCLI@2 + displayName: Mark Package Released + condition: >- + and( + succeeded(), + ne(variables['IsManagementPackage'], 'true'), + not( + and( + eq(variables['Skip.MarkPackageReleased'], 'true'), + eq(variables['IsRequesterAuthorizedToSkipApiReview'], 'true') + ) + ) + ) + inputs: + azureSubscription: "ADO to ARH Service Connection" + scriptType: pscore + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/scripts/Mark-PackageReleased.ps1 + arguments: > + -PackageInfoFiles @('$(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{ parameters.Artifact.name }}.json') + -RepoOwner 'Azure' + -AzSdkExePath '$(AZSDK)' + workingDirectory: $(Pipeline.Workspace) + + - ${{if ne(parameters.Artifact.skipPublishDocGithubIo, 'true')}}: + - job: PublishGitHubIODocs + displayName: Publish Docs to GitHubIO Blob Storage + condition: >- + and( + succeeded(), + ne(variables['Skip.PublishDocs'], 'true'), + ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr') + ) + dependsOn: PublishPackage + + pool: + name: azsdk-pool + image: windows-2022 + os: windows + + steps: + - checkout: self + + - download: current + artifact: ${{parameters.ArtifactName}} + timeoutInMinutes: 5 + + - download: current + artifact: ${{parameters.DocArtifact}} + timeoutInMinutes: 5 + + - pwsh: | + if (Test-Path "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{parameters.Artifact.name}}") { + Get-ChildItem -Recurse "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{parameters.Artifact.name}}" + } + else { + New-Item -ItemType Directory -Force -Path "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{parameters.Artifact.name}}" + } + workingDirectory: $(Pipeline.Workspace) + displayName: Output Visible Artifacts + + - template: /eng/common/pipelines/templates/steps/publish-blobs.yml + parameters: + FolderForUpload: '$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{parameters.Artifact.name}}' + TargetLanguage: 'python' + ArtifactLocation: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}' + + - ${{if ne(parameters.Artifact.skipPublishDocMs, 'true')}}: + - job: PublishDocs + displayName: Docs.MS Release + condition: >- + and( + succeeded(), + ne(variables['Skip.PublishDocs'], 'true'), + ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr') + ) + dependsOn: PublishPackage + + pool: + image: ubuntu-24.04 + name: azsdk-pool + os: linux + + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - sdk/**/*.md + - .github/CODEOWNERS + + - download: current + + # py2docfx requires Python >= 3.12.x, match docs pipeline version specification + - task: UsePythonVersion@0 + displayName: 'Use Python 3.12.x' + inputs: + versionSpec: '3.12.x' + + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml + parameters: + PackageInfoLocations: + - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{parameters.Artifact.name}}.json + WorkingDirectory: $(System.DefaultWorkingDirectory) + TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} + TargetDocRepoName: ${{parameters.TargetDocRepoName}} + Language: 'python' + SparseCheckoutPaths: + - docs-ref-services/ + - metadata/ + + - job: UpdatePackageVersion + displayName: "Update Package Version" + condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) + dependsOn: PublishPackage + + pool: + image: ubuntu-24.04 + name: azsdk-pool + os: linux + + steps: + - checkout: self + - task: UsePythonVersion@0 + + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + + - script: | + python -m pip install "./eng/tools/azure-sdk-tools" + displayName: Install versioning tool dependencies + + - pwsh: | + sdk_increment_version --package-name ${{ parameters.Artifact.name }} --service ${{ parameters.ServiceDirectory }} + if (Test-Path component-detection-pip-report.json) { + Write-Host "Deleting component-detection-pip-report.json" + rm component-detection-pip-report.json + } + displayName: Increment package version + + - template: /eng/common/pipelines/templates/steps/create-pull-request.yml + parameters: + RepoName: azure-sdk-for-python + PRBranchName: increment-package-version-${{ parameters.ServiceDirectory }}-$(Build.BuildId) + CommitMsg: "Increment package version after release of ${{ parameters.Artifact.name }}" + PRTitle: "Increment version for ${{ parameters.ServiceDirectory }} releases" + CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' + AuthToken: '' + + - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: + - template: /eng/pipelines/templates/jobs/smoke.tests.yml + parameters: + Daily: false + ArtifactName: ${{ parameters.ArtifactName }} + Artifact: ${{ parameters.Artifact }} + DevFeedName: ${{ parameters.DevFeedName }} diff --git a/eng/pipelines/templates/stages/sign-binaries.yml b/eng/pipelines/templates/stages/sign-binaries.yml new file mode 100644 index 000000000000..498c467aa0fd --- /dev/null +++ b/eng/pipelines/templates/stages/sign-binaries.yml @@ -0,0 +1,240 @@ +# Emits a single `Sign_` stage for one artifact that ships compiled binaries. +# +# Packages opt in from their service ci.yml by setting `signBinaries: true` on the artifact. +# This stage is emitted by archetype-python-release.yml, which owns the gating that decides +# when signing runs (release + scheduled builds, never PR builds). +# +# The mac and windows wheels built by the Build stage contain unsigned binaries. This stage +# unpacks only this artifact's wheels, sends the binaries through ESRP, rebuilds the wheels, +# and republishes them as `packages__signed`. +# +# The published artifact deliberately mirrors the layout of `packages_extended`: +# +# packages__signed/ +# PackageInfo/.json +# /*.whl, *.tar.gz, apistub tokens +# +# so that every downstream release job works against it unchanged, just by swapping the +# artifact name. `packages_extended` itself is left untouched and still contains the +# unsigned wheels for this package alongside every other package in the service. + +parameters: + - name: Artifact + type: object + - name: DependsOn + type: string + default: Build + +stages: + - stage: Sign_${{ parameters.Artifact.safeName }} + displayName: 'Sign: ${{ parameters.Artifact.name }}' + dependsOn: ${{ parameters.DependsOn }} + # Intentionally does not check Skip.Release or SetDevVersion. Signing must run whenever + # *either* consumer might need the signed wheels: the Release_ stage or the + # Integration dev feed publish. Skip logic belongs on those consumers, which already carry + # it. Adding it here would propagate through Integration's dependency and silently break + # the nightly and manual dev version alpha publishes. + # + # Run reasons are filtered here rather than at compile time so the emission gate in + # archetype-python-release.yml can stay short enough to repeat verbatim wherever a + # Sign_* stage is referenced. Release builds are manual runs and auto-release CI on main; + # scheduled builds sign the alpha packages that the Integration stage publishes. + # + # The 'Schedule' term is load bearing and coupled to daily-dev-build-variable.yml, which + # sets SetDevVersion=true only when Build.Reason is exactly 'Schedule'. Alpha wheels exist + # only when that is true, so Integration needs signed wheels under exactly the same reasons. + # If that check ever widens (for example to ScheduleForced), widen this list to match or + # scheduled signing silently stops and the alpha publish falls back to skipping the package. + # Reasons absent from this list (BatchedCI, ScheduleForced, IndividualCI off main) produce + # neither alpha wheels nor a release stage, so skipping signing for them is correct. + condition: >- + and( + succeeded(), + ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr'), + or( + in(variables['Build.Reason'], 'Manual', '', 'Schedule'), + and( + eq(variables['Build.Reason'], 'IndividualCI'), + eq(variables['Build.SourceBranch'], 'refs/heads/main') + ) + ) + ) + + variables: + - template: /eng/pipelines/templates/variables/globals.yml + - template: /eng/pipelines/templates/variables/image.yml + + jobs: + - job: Sign_macOS + displayName: Sign macOS Wheels + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + steps: + # The default sparse checkout always includes /eng, which is all these jobs need. + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - task: UsePythonVersion@0 + displayName: "Use Python $(PythonVersion)" + inputs: + versionSpec: $(PythonVersion) + + - task: DownloadPipelineArtifact@2 + displayName: Download unsigned mac wheels + inputs: + artifactName: packages_mac + # Only this artifact's wheels. Other packages in the service are untouched. + itemPattern: '${{ parameters.Artifact.name }}/**' + targetPath: $(Build.ArtifactStagingDirectory)/packages_mac + + - pwsh: | + python eng/scripts/wheel_signing/extract_sign_inputs.py ` + --platform mac ` + --wheels-dir "$(Build.ArtifactStagingDirectory)/packages_mac/${{ parameters.Artifact.name }}" ` + --work-dir "$(Build.ArtifactStagingDirectory)/mac-sign-work" ` + --sign-input-zip "$(Build.ArtifactStagingDirectory)/mac-sign-input.zip" + displayName: Extract mac wheel binaries + + - template: pipelines/steps/azd-cli-mac-signing.yml@azure-sdk-build-tools + parameters: + MacPath: "$(Build.ArtifactStagingDirectory)" + MacPattern: "mac-sign-input.zip" + Notarize: false + + - pwsh: | + python eng/scripts/wheel_signing/repackage_signed_wheels.py ` + --platform mac ` + --work-dir "$(Build.ArtifactStagingDirectory)/mac-sign-work" ` + --signed-input-zip "$(Build.ArtifactStagingDirectory)/mac-sign-input.zip" ` + --output-wheels-dir "$(Build.ArtifactStagingDirectory)/mac-wheels-signed" + displayName: Repackage mac wheels + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: '$(Build.ArtifactStagingDirectory)/mac-wheels-signed' + ArtifactName: 'packages_${{ parameters.Artifact.safeName }}_mac_signed' + + - job: Sign_Windows + displayName: Sign Windows Wheels + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + steps: + # The default sparse checkout always includes /eng, which is all these jobs need. + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - task: UsePythonVersion@0 + displayName: "Use Python $(PythonVersion)" + inputs: + versionSpec: $(PythonVersion) + + - task: DownloadPipelineArtifact@2 + displayName: Download unsigned windows wheels + inputs: + artifactName: packages_windows + itemPattern: '${{ parameters.Artifact.name }}/**' + targetPath: $(Build.ArtifactStagingDirectory)/packages_windows + + - pwsh: | + python eng/scripts/wheel_signing/extract_sign_inputs.py ` + --platform windows ` + --wheels-dir "$(Build.ArtifactStagingDirectory)/packages_windows/${{ parameters.Artifact.name }}" ` + --work-dir "$(Build.ArtifactStagingDirectory)/win-sign-work" ` + --sign-input-dir "$(Build.ArtifactStagingDirectory)/win-sign-input" + displayName: Extract windows wheel binaries + + - template: pipelines/steps/azd-cli-win-signing.yml@azure-sdk-build-tools + parameters: + WinPath: "$(Build.ArtifactStagingDirectory)/win-sign-input" + WinPattern: '**/*.pyd' + + - pwsh: | + python eng/scripts/wheel_signing/repackage_signed_wheels.py ` + --platform windows ` + --work-dir "$(Build.ArtifactStagingDirectory)/win-sign-work" ` + --signed-input-dir "$(Build.ArtifactStagingDirectory)/win-sign-input" ` + --output-wheels-dir "$(Build.ArtifactStagingDirectory)/win-wheels-signed" + displayName: Repackage windows wheels + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: '$(Build.ArtifactStagingDirectory)/win-wheels-signed' + ArtifactName: 'packages_${{ parameters.Artifact.safeName }}_win_signed' + + - job: Assemble + displayName: Assemble signed artifact + dependsOn: + - Sign_macOS + - Sign_Windows + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + steps: + - checkout: none + + # packages_extended is the merged output of the three platform builds plus the + # apistub tokens and PackageInfo. Take this package's slice of it as the base, + # then overwrite the mac and windows wheels with their signed replacements. + - task: DownloadPipelineArtifact@2 + displayName: Download package files + inputs: + artifactName: packages_extended + itemPattern: | + ${{ parameters.Artifact.name }}/** + PackageInfo/${{ parameters.Artifact.name }}.json + targetPath: $(Build.ArtifactStagingDirectory)/signed + + - task: DownloadPipelineArtifact@2 + displayName: Download signed mac wheels + inputs: + artifactName: packages_${{ parameters.Artifact.safeName }}_mac_signed + targetPath: $(Build.ArtifactStagingDirectory)/mac_signed + + - task: DownloadPipelineArtifact@2 + displayName: Download signed windows wheels + inputs: + artifactName: packages_${{ parameters.Artifact.safeName }}_win_signed + targetPath: $(Build.ArtifactStagingDirectory)/win_signed + + - pwsh: | + $ErrorActionPreference = 'Stop' + + $packageDir = "$(Build.ArtifactStagingDirectory)/signed/${{ parameters.Artifact.name }}" + $packageInfo = "$(Build.ArtifactStagingDirectory)/signed/PackageInfo/${{ parameters.Artifact.name }}.json" + + if (-not (Test-Path $packageDir)) { + throw "Expected package folder '$packageDir' in packages_extended." + } + if (-not (Test-Path $packageInfo)) { + throw "Expected package info file '$packageInfo' in packages_extended." + } + + $signedWheels = @( + Get-ChildItem "$(Build.ArtifactStagingDirectory)/mac_signed" -Recurse -Filter "*.whl" + Get-ChildItem "$(Build.ArtifactStagingDirectory)/win_signed" -Recurse -Filter "*.whl" + ) + + if (-not $signedWheels) { + throw "No signed wheels were produced for ${{ parameters.Artifact.name }}." + } + + foreach ($wheel in $signedWheels) { + $target = Join-Path $packageDir $wheel.Name + if (-not (Test-Path $target)) { + throw "Signed wheel '$($wheel.Name)' has no unsigned counterpart in packages_extended. The signed artifact would not match the build output." + } + Write-Host "Replacing $($wheel.Name) with its signed build." + Copy-Item -Path $wheel.FullName -Destination $target -Force + } + + Write-Host "`nFinal contents of the signed artifact:" + Get-ChildItem -Recurse "$(Build.ArtifactStagingDirectory)/signed" | Select-Object -ExpandProperty FullName + displayName: Overlay signed wheels + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: '$(Build.ArtifactStagingDirectory)/signed' + ArtifactName: 'packages_${{ parameters.Artifact.safeName }}_signed' diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 5f026f19ded9..a4bf41dba5d1 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -91,8 +91,8 @@ steps: BuildTargetingString: ${{ parameters.BuildTargetingString }} PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo - # todo, walk the artifacts and ensure that one which includes an extension package is present - # if not, we only need to build on linux. if so, we need to build on all platforms + # Decides whether the mac/windows builds produce anything, based on whether any targeted package + # declares compiled binaries. If none do, only the linux build runs. - template: /eng/pipelines/templates/steps/resolve-build-platforms.yml parameters: PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo diff --git a/eng/pipelines/templates/steps/publish-alpha-package.yml b/eng/pipelines/templates/steps/publish-alpha-package.yml new file mode 100644 index 000000000000..dc246652c884 --- /dev/null +++ b/eng/pipelines/templates/steps/publish-alpha-package.yml @@ -0,0 +1,49 @@ +# Publishes one package's alpha (daily dev build) wheel and sdist to the dev feed. +# +# Extracted from the Integration stage in archetype-python-release.yml so the package path can +# vary per artifact. Packages built with `signBinaries: true` publish from their signed +# artifact; everything else publishes from `packages_extended`. + +parameters: + - name: Artifact + type: object + # Folder holding this package's built files, already resolved by the caller. + - name: PackagePath + type: string + +steps: + - pwsh: | + # The signed artifact is absent when a Sign_* stage failed or was skipped. Skip this one + # package rather than failing the dev feed publish for every other package in the service. + # Logged as a build issue, not just Write-Warning, so a signing failure degrading to + # "this package was not published" is visible in the run summary instead of buried in logs. + if (-not (Test-Path "${{ parameters.PackagePath }}")) { + Write-Host "##vso[task.logissue type=warning]No build output at '${{ parameters.PackagePath }}'. Skipping alpha publish for ${{ parameters.Artifact.name }}. If this package sets signBinaries, check whether its Sign stage failed or was skipped." + exit 0 + } + + # If BuildTargetingString is set, check whether this artifact matches any of the + # (possibly comma-separated) glob patterns before attempting to publish. + # This handles scoped builds where only a subset of packages are built. + $targetingString = $env:BUILDTARGETINGSTRING + if ($targetingString) { + $globs = $targetingString -split "," + $isTargeted = $globs | Where-Object { "${{ parameters.Artifact.name }}" -like $_.Trim() } + if (-not $isTargeted) { + Write-Host "Package '${{ parameters.Artifact.name }}' does not match BuildTargetingString '$targetingString'. Skipping integration publish." + exit 0 + } + } + + $fileCount = (Get-ChildItem ${{ parameters.PackagePath }} | ? {$_.Name -match "-[0-9]*.[0-9]*.[0-9]*a[0-9]*" } | Measure-Object).Count + + if ($fileCount -eq 0) { + Write-Host "No alpha packages for ${{ parameters.Artifact.name }} to publish." + exit 0 + } + + twine upload --repository $(DevFeedName) --config-file $(PYPIRC_PATH) ${{ parameters.PackagePath }}/*-*a*.whl + echo "Uploaded whl to devops feed $(DevFeedName)" + twine upload --repository $(DevFeedName) --config-file $(PYPIRC_PATH) ${{ parameters.PackagePath }}/*-*a*.tar.gz + echo "Uploaded sdist to devops feed $(DevFeedName)" + displayName: 'Publish ${{ parameters.Artifact.name }} alpha package' diff --git a/eng/pipelines/templates/steps/resolve-build-platforms.yml b/eng/pipelines/templates/steps/resolve-build-platforms.yml index 43db2e725352..2de46faf2585 100644 --- a/eng/pipelines/templates/steps/resolve-build-platforms.yml +++ b/eng/pipelines/templates/steps/resolve-build-platforms.yml @@ -4,15 +4,32 @@ parameters: default: '' steps: - # when we merge pipeline v3, this check will change to examining the targeting string $(TargetingString) - # as the generate-pr-diff call + resolution will be present in resolve-package-targeting.yml. - # until then, we simply check to see if we're targeting storage service directory + # Packages that ship compiled binaries opt in with `signBinaries: true` on their artifact entry in + # the service ci.yml. Save-Package-Properties copies that artifact entry verbatim into each package + # info file as `ArtifactDetails`, so the opt-in is readable here without threading the artifact list + # through every caller. + # + # resolve-package-targeting.yml runs immediately before this and deletes the package info files that + # this run is not targeting, so the folder is already narrowed to the packages being built; on 'auto' + # pull request builds save-package-properties.yml narrows it to the PR diff first. This deliberately + # keeps reading the folder rather than the $(TargetingString) those steps also set, because that + # variable carries only package names and the decision below needs an artifact property. - pwsh: | - $packageProperties = Get-ChildItem -Recurse -Force "${{ parameters.PackagePropertiesFolder }}/*.json" ` - | ForEach-Object { $_.Name.Replace(".json", "") } + $binaryPackages = @() - if ($packageProperties -contains "azure-storage-extensions") { - Write-Host "Targeting storage, enabling extension build." + foreach ($packageInfoPath in (Get-ChildItem -Recurse -Force "${{ parameters.PackagePropertiesFolder }}/*.json")) { + $packageInfo = Get-Content -Raw -Path $packageInfoPath.FullName | ConvertFrom-Json + + if ($packageInfo.ArtifactDetails.signBinaries -eq $true) { + $binaryPackages += $packageInfo.Name + } + } + + if ($binaryPackages) { + Write-Host "Targeting package(s) with compiled binaries ($($binaryPackages -join ', ')), enabling extension build." Write-Host "##vso[task.setvariable variable=ENABLE_EXTENSION_BUILD]true" } + else { + Write-Host "No targeted package declares compiled binaries, building on linux only." + } displayName: Check extension package presence diff --git a/eng/scripts/wheel_signing/extract_sign_inputs.py b/eng/scripts/wheel_signing/extract_sign_inputs.py new file mode 100644 index 000000000000..045dc4ea1c5c --- /dev/null +++ b/eng/scripts/wheel_signing/extract_sign_inputs.py @@ -0,0 +1,178 @@ +import argparse +import json +import shutil +import zipfile +from pathlib import Path +from typing import Dict, List + + +SIGNABLE_SUFFIXES = {".so", ".dylib", ".dll", ".pyd"} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Extract wheel files, collect signable binaries, and generate " + "signing payloads plus a manifest for wheel repackaging." + ) + ) + parser.add_argument("--platform", choices=["mac", "windows"], required=True) + parser.add_argument("--wheels-dir", required=True, help="Directory containing input wheel files.") + parser.add_argument("--work-dir", required=True, help="Working directory for unpacked wheels and manifest.") + parser.add_argument( + "--sign-input-zip", + default=None, + help="Output zip containing binaries to sign (required for --platform mac).", + ) + parser.add_argument( + "--sign-input-dir", + default=None, + help="Output folder containing binaries to sign (required for --platform windows).", + ) + return parser.parse_args() + + +def validate_args(args: argparse.Namespace) -> None: + if args.platform == "mac": + if not args.sign_input_zip: + raise ValueError("--sign-input-zip is required for --platform mac.") + if args.sign_input_dir: + raise ValueError("--sign-input-dir is not valid for --platform mac.") + else: + if not args.sign_input_dir: + raise ValueError("--sign-input-dir is required for --platform windows.") + if args.sign_input_zip: + raise ValueError("--sign-input-zip is not valid for --platform windows.") + + +def reset_dir(path: Path) -> None: + if path.exists(): + shutil.rmtree(path) + path.mkdir(parents=True, exist_ok=True) + + +def collect_wheels(wheels_dir: Path) -> List[Path]: + if not wheels_dir.is_dir(): + raise FileNotFoundError(f"Wheel directory not found: {wheels_dir}") + return sorted(wheels_dir.rglob("*.whl")) + + +def collect_signable_files(unpacked_wheel_dir: Path) -> List[Path]: + files = [] + for path in sorted(unpacked_wheel_dir.rglob("*")): + if path.is_file() and path.suffix.lower() in SIGNABLE_SUFFIXES: + files.append(path) + return files + + +def write_manifest(manifest_path: Path, manifest_data: Dict) -> None: + manifest_path.parent.mkdir(parents=True, exist_ok=True) + with manifest_path.open("w", encoding="utf-8") as handle: + json.dump(manifest_data, handle, indent=2, sort_keys=True) + handle.write("\n") + + +def create_zip_from_dir(source_dir: Path, output_zip: Path) -> None: + output_zip.parent.mkdir(parents=True, exist_ok=True) + if output_zip.exists(): + output_zip.unlink() + + with zipfile.ZipFile(output_zip, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for file_path in sorted(source_dir.rglob("*")): + if file_path.is_file(): + archive.write(file_path, file_path.relative_to(source_dir).as_posix()) + + +def main() -> None: + args = parse_args() + validate_args(args) + + wheels_dir = Path(args.wheels_dir).resolve() + work_dir = Path(args.work_dir).resolve() + unpack_root = work_dir / "unpacked" + manifest_path = work_dir / "signing-manifest.json" + + if args.platform == "windows": + payload_dir = Path(args.sign_input_dir).resolve() + else: + payload_dir = work_dir / "mac-sign-input" + + reset_dir(work_dir) + reset_dir(unpack_root) + reset_dir(payload_dir) + + wheels = collect_wheels(wheels_dir) + print(f"Platform: {args.platform}") + print(f"Input wheels dir: {wheels_dir}") + print(f"Work dir: {work_dir}") + + manifest: Dict[str, object] = { + "platform": args.platform, + "wheels": [], + "entries": [], + } + + payload_index = 0 + + for wheel_path in wheels: + unpack_dir_name = wheel_path.name[:-4] + unpacked_wheel_dir = unpack_root / unpack_dir_name + unpacked_wheel_dir.mkdir(parents=True, exist_ok=True) + print(f"[EXTRACT] wheel={wheel_path.name} unpack_dir={unpacked_wheel_dir}") + + with zipfile.ZipFile(wheel_path, "r") as archive: + archive.extractall(unpacked_wheel_dir) + + manifest["wheels"].append( + { + "wheel_filename": wheel_path.name, + "unpack_dir": unpack_dir_name, + } + ) + + signable_files = collect_signable_files(unpacked_wheel_dir) + print(f"[EXTRACT] wheel={wheel_path.name} signable_count={len(signable_files)}") + for signable_file in signable_files: + relative_path = signable_file.relative_to(unpacked_wheel_dir).as_posix() + payload_subdir = f"{payload_index:05d}" + payload_index += 1 + + payload_path = f"{payload_subdir}/{signable_file.name}" + payload_file = payload_dir / payload_subdir / signable_file.name + payload_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(signable_file, payload_file) + print( + "[MAP_EXTRACT] " + f"payload={payload_path} " + f"source_wheel={wheel_path.name} " + f"source_relative_path={relative_path} " + f"source_file={signable_file} " + f"payload_file={payload_file}" + ) + + manifest["entries"].append( + { + "wheel_filename": wheel_path.name, + "unpack_dir": unpack_dir_name, + "relative_path": relative_path, + "payload_path": payload_path, + } + ) + + write_manifest(manifest_path, manifest) + + if args.platform == "mac": + sign_zip = Path(args.sign_input_zip).resolve() + create_zip_from_dir(payload_dir, sign_zip) + + print(f"Wheels processed: {len(wheels)}") + print(f"Signable binaries collected: {len(manifest['entries'])}") + print(f"Manifest: {manifest_path}") + if args.platform == "mac": + print(f"Sign payload zip: {Path(args.sign_input_zip).resolve()}") + else: + print(f"Sign payload dir: {payload_dir}") + + +if __name__ == "__main__": + main() diff --git a/eng/scripts/wheel_signing/repackage_signed_wheels.py b/eng/scripts/wheel_signing/repackage_signed_wheels.py new file mode 100644 index 000000000000..0fa25172ca3d --- /dev/null +++ b/eng/scripts/wheel_signing/repackage_signed_wheels.py @@ -0,0 +1,199 @@ +import argparse +import base64 +import csv +import hashlib +import json +import shutil +import tempfile +import zipfile +from pathlib import Path +from typing import Dict, List + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Reinsert signed binaries into unpacked wheel trees and rebuild wheel files " + "with original filenames." + ) + ) + parser.add_argument("--platform", choices=["mac", "windows"], required=True) + parser.add_argument("--work-dir", required=True, help="Working directory created by extract_sign_inputs.py.") + parser.add_argument( + "--signed-input-zip", + default=None, + help="Signed binary zip payload (required for --platform mac).", + ) + parser.add_argument( + "--signed-input-dir", + default=None, + help="Signed binary folder payload (required for --platform windows).", + ) + parser.add_argument("--output-wheels-dir", required=True, help="Directory where rebuilt wheel files are written.") + return parser.parse_args() + + +def validate_args(args: argparse.Namespace) -> None: + if args.platform == "mac": + if not args.signed_input_zip: + raise ValueError("--signed-input-zip is required for --platform mac.") + if args.signed_input_dir: + raise ValueError("--signed-input-dir is not valid for --platform mac.") + else: + if not args.signed_input_dir: + raise ValueError("--signed-input-dir is required for --platform windows.") + if args.signed_input_zip: + raise ValueError("--signed-input-zip is not valid for --platform windows.") + + +def load_manifest(manifest_path: Path) -> Dict: + if not manifest_path.is_file(): + raise FileNotFoundError(f"Manifest not found: {manifest_path}") + with manifest_path.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def reset_dir(path: Path) -> None: + if path.exists(): + shutil.rmtree(path) + path.mkdir(parents=True, exist_ok=True) + + +def get_digest_and_size(file_path: Path) -> List[str]: + content = file_path.read_bytes() + digest = base64.urlsafe_b64encode(hashlib.sha256(content).digest()).decode("ascii").rstrip("=") + return [f"sha256={digest}", str(len(content))] + + +def find_record_path(unpacked_wheel_dir: Path) -> Path: + candidates = sorted(unpacked_wheel_dir.glob("*.dist-info/RECORD")) + if len(candidates) != 1: + raise RuntimeError( + f"Expected exactly one RECORD file under {unpacked_wheel_dir}, found {len(candidates)}." + ) + return candidates[0] + + +def rewrite_record(unpacked_wheel_dir: Path) -> None: + record_path = find_record_path(unpacked_wheel_dir) + record_rel = record_path.relative_to(unpacked_wheel_dir).as_posix() + + rows: List[List[str]] = [] + for file_path in sorted(unpacked_wheel_dir.rglob("*")): + if not file_path.is_file(): + continue + rel = file_path.relative_to(unpacked_wheel_dir).as_posix() + if rel == record_rel: + continue + digest, size = get_digest_and_size(file_path) + rows.append([rel, digest, size]) + + rows.append([record_rel, "", ""]) + + with record_path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.writer(handle, lineterminator="\n") + writer.writerows(rows) + + +def build_wheel(unpacked_wheel_dir: Path, output_wheel_path: Path) -> None: + output_wheel_path.parent.mkdir(parents=True, exist_ok=True) + if output_wheel_path.exists(): + output_wheel_path.unlink() + + rewrite_record(unpacked_wheel_dir) + + with zipfile.ZipFile(output_wheel_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for file_path in sorted(unpacked_wheel_dir.rglob("*")): + if file_path.is_file(): + archive.write(file_path, file_path.relative_to(unpacked_wheel_dir).as_posix()) + + +def main() -> None: + args = parse_args() + validate_args(args) + + work_dir = Path(args.work_dir).resolve() + unpack_root = work_dir / "unpacked" + manifest_path = work_dir / "signing-manifest.json" + output_wheels_dir = Path(args.output_wheels_dir).resolve() + + manifest = load_manifest(manifest_path) + if manifest.get("platform") != args.platform: + raise RuntimeError( + f"Manifest platform '{manifest.get('platform')}' does not match argument platform '{args.platform}'." + ) + + if not unpack_root.is_dir(): + raise FileNotFoundError(f"Unpacked wheel directory not found: {unpack_root}") + + print(f"Platform: {args.platform}") + print(f"Work dir: {work_dir}") + print(f"Manifest: {manifest_path}") + + with tempfile.TemporaryDirectory(prefix="signed-binaries-") as tmpdir: + if args.platform == "mac": + signed_payload_dir = Path(tmpdir) / "signed-payload" + signed_payload_dir.mkdir(parents=True, exist_ok=True) + signed_zip = Path(args.signed_input_zip).resolve() + if not signed_zip.is_file(): + raise FileNotFoundError(f"Signed payload zip not found: {signed_zip}") + with zipfile.ZipFile(signed_zip, "r") as archive: + archive.extractall(signed_payload_dir) + print(f"Signed payload zip: {signed_zip}") + else: + signed_payload_dir = Path(args.signed_input_dir).resolve() + if not signed_payload_dir.is_dir(): + raise FileNotFoundError(f"Signed payload directory not found: {signed_payload_dir}") + print(f"Signed payload dir: {signed_payload_dir}") + + for entry in manifest.get("entries", []): + wheel_filename = entry["wheel_filename"] + unpack_dir = entry["unpack_dir"] + relative_path = entry["relative_path"] + payload_path = entry["payload_path"] + + source_signed_binary = signed_payload_dir / payload_path + target_binary = unpack_root / unpack_dir / relative_path + + if not source_signed_binary.is_file(): + raise FileNotFoundError(f"Signed binary missing: {source_signed_binary}") + if not target_binary.is_file(): + raise FileNotFoundError(f"Target binary missing in unpacked wheel: {target_binary}") + + shutil.copy2(source_signed_binary, target_binary) + print( + "[MAP_REPACKAGE] " + f"payload={payload_path} " + f"target_wheel={wheel_filename} " + f"target_relative_path={relative_path} " + f"signed_source={source_signed_binary} " + f"target_file={target_binary}" + ) + + reset_dir(output_wheels_dir) + + rebuilt_count = 0 + for wheel_info in manifest.get("wheels", []): + wheel_filename = wheel_info["wheel_filename"] + unpack_dir = wheel_info["unpack_dir"] + + unpacked_wheel_dir = unpack_root / unpack_dir + if not unpacked_wheel_dir.is_dir(): + raise FileNotFoundError(f"Unpacked wheel directory missing: {unpacked_wheel_dir}") + + output_wheel_path = output_wheels_dir / wheel_filename + print( + "[REBUILD] " + f"wheel={wheel_filename} " + f"unpacked_dir={unpacked_wheel_dir} " + f"output_wheel={output_wheel_path}" + ) + build_wheel(unpacked_wheel_dir, output_wheel_path) + rebuilt_count += 1 + + print(f"Wheels rebuilt: {rebuilt_count}") + print(f"Output wheels dir: {output_wheels_dir}") + + +if __name__ == "__main__": + main() diff --git a/sdk/storage/azure-storage-extensions/CHANGELOG.md b/sdk/storage/azure-storage-extensions/CHANGELOG.md index 8f529305a53b..ceaea28525d4 100644 --- a/sdk/storage/azure-storage-extensions/CHANGELOG.md +++ b/sdk/storage/azure-storage-extensions/CHANGELOG.md @@ -7,4 +7,5 @@ ## 0.1.0 (2026-07-28) ### Features Added + - Initial release. diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index a05078529031..da3d9c40a91c 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -55,6 +55,9 @@ extends: safeName: azurestoragequeue - name: azure-storage-extensions safeName: azurestorageextensions + # Ships compiled binaries, so its wheels are routed through a dedicated + # Sign_azurestorageextensions stage before release and dev feed publishing. + signBinaries: true triggeringPaths: - /sdk/storage/azure-storage-blob - /sdk/storage/azure-storage-file-datalake