diff --git a/.github/workflows/test-providers.yml b/.github/workflows/test-providers.yml index db1f31f2453aa..671497af234dd 100644 --- a/.github/workflows/test-providers.yml +++ b/.github/workflows/test-providers.yml @@ -128,6 +128,10 @@ jobs: run: > breeze release-management prepare-task-sdk-distributions --distribution-format ${{ matrix.package-format }} + - name: "Prepare airflow-ctl package: ${{ matrix.package-format }}" + run: > + breeze release-management prepare-airflow-ctl-distributions + --distribution-format ${{ matrix.package-format }} - name: "Verify ${{ matrix.package-format }} packages with twine" run: | uv tool uninstall twine || true diff --git a/Dockerfile b/Dockerfile index 1afcd7320525b..9d4e63d74ceb8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1048,8 +1048,11 @@ function install_airflow_and_providers_from_docker_context_files(){ install_airflow_core_distribution=("apache-airflow-core==${AIRFLOW_VERSION}") fi - # Find Provider/TaskSDK/CTL distributions in docker-context files - readarray -t airflow_distributions< <(python /scripts/docker/get_distribution_specs.py /docker-context-files/apache?airflow?{providers,task?sdk,airflowctl}*.{whl,tar.gz} 2>/dev/null || true) + # Find Provider/TaskSDK/CTL distributions in docker-context files. + # NOTE: the ctl wheel is named ``apache_airflow_ctl-*.whl`` (distribution + # ``apache-airflow-ctl``), not ``apache_airflow_airflowctl-*.whl`` — the + # glob must say ``ctl``, not ``airflowctl``. + readarray -t airflow_distributions< <(python /scripts/docker/get_distribution_specs.py /docker-context-files/apache?airflow?{providers,task?sdk,ctl}*.{whl,tar.gz} 2>/dev/null || true) echo echo "${COLOR_BLUE}Found provider distributions in docker-context-files folder: ${airflow_distributions[*]}${COLOR_RESET}" echo diff --git a/airflow-core/pyproject.toml b/airflow-core/pyproject.toml index 3158a36124675..783ac958fa57b 100644 --- a/airflow-core/pyproject.toml +++ b/airflow-core/pyproject.toml @@ -150,7 +150,7 @@ dependencies = [ "universal-pathlib>=0.3.8", "uuid6>=2024.7.10", "apache-airflow-task-sdk<1.4.0,>=1.3.0", - "apache-airflow-ctl<0.1.5,>=0.1.4", + "apache-airflow-ctl<0.1.6,>=0.1.5", # pre-installed providers "apache-airflow-providers-common-compat>=1.7.4", "apache-airflow-providers-common-io>=1.6.3", diff --git a/airflow-ctl/src/airflowctl/__init__.py b/airflow-ctl/src/airflowctl/__init__.py index d2dd19e81e26f..a085e738d8394 100644 --- a/airflow-ctl/src/airflowctl/__init__.py +++ b/airflow-ctl/src/airflowctl/__init__.py @@ -19,4 +19,4 @@ __path__ = __import__("pkgutil").extend_path(__path__, __name__) -__version__ = "0.1.4" +__version__ = "0.1.5" diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh index 5ce45267f8fbd..089b0f2d74bd4 100644 --- a/scripts/docker/install_from_docker_context_files.sh +++ b/scripts/docker/install_from_docker_context_files.sh @@ -85,8 +85,11 @@ function install_airflow_and_providers_from_docker_context_files(){ install_airflow_core_distribution=("apache-airflow-core==${AIRFLOW_VERSION}") fi - # Find Provider/TaskSDK/CTL distributions in docker-context files - readarray -t airflow_distributions< <(python /scripts/docker/get_distribution_specs.py /docker-context-files/apache?airflow?{providers,task?sdk,airflowctl}*.{whl,tar.gz} 2>/dev/null || true) + # Find Provider/TaskSDK/CTL distributions in docker-context files. + # NOTE: the ctl wheel is named ``apache_airflow_ctl-*.whl`` (distribution + # ``apache-airflow-ctl``), not ``apache_airflow_airflowctl-*.whl`` — the + # glob must say ``ctl``, not ``airflowctl``. + readarray -t airflow_distributions< <(python /scripts/docker/get_distribution_specs.py /docker-context-files/apache?airflow?{providers,task?sdk,ctl}*.{whl,tar.gz} 2>/dev/null || true) echo echo "${COLOR_BLUE}Found provider distributions in docker-context-files folder: ${airflow_distributions[*]}${COLOR_RESET}" echo diff --git a/scripts/in_container/install_airflow_and_providers.py b/scripts/in_container/install_airflow_and_providers.py index c8223f3eeff10..4f203e8c6c3f0 100755 --- a/scripts/in_container/install_airflow_and_providers.py +++ b/scripts/in_container/install_airflow_and_providers.py @@ -1183,9 +1183,11 @@ def _install_airflow_ctl_with_constraints(installation_spec: InstallationSpec, g "pip", "install", ] - # if airflow is also being installed we should add airflow to the base_install_providers_cmd - # to avoid accidentally upgrading airflow to a version that is different from installed in the - # previous step + # Install the ctl distribution itself, plus pin airflow alongside (if it's being + # installed) so the resolver does not accidentally upgrade airflow to a version + # different from the one installed in the previous step. + if installation_spec.airflow_ctl_distribution: + base_install_airflow_ctl_cmd.append(installation_spec.airflow_ctl_distribution) if installation_spec.airflow_distribution: base_install_airflow_ctl_cmd.append(installation_spec.airflow_distribution) install_airflow_ctl_cmd = base_install_airflow_ctl_cmd.copy() @@ -1246,6 +1248,13 @@ def _install_only_airflow_airflow_core_task_sdk_with_constraints( f"{installation_spec.airflow_task_sdk_distribution} with constraints" ) console.print() + if installation_spec.airflow_ctl_distribution: + base_install_airflow_cmd.append(installation_spec.airflow_ctl_distribution) + console.print( + f"\n[bright_blue]Installing airflow ctl distribution alongside core: " + f"{installation_spec.airflow_ctl_distribution}" + ) + console.print() install_airflow_cmd = base_install_airflow_cmd.copy() if installation_spec.airflow_constraints_location: console.print(f"[bright_blue]Use constraints: {installation_spec.airflow_constraints_location}")