diff --git a/eng/ci_tools.txt b/eng/ci_tools.txt index 85779e81f7e9..7515961afc55 100644 --- a/eng/ci_tools.txt +++ b/eng/ci_tools.txt @@ -30,4 +30,4 @@ urllib3==2.2.3 six==1.17.0 # local dev packages -./eng/tools/azure-sdk-tools +./eng/tools/azure-sdk-tools[ghtools] diff --git a/eng/pipelines/templates/stages/python-analyze-weekly.yml b/eng/pipelines/templates/stages/python-analyze-weekly.yml index ab1d0522d275..51469f807f7d 100644 --- a/eng/pipelines/templates/stages/python-analyze-weekly.yml +++ b/eng/pipelines/templates/stages/python-analyze-weekly.yml @@ -36,11 +36,11 @@ stages: displayName: 'Run Pylint Next' continueOnError: true inputs: - scriptPath: 'scripts/devops_tasks/dispatch_tox.py' + scriptPath: 'eng/scripts/dispatch_checks.py' arguments: >- ${{ parameters.BuildTargetingString }} --service="${{ parameters.ServiceDirectory }}" - --toxenv="next-pylint" + --checks="next-pylint" --disablecov --filter-type="Omit_management" env: @@ -50,11 +50,11 @@ stages: displayName: 'Run MyPy Next' continueOnError: true inputs: - scriptPath: 'scripts/devops_tasks/dispatch_tox.py' + scriptPath: 'eng/scripts/dispatch_checks.py' arguments: >- ${{ parameters.BuildTargetingString }} --service="${{ parameters.ServiceDirectory }}" - --toxenv="next-mypy" + --checks="next-mypy" --disablecov env: GH_TOKEN: $(azuresdk-github-pat) @@ -63,11 +63,11 @@ stages: displayName: 'Run Pyright Next' continueOnError: true inputs: - scriptPath: 'scripts/devops_tasks/dispatch_tox.py' + scriptPath: 'eng/scripts/dispatch_checks.py' arguments: >- ${{ parameters.BuildTargetingString }} --service="${{ parameters.ServiceDirectory }}" - --toxenv="next-pyright" + --checks="next-pyright" --disablecov env: GH_TOKEN: $(azuresdk-github-pat) @@ -76,11 +76,11 @@ stages: displayName: 'Run Ruff' continueOnError: true inputs: - scriptPath: 'scripts/devops_tasks/dispatch_tox.py' + scriptPath: 'eng/scripts/dispatch_checks.py' arguments: >- ${{ parameters.BuildTargetingString }} --service="${{ parameters.ServiceDirectory }}" - --toxenv="ruff" + --checks="ruff" --disablecov env: GH_TOKEN: $(azuresdk-github-pat) @@ -115,10 +115,10 @@ stages: displayName: 'Generate Docs Next' continueOnError: true inputs: - scriptPath: 'scripts/devops_tasks/dispatch_tox.py' + scriptPath: 'eng/scripts/dispatch_checks.py' arguments: >- ${{ parameters.BuildTargetingString }} --service="${{ parameters.ServiceDirectory }}" - --toxenv="next-sphinx" + --checks="next-sphinx" env: GH_TOKEN: $(azuresdk-github-pat) diff --git a/eng/tools/azure-sdk-tools/azpysdk/Check.py b/eng/tools/azure-sdk-tools/azpysdk/Check.py index 2c87589f91e4..9573f5054ab5 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/Check.py +++ b/eng/tools/azure-sdk-tools/azpysdk/Check.py @@ -1,6 +1,7 @@ import abc import os import argparse +import re import traceback import sys import shutil @@ -92,7 +93,7 @@ def create_venv(self, isolate: bool, venv_location: str) -> str: if prebuilt_whl: install_location = os.path.join(wheel_dir, prebuilt_whl) - install_into_venv(venv_location, [f"{install_location}[build]"], REPO_ROOT) + install_into_venv(venv_location, [install_location], REPO_ROOT) else: logger.error( "Falling back to manual build and install of azure-sdk-tools into isolated env," @@ -313,3 +314,24 @@ def _build_pytest_args_base( def _build_pytest_args(self, package_dir: str, args: argparse.Namespace) -> List[str]: """Build pytest args for a package directory.""" return self._build_pytest_args_base(package_dir, args) + + def get_check_version(self, executable: str, module_name: str) -> Optional[str]: + """Get the version of a tool installed in the virtual environment. + + Runs `` -m --version`` and extracts the first + semver-style version string from the output. + + :param executable: Path to the Python executable in the target venv. + :param module_name: The module to query (e.g. "pylint", "mypy"). + :returns: The version string, or None if it could not be determined. + """ + try: + result = subprocess.run( + [executable, "-m", module_name, "--version"], + capture_output=True, + ) + version_output = result.stdout.rstrip().decode("utf-8") + matches = re.findall(r"(\d+\.\d+\.\d+)", version_output) + return matches[0] if matches else None + except Exception: + return None diff --git a/eng/tools/azure-sdk-tools/azpysdk/mypy.py b/eng/tools/azure-sdk-tools/azpysdk/mypy.py index 5ca2fa5a7c42..64b759a8ad53 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/mypy.py +++ b/eng/tools/azure-sdk-tools/azpysdk/mypy.py @@ -7,9 +7,7 @@ from subprocess import CalledProcessError, check_call from .Check import Check -from ci_tools.parsing import ParsedSetup from ci_tools.functions import install_into_venv -from ci_tools.scenario.generation import create_package_and_install from ci_tools.variables import in_ci, set_envvar_defaults from ci_tools.environment_exclusions import is_check_enabled, is_typing_ignored from ci_tools.logging import logger @@ -21,7 +19,6 @@ "types-requests==2.31.0.6", "types-six==1.16.21.9", "types-redis==4.6.0.7", - "PyGitHub>=1.59.0", ] @@ -128,11 +125,14 @@ def run(self, args: argparse.Namespace) -> int: results.append(sample_error.returncode) if args.next and in_ci() and not is_typing_ignored(package_name): - from gh_tools.vnext_issue_creator import create_vnext_issue, close_vnext_issue - if src_code_error or sample_code_error: - create_vnext_issue(package_dir, "mypy") + from gh_tools.vnext_issue_creator import create_vnext_issue + + check_version = self.get_check_version(executable, "mypy") + create_vnext_issue(package_dir, "mypy", check_version) else: + from gh_tools.vnext_issue_creator import close_vnext_issue + close_vnext_issue(package_name, "mypy") return max(results) if results else 0 diff --git a/eng/tools/azure-sdk-tools/azpysdk/pylint.py b/eng/tools/azure-sdk-tools/azpysdk/pylint.py index 77d6d2a16702..5815d8bdb30e 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/pylint.py +++ b/eng/tools/azure-sdk-tools/azpysdk/pylint.py @@ -3,11 +3,10 @@ import sys from typing import Optional, List -import subprocess from subprocess import CalledProcessError, check_call from .Check import Check -from ci_tools.functions import install_into_venv, get_pip_command +from ci_tools.functions import install_into_venv from ci_tools.scenario.generation import create_package_and_install from ci_tools.variables import discover_repo_root, in_ci, set_envvar_defaults from ci_tools.environment_exclusions import is_check_enabled @@ -15,7 +14,6 @@ REPO_ROOT = discover_repo_root() PYLINT_VERSION = "3.2.7" -PYGITHUB_VERSION = "1.59.0" class pylint(Check): @@ -42,6 +40,7 @@ def run(self, args: argparse.Namespace) -> int: logger.info("Running pylint check...") set_envvar_defaults() + targeted = self.get_targeted_directories(args) results: List[int] = [] @@ -53,6 +52,7 @@ def run(self, args: argparse.Namespace) -> int: package_name = parsed.name executable, staging_directory = self.get_executable(args.isolate, args.command, sys.executable, package_dir) logger.info(f"Processing {package_name} for pylint check") + package_failed = False # install dependencies self.install_dev_reqs(executable, args, package_dir) @@ -85,7 +85,7 @@ def run(self, args: argparse.Namespace) -> int: try: if args.next: # use latest version of pylint - install_into_venv(executable, ["pylint", f"PyGithub=={PYGITHUB_VERSION}"], package_dir) + install_into_venv(executable, ["pylint"], package_dir) else: install_into_venv(executable, [f"pylint=={PYLINT_VERSION}"], package_dir) except CalledProcessError as e: @@ -139,6 +139,7 @@ def run(self, args: argparse.Namespace) -> int: ) ) results.append(e.returncode) + package_failed = True # Run pylint on tests and samples with appropriate pylintrc if they exist and next pylint is being used if args.next: @@ -178,6 +179,7 @@ def run(self, args: argparse.Namespace) -> int: ) ) results.append(e.returncode) + package_failed = True # Run samples with samples_pylintrc if os.path.exists(samples_dir): @@ -212,15 +214,17 @@ def run(self, args: argparse.Namespace) -> int: ) ) results.append(e.returncode) - - if args.next and in_ci() and any(result > 0 for result in results): - from gh_tools.vnext_issue_creator import create_vnext_issue - - create_vnext_issue(package_dir, "pylint") + package_failed = True if args.next and in_ci(): - from gh_tools.vnext_issue_creator import close_vnext_issue + if package_failed: + from gh_tools.vnext_issue_creator import create_vnext_issue + + check_version = self.get_check_version(executable, "pylint") + create_vnext_issue(package_dir, "pylint", check_version) + else: + from gh_tools.vnext_issue_creator import close_vnext_issue - close_vnext_issue(package_name, "pylint") + close_vnext_issue(package_name, "pylint") return max(results) if results else 0 diff --git a/eng/tools/azure-sdk-tools/azpysdk/pyright.py b/eng/tools/azure-sdk-tools/azpysdk/pyright.py index 11613629f531..2be47bacc848 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/pyright.py +++ b/eng/tools/azure-sdk-tools/azpysdk/pyright.py @@ -4,12 +4,11 @@ import sys from typing import Optional, List -from subprocess import CalledProcessError, check_call +from subprocess import CalledProcessError from .Check import Check from ci_tools.functions import install_into_venv -from ci_tools.variables import in_ci, set_envvar_defaults -from ci_tools.variables import discover_repo_root +from ci_tools.variables import in_ci, set_envvar_defaults, discover_repo_root from ci_tools.environment_exclusions import is_check_enabled, is_typing_ignored from ci_tools.scenario.generation import create_package_and_install @@ -68,6 +67,7 @@ def run(self, args: argparse.Namespace) -> int: logger.info("Running pyright check...") set_envvar_defaults() + targeted = self.get_targeted_directories(args) results: List[int] = [] @@ -144,19 +144,20 @@ def run(self, args: argparse.Namespace) -> int: if ( args.next and in_ci() - and is_check_enabled(args.target_package, "pyright") + and is_check_enabled(package_dir, "pyright") and not is_typing_ignored(package_name) ): from gh_tools.vnext_issue_creator import create_vnext_issue - create_vnext_issue(package_dir, "pyright") + check_version = self.get_check_version(executable, "pyright") + create_vnext_issue(package_dir, "pyright", check_version) print("See https://aka.ms/python/typing-guide for information.\n\n") results.append(1) + else: + if args.next and in_ci() and not is_typing_ignored(package_name): + from gh_tools.vnext_issue_creator import close_vnext_issue - if args.next and in_ci() and not is_typing_ignored(package_name): - from gh_tools.vnext_issue_creator import close_vnext_issue - - close_vnext_issue(package_name, "pyright") + close_vnext_issue(package_name, "pyright") return max(results) if results else 0 diff --git a/eng/tools/azure-sdk-tools/azpysdk/sphinx.py b/eng/tools/azure-sdk-tools/azpysdk/sphinx.py index b80bf5f797fa..53c91a6b8451 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/sphinx.py +++ b/eng/tools/azure-sdk-tools/azpysdk/sphinx.py @@ -13,9 +13,7 @@ from .Check import Check from ci_tools.functions import install_into_venv, unzip_file_to_directory from ci_tools.scenario.generation import create_package_and_install -from ci_tools.variables import in_ci, set_envvar_defaults -from ci_tools.variables import discover_repo_root -from ci_tools.variables import in_analyze_weekly +from ci_tools.variables import in_ci, set_envvar_defaults, discover_repo_root, in_analyze_weekly from ci_tools.logging import logger @@ -24,7 +22,6 @@ SPHINX_RTD_THEME_VERSION = "3.0.2" MYST_PARSER_VERSION = "4.0.1" SPHINX_CONTRIB_JQUERY_VERSION = "4.1" -PYGITHUB_VERSION = "1.59.0" RST_EXTENSION_FOR_INDEX = """ @@ -258,7 +255,6 @@ def run(self, args: argparse.Namespace) -> int: "sphinx_rtd_theme", "myst_parser", "sphinxcontrib-jquery", - f"PyGithub=={PYGITHUB_VERSION}", ], package_dir, ) @@ -291,22 +287,20 @@ def run(self, args: argparse.Namespace) -> int: # run apidoc output_dir = os.path.join(staging_directory, f"{UNZIPPPED_DIR_NAME}/{DOCGEN_DIR_NAME}") if is_mgmt_package(package_name): - results.append(self.mgmt_apidoc(output_dir, package_dir, executable)) + apidoc_result = self.mgmt_apidoc(output_dir, package_dir, executable) else: - results.append(self.sphinx_apidoc(staging_directory, parsed.namespace, executable)) + apidoc_result = self.sphinx_apidoc(staging_directory, parsed.namespace, executable) + results.append(apidoc_result) # build # Only data-plane libraries run strict sphinx at the moment fail_on_warning = not is_mgmt_package(package_name) - results.append( - # doc_folder = source - # site_folder = output - self.sphinx_build(package_dir, doc_folder, site_folder, fail_on_warning, executable) - ) + build_result = self.sphinx_build(package_dir, doc_folder, site_folder, fail_on_warning, executable) + results.append(build_result) if in_ci() or args.in_ci: move_output_and_compress(site_folder, package_dir, package_name) - if in_analyze_weekly(): + if in_analyze_weekly() and apidoc_result == 0 and build_result == 0: from gh_tools.vnext_issue_creator import close_vnext_issue close_vnext_issue(package_name, "sphinx") @@ -342,7 +336,8 @@ def sphinx_build( if in_analyze_weekly(): from gh_tools.vnext_issue_creator import create_vnext_issue - create_vnext_issue(package_dir, "sphinx") + check_version = self.get_check_version(executable, "sphinx") + create_vnext_issue(package_dir, "sphinx", check_version) return 1 return 0 diff --git a/eng/tools/azure-sdk-tools/gh_tools/vnext_issue_creator.py b/eng/tools/azure-sdk-tools/gh_tools/vnext_issue_creator.py index 580c285645cd..671cc4f95c9d 100644 --- a/eng/tools/azure-sdk-tools/gh_tools/vnext_issue_creator.py +++ b/eng/tools/azure-sdk-tools/gh_tools/vnext_issue_creator.py @@ -5,6 +5,7 @@ # This script is used to create issues for client libraries failing the vnext of mypy, pyright, and pylint. from __future__ import annotations +from typing import Optional import sys import os @@ -140,7 +141,7 @@ def get_labels(package_name: str, service: str) -> tuple[list[str], list[str]]: return labels, assignees -def create_vnext_issue(package_dir: str, check_type: CHECK_TYPE) -> None: +def create_vnext_issue(package_dir: str, check_type: CHECK_TYPE, check_version: Optional[str] = None) -> None: """This is called when a client library fails a vnext check. An issue is created with the details or an existing issue is updated with the latest information.""" @@ -156,7 +157,7 @@ def create_vnext_issue(package_dir: str, check_type: CHECK_TYPE) -> None: issues = repo.get_issues(state="open", labels=[check_type], creator="azure-sdk") vnext_issue = [issue for issue in issues if issue.title.split("needs")[0].strip() == package_name] - version = get_version_running(check_type) + version = check_version or get_version_running(check_type) build_link = get_build_link(check_type) merge_date = get_date_for_version_bump(today) error_type = "linting" if check_type == "pylint" else "docstring" if check_type == "sphinx" else "typing"