From b9ddbbde36784260c35867086b52049c38ae30c9 Mon Sep 17 00:00:00 2001 From: AravCS Date: Wed, 22 Jul 2026 20:40:56 -0700 Subject: [PATCH 1/2] Add mypy type checking to CI (#226) Signed-off-by: AravCS --- .github/workflows/main.yml | 3 ++ Makefile | 6 ++- mypy.ini | 78 ++++++++++++++++++++++++++++++++++++++ requirements-dev.txt | 1 + setup.cfg | 2 + 5 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 mypy.ini diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8fa22d63b..76e843047 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,6 +50,9 @@ jobs: # - name: Validate code format # run: make check + - name: Check types + run: make mypy + - name: Run tests run: make test env: diff --git a/Makefile b/Makefile index f3e21eda3..34b7d774b 100644 --- a/Makefile +++ b/Makefile @@ -131,6 +131,10 @@ webtest: @echo "-> Run web tests" ${ACTIVATE} ${PYTHON_EXE} -m pytest -vvs -m "webtest" +mypy: + @echo "-> Run mypy type checking" + @${ACTIVATE} mypy + bump: @echo "-> Bump the version" bin/bumpver update --no-fetch --patch @@ -148,4 +152,4 @@ docker-images: @mkdir -p dist/ @docker save postgres vulnerablecode_vulnerablecode nginx | gzip > dist/vulnerablecode-images-`git describe --tags`.tar.gz -.PHONY: virtualenv conf dev envfile install check valid isort clean migrate postgres sqlite run test bump docs docker-images +.PHONY: virtualenv conf dev envfile install check valid isort clean migrate postgres sqlite run test mypy bump docs docker-images diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 000000000..00f68edf4 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,78 @@ +[mypy] +ignore_missing_imports = True +follow_imports = silent +explicit_package_bases = True +namespace_packages = True + +check_untyped_defs = True + +warn_unused_configs = True +warn_redundant_casts = True +warn_unused_ignores = True + +strict_equality = True +warn_return_any = True + +files = + # fetch and parse vulnerability data from upstream sources + vulnerabilities/importers/debian_oval.py, + vulnerabilities/importers/epss.py, + vulnerabilities/importers/project_kb_msr2019.py, + vulnerabilities/importers/redhat.py, + vulnerabilities/importers/suse_scores.py, + vulnerabilities/importers/ubuntu_usn.py, + vulnerabilities/importers/xen.py, + + # shared logic that transforms advisory data after import + vulnerabilities/pipes/apache_kafka.py, + vulnerabilities/pipes/export.py, + vulnerabilities/pipes/extractcode_utils.py, + vulnerabilities/pipes/federatedcode.py, + vulnerabilities/pipes/risk_score.py, + + # public REST API (v2): the project's external contract + vulnerabilities/api_v2.py, + + # CLI entry points for imports, improvers, and scheduling + vulnerabilities/management/commands/create_api_user.py, + vulnerabilities/management/commands/import.py, + vulnerabilities/management/commands/improve.py, + vulnerabilities/management/commands/purl2cpe.py, + vulnerabilities/management/commands/run_scheduler.py, + + # how vulnerabilities are scored, related, and scheduled + vulnerabilities/risk.py, + vulnerabilities/references.py, + vulnerabilities/schedules.py, + vulnerabilities/improvers/vulnerability_status.py, + vulnerabilities/pipelines/v2_importers/collect_fix_commits.py, + + # forms, API pagination, and rate limiting + vulnerabilities/forms.py, + vulnerabilities/pagination.py, + vulnerabilities/throttling.py, + + # supporting utilities and admin config + vulnerabilities/admin.py, + vulnerabilities/rpm_utils.py, + vulnerabilities/weight_config.py, + + # fully clean directories: request middleware and template tags + vulnerabilities/middleware, + vulnerabilities/templatetags, + + # Django project configuration + vulnerablecode/__init__.py, + vulnerablecode/context_processors.py, + vulnerablecode/urls.py, + vulnerablecode/wsgi.py, + + # shared library for deterministic advisory IDs + aboutcode, + + # bundled CLI tool for cross-checking vulnerability sources + vulntotal/__init__.py, + vulntotal/vulntotal_utils.py, + vulntotal/ecosystem, + vulntotal/datasources/safetydb.py, + vulntotal/tests, diff --git a/requirements-dev.txt b/requirements-dev.txt index 75966c993..cf4ce08ab 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -80,6 +80,7 @@ MarkupSafe==3.0.3 matplotlib-inline==0.2.1 mdurl==0.1.2 more-itertools==10.8.0 +mypy==2.1.0 mypy_extensions==1.1.0 nh3==0.3.4 packageurl-python==0.17.6 diff --git a/setup.cfg b/setup.cfg index 83a4bcbae..c0fed87be 100644 --- a/setup.cfg +++ b/setup.cfg @@ -127,6 +127,8 @@ dev = # Tests pytest==7.1.1 pytest-django==4.5.2 + # Type checking + mypy==2.1.0 # misc ipython==8.10.0 From 7e844f00405d26857b3cc6052011ef2ffb45ace6 Mon Sep 17 00:00:00 2001 From: AravCS Date: Wed, 12 Aug 2026 16:58:08 -0700 Subject: [PATCH 2/2] using an exclude list Signed-off-by: AravCS --- Makefile | 2 + mypy.ini | 239 +++++++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 189 insertions(+), 52 deletions(-) diff --git a/Makefile b/Makefile index 34b7d774b..c4f9b1bac 100644 --- a/Makefile +++ b/Makefile @@ -93,6 +93,8 @@ check: @${ACTIVATE} isort --check-only . @echo "-> Run black validation" @${ACTIVATE} black --check ${BLACK_ARGS} + @echo "-> Run mypy type checking" + @${ACTIVATE} mypy clean: @echo "-> Clean the Python env" diff --git a/mypy.ini b/mypy.ini index c4c3ab89b..616d0df07 100644 --- a/mypy.ini +++ b/mypy.ini @@ -14,57 +14,192 @@ strict_equality = True warn_return_any = True files = - # fetch and parse vulnerability data from upstream sources - vulnerabilities/importers/debian_oval.py, - vulnerabilities/importers/epss.py, - vulnerabilities/importers/project_kb_msr2019.py, - vulnerabilities/importers/redhat.py, - vulnerabilities/importers/suse_scores.py, - vulnerabilities/importers/ubuntu_usn.py, - vulnerabilities/importers/xen.py, - - # shared logic that transforms advisory data after import - vulnerabilities/pipes/apache_kafka.py, - vulnerabilities/pipes/export.py, - vulnerabilities/pipes/extractcode_utils.py, - vulnerabilities/pipes/federatedcode.py, - - # CLI entry points for imports, improvers, and scheduling - vulnerabilities/management/commands/create_api_user.py, - vulnerabilities/management/commands/import.py, - vulnerabilities/management/commands/improve.py, - vulnerabilities/management/commands/run_scheduler.py, - - # how vulnerabilities are scored, related, and scheduled - vulnerabilities/risk.py, - vulnerabilities/references.py, - vulnerabilities/schedules.py, - vulnerabilities/pipelines/v2_importers/collect_fix_commits.py, - - # API pagination and rate limiting - vulnerabilities/pagination.py, - vulnerabilities/throttling.py, - - # supporting utilities - vulnerabilities/rpm_utils.py, - vulnerabilities/weight_config.py, - - # fully clean directories: request middleware and template tags - vulnerabilities/middleware, - vulnerabilities/templatetags, - - # Django project configuration - vulnerablecode/__init__.py, - vulnerablecode/context_processors.py, - vulnerablecode/urls.py, - vulnerablecode/wsgi.py, - - # shared library for deterministic advisory IDs + vulnerabilities, + vulnerablecode, aboutcode, + vulntotal, - # bundled CLI tool for cross-checking vulnerability sources - vulntotal/__init__.py, - vulntotal/vulntotal_utils.py, - vulntotal/ecosystem, - vulntotal/datasources/safetydb.py, - vulntotal/tests, +exclude = (?x)( + # permanently excluded, migrations are auto-generated + ^vulnerabilities/migrations/ + + # everything below is excluded only until it is typed + + # directories with no files typed + | ^vulnerabilities/pipelines/v2_improvers/ + | ^vulnerabilities/tests/pipelines/exporters/ + + # top level files in pipelines/ not typed, subdirectories are listed below + | ^vulnerabilities/pipelines/[^/]+\.py$ + + # core app modules not yet typed + | ^vulnerabilities/api\.py$ + | ^vulnerabilities/api_extension\.py$ + | ^vulnerabilities/api_v3\.py$ + | ^vulnerabilities/import_runner\.py$ + | ^vulnerabilities/importer\.py$ + | ^vulnerabilities/improve_runner\.py$ + | ^vulnerabilities/improver\.py$ + | ^vulnerabilities/lib_oval\.py$ + | ^vulnerabilities/models\.py$ + | ^vulnerabilities/oval_parser\.py$ + | ^vulnerabilities/severity_systems\.py$ + | ^vulnerabilities/tasks\.py$ + | ^vulnerabilities/utils\.py$ + | ^vulnerabilities/views\.py$ + + # importer implementations not yet typed + | ^vulnerabilities/importers/apache_httpd\.py$ + | ^vulnerabilities/importers/apache_kafka\.py$ + | ^vulnerabilities/importers/apache_tomcat\.py$ + | ^vulnerabilities/importers/archlinux\.py$ + | ^vulnerabilities/importers/curl\.py$ + | ^vulnerabilities/importers/debian\.py$ + | ^vulnerabilities/importers/elixir_security\.py$ + | ^vulnerabilities/importers/fireeye\.py$ + | ^vulnerabilities/importers/gentoo\.py$ + | ^vulnerabilities/importers/github_osv\.py$ + | ^vulnerabilities/importers/gsd\.py$ + | ^vulnerabilities/importers/istio\.py$ + | ^vulnerabilities/importers/kaybee\.py$ + | ^vulnerabilities/importers/mattermost\.py$ + | ^vulnerabilities/importers/mozilla\.py$ + | ^vulnerabilities/importers/openssl\.py$ + | ^vulnerabilities/importers/oss_fuzz\.py$ + | ^vulnerabilities/importers/osv\.py$ + | ^vulnerabilities/importers/postgresql\.py$ + | ^vulnerabilities/importers/retiredotnet\.py$ + | ^vulnerabilities/importers/ruby\.py$ + | ^vulnerabilities/importers/rust\.py$ + | ^vulnerabilities/importers/suse_backports\.py$ + | ^vulnerabilities/importers/suse_oval\.py$ + | ^vulnerabilities/importers/vulnrichment\.py$ + + # pipeline modules not yet typed (v2 importers/exporters) + | ^vulnerabilities/pipelines/exporters/federate_vulnerabilities\.py$ + | ^vulnerabilities/pipelines/v2_importers/alpine_linux_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/aosp_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/apache_httpd_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/apache_kafka_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/apache_tomcat_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/archlinux_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/curl_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/debian_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/elixir_security_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/epss_importer_v2\.py$ + | ^vulnerabilities/pipelines/v2_importers/fireeye_importer_v2\.py$ + | ^vulnerabilities/pipelines/v2_importers/gentoo_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/github_osv_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/gitlab_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/glibc_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/istio_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/linux_kernel_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/mattermost_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/mozilla_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/nginx_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/npm_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/nvd_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/openssl_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/oss_fuzz\.py$ + | ^vulnerabilities/pipelines/v2_importers/postgresql_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/project_kb_msr2019_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/project_kb_statements_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/pypa_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/pysec_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/redhat_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/retiredotnet_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/ruby_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/suse_score_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/ubuntu_osv_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/vulnrichment_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/xen_importer\.py$ + + # data-transform pipes not yet typed + | ^vulnerabilities/pipes/advisory\.py$ + | ^vulnerabilities/pipes/fetchcode_utils\.py$ + | ^vulnerabilities/pipes/group_advisories\.py$ + | ^vulnerabilities/pipes/openssl\.py$ + | ^vulnerabilities/pipes/osv_v2\.py$ + | ^vulnerabilities/pipes/univers_utils\.py$ + | ^vulnerabilities/pipes/vcs_collector_utils\.py$ + + # improvers not yet typed + | ^vulnerabilities/improvers/add_missing_refid\.py$ + | ^vulnerabilities/improvers/default\.py$ + | ^vulnerabilities/improvers/valid_versions\.py$ + + # management commands not yet typed + | ^vulnerabilities/management/commands/commit_export\.py$ + | ^vulnerabilities/management/commands/export\.py$ + + # test suite not yet typed + | ^vulnerabilities/tests/pipelines/test_base_pipeline\.py$ + | ^vulnerabilities/tests/pipelines/test_compute_advisory_todo\.py$ + | ^vulnerabilities/tests/pipelines/test_npm_importer_pipeline\.py$ + | ^vulnerabilities/tests/pipelines/test_pipeline_id\.py$ + | ^vulnerabilities/tests/pipelines/test_pypa_importer_pipeline\.py$ + | ^vulnerabilities/tests/pipelines/test_pysec_importer_pipeline\.py$ + | ^vulnerabilities/tests/pipelines/test_remove_duplicate_advisories\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_apache_httpd_importer_pipeline_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_apache_kafka_importer\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_apache_tomcat_importer_pipeline\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_collect_fix_commit\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_debian_importer\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_elixir_security_importer_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_github_osv_importer_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_gitlab_importer_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_istio_importer_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_mattermost_importer_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_mozilla_importer_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_npm_importer_pipeline_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_openssl_importer\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_postgresql_importer_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_pypa_importer_pipeline_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_pysec_importer_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_ubuntu_osv_importer\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_xen_importer_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_improvers/test_unfurl_version_range\.py$ + | ^vulnerabilities/tests/pipes/test_apache_kafka\.py$ + | ^vulnerabilities/tests/pipes/test_osv_v2\.py$ + | ^vulnerabilities/tests/pipes/test_vulnerablecode_importer_pipeline_v2\.py$ + | ^vulnerabilities/tests/test_affected_package\.py$ + | ^vulnerabilities/tests/test_api_extension\.py$ + | ^vulnerabilities/tests/test_api_v3\.py$ + | ^vulnerabilities/tests/test_changelog\.py$ + | ^vulnerabilities/tests/test_data_migrations\.py$ + | ^vulnerabilities/tests/test_data_source\.py$ + | ^vulnerabilities/tests/test_debian_oval\.py$ + | ^vulnerabilities/tests/test_default_improver\.py$ + | ^vulnerabilities/tests/test_example\.py$ + | ^vulnerabilities/tests/test_github_osv\.py$ + | ^vulnerabilities/tests/test_import_command\.py$ + | ^vulnerabilities/tests/test_import_runner\.py$ + | ^vulnerabilities/tests/test_improve_command\.py$ + | ^vulnerabilities/tests/test_improve_runner\.py$ + | ^vulnerabilities/tests/test_models\.py$ + | ^vulnerabilities/tests/test_openssl\.py$ + | ^vulnerabilities/tests/test_oss_fuzz\.py$ + | ^vulnerabilities/tests/test_osv\.py$ + | ^vulnerabilities/tests/test_retiredotnet\.py$ + | ^vulnerabilities/tests/test_rust\.py$ + | ^vulnerabilities/tests/test_suse\.py$ + | ^vulnerabilities/tests/test_suse_oval\.py$ + | ^vulnerabilities/tests/test_tasks\.py$ + | ^vulnerabilities/tests/test_utils\.py$ + + # vulntotal data sources not yet typed + | ^vulntotal/datasources/deps\.py$ + | ^vulntotal/datasources/github\.py$ + | ^vulntotal/datasources/gitlab\.py$ + | ^vulntotal/datasources/oss_index\.py$ + | ^vulntotal/datasources/osv\.py$ + | ^vulntotal/datasources/snyk\.py$ + | ^vulntotal/datasources/vulnerablecode\.py$ + + # vulntotal core not yet typed + | ^vulntotal/validator\.py$ + | ^vulntotal/vulntotal_cli\.py$ + + # Django settings not yet typed + | ^vulnerablecode/settings\.py$ + ) \ No newline at end of file