diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8fa22d63b..76e843047 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,6 +50,9 @@ jobs: # - name: Validate code format # run: make check + - name: Check types + run: make mypy + - name: Run tests run: make test env: diff --git a/Makefile b/Makefile index f3e21eda3..c4f9b1bac 100644 --- a/Makefile +++ b/Makefile @@ -93,6 +93,8 @@ check: @${ACTIVATE} isort --check-only . @echo "-> Run black validation" @${ACTIVATE} black --check ${BLACK_ARGS} + @echo "-> Run mypy type checking" + @${ACTIVATE} mypy clean: @echo "-> Clean the Python env" @@ -131,6 +133,10 @@ webtest: @echo "-> Run web tests" ${ACTIVATE} ${PYTHON_EXE} -m pytest -vvs -m "webtest" +mypy: + @echo "-> Run mypy type checking" + @${ACTIVATE} mypy + bump: @echo "-> Bump the version" bin/bumpver update --no-fetch --patch @@ -148,4 +154,4 @@ docker-images: @mkdir -p dist/ @docker save postgres vulnerablecode_vulnerablecode nginx | gzip > dist/vulnerablecode-images-`git describe --tags`.tar.gz -.PHONY: virtualenv conf dev envfile install check valid isort clean migrate postgres sqlite run test bump docs docker-images +.PHONY: virtualenv conf dev envfile install check valid isort clean migrate postgres sqlite run test mypy bump docs docker-images diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 000000000..616d0df07 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,205 @@ +[mypy] +ignore_missing_imports = True +follow_imports = silent +explicit_package_bases = True +namespace_packages = True + +check_untyped_defs = True + +warn_unused_configs = True +warn_redundant_casts = True +warn_unused_ignores = True + +strict_equality = True +warn_return_any = True + +files = + vulnerabilities, + vulnerablecode, + aboutcode, + vulntotal, + +exclude = (?x)( + # permanently excluded, migrations are auto-generated + ^vulnerabilities/migrations/ + + # everything below is excluded only until it is typed + + # directories with no files typed + | ^vulnerabilities/pipelines/v2_improvers/ + | ^vulnerabilities/tests/pipelines/exporters/ + + # top level files in pipelines/ not typed, subdirectories are listed below + | ^vulnerabilities/pipelines/[^/]+\.py$ + + # core app modules not yet typed + | ^vulnerabilities/api\.py$ + | ^vulnerabilities/api_extension\.py$ + | ^vulnerabilities/api_v3\.py$ + | ^vulnerabilities/import_runner\.py$ + | ^vulnerabilities/importer\.py$ + | ^vulnerabilities/improve_runner\.py$ + | ^vulnerabilities/improver\.py$ + | ^vulnerabilities/lib_oval\.py$ + | ^vulnerabilities/models\.py$ + | ^vulnerabilities/oval_parser\.py$ + | ^vulnerabilities/severity_systems\.py$ + | ^vulnerabilities/tasks\.py$ + | ^vulnerabilities/utils\.py$ + | ^vulnerabilities/views\.py$ + + # importer implementations not yet typed + | ^vulnerabilities/importers/apache_httpd\.py$ + | ^vulnerabilities/importers/apache_kafka\.py$ + | ^vulnerabilities/importers/apache_tomcat\.py$ + | ^vulnerabilities/importers/archlinux\.py$ + | ^vulnerabilities/importers/curl\.py$ + | ^vulnerabilities/importers/debian\.py$ + | ^vulnerabilities/importers/elixir_security\.py$ + | ^vulnerabilities/importers/fireeye\.py$ + | ^vulnerabilities/importers/gentoo\.py$ + | ^vulnerabilities/importers/github_osv\.py$ + | ^vulnerabilities/importers/gsd\.py$ + | ^vulnerabilities/importers/istio\.py$ + | ^vulnerabilities/importers/kaybee\.py$ + | ^vulnerabilities/importers/mattermost\.py$ + | ^vulnerabilities/importers/mozilla\.py$ + | ^vulnerabilities/importers/openssl\.py$ + | ^vulnerabilities/importers/oss_fuzz\.py$ + | ^vulnerabilities/importers/osv\.py$ + | ^vulnerabilities/importers/postgresql\.py$ + | ^vulnerabilities/importers/retiredotnet\.py$ + | ^vulnerabilities/importers/ruby\.py$ + | ^vulnerabilities/importers/rust\.py$ + | ^vulnerabilities/importers/suse_backports\.py$ + | ^vulnerabilities/importers/suse_oval\.py$ + | ^vulnerabilities/importers/vulnrichment\.py$ + + # pipeline modules not yet typed (v2 importers/exporters) + | ^vulnerabilities/pipelines/exporters/federate_vulnerabilities\.py$ + | ^vulnerabilities/pipelines/v2_importers/alpine_linux_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/aosp_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/apache_httpd_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/apache_kafka_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/apache_tomcat_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/archlinux_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/curl_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/debian_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/elixir_security_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/epss_importer_v2\.py$ + | ^vulnerabilities/pipelines/v2_importers/fireeye_importer_v2\.py$ + | ^vulnerabilities/pipelines/v2_importers/gentoo_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/github_osv_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/gitlab_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/glibc_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/istio_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/linux_kernel_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/mattermost_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/mozilla_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/nginx_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/npm_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/nvd_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/openssl_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/oss_fuzz\.py$ + | ^vulnerabilities/pipelines/v2_importers/postgresql_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/project_kb_msr2019_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/project_kb_statements_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/pypa_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/pysec_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/redhat_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/retiredotnet_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/ruby_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/suse_score_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/ubuntu_osv_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/vulnrichment_importer\.py$ + | ^vulnerabilities/pipelines/v2_importers/xen_importer\.py$ + + # data-transform pipes not yet typed + | ^vulnerabilities/pipes/advisory\.py$ + | ^vulnerabilities/pipes/fetchcode_utils\.py$ + | ^vulnerabilities/pipes/group_advisories\.py$ + | ^vulnerabilities/pipes/openssl\.py$ + | ^vulnerabilities/pipes/osv_v2\.py$ + | ^vulnerabilities/pipes/univers_utils\.py$ + | ^vulnerabilities/pipes/vcs_collector_utils\.py$ + + # improvers not yet typed + | ^vulnerabilities/improvers/add_missing_refid\.py$ + | ^vulnerabilities/improvers/default\.py$ + | ^vulnerabilities/improvers/valid_versions\.py$ + + # management commands not yet typed + | ^vulnerabilities/management/commands/commit_export\.py$ + | ^vulnerabilities/management/commands/export\.py$ + + # test suite not yet typed + | ^vulnerabilities/tests/pipelines/test_base_pipeline\.py$ + | ^vulnerabilities/tests/pipelines/test_compute_advisory_todo\.py$ + | ^vulnerabilities/tests/pipelines/test_npm_importer_pipeline\.py$ + | ^vulnerabilities/tests/pipelines/test_pipeline_id\.py$ + | ^vulnerabilities/tests/pipelines/test_pypa_importer_pipeline\.py$ + | ^vulnerabilities/tests/pipelines/test_pysec_importer_pipeline\.py$ + | ^vulnerabilities/tests/pipelines/test_remove_duplicate_advisories\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_apache_httpd_importer_pipeline_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_apache_kafka_importer\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_apache_tomcat_importer_pipeline\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_collect_fix_commit\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_debian_importer\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_elixir_security_importer_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_github_osv_importer_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_gitlab_importer_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_istio_importer_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_mattermost_importer_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_mozilla_importer_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_npm_importer_pipeline_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_openssl_importer\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_postgresql_importer_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_pypa_importer_pipeline_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_pysec_importer_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_ubuntu_osv_importer\.py$ + | ^vulnerabilities/tests/pipelines/v2_importers/test_xen_importer_v2\.py$ + | ^vulnerabilities/tests/pipelines/v2_improvers/test_unfurl_version_range\.py$ + | ^vulnerabilities/tests/pipes/test_apache_kafka\.py$ + | ^vulnerabilities/tests/pipes/test_osv_v2\.py$ + | ^vulnerabilities/tests/pipes/test_vulnerablecode_importer_pipeline_v2\.py$ + | ^vulnerabilities/tests/test_affected_package\.py$ + | ^vulnerabilities/tests/test_api_extension\.py$ + | ^vulnerabilities/tests/test_api_v3\.py$ + | ^vulnerabilities/tests/test_changelog\.py$ + | ^vulnerabilities/tests/test_data_migrations\.py$ + | ^vulnerabilities/tests/test_data_source\.py$ + | ^vulnerabilities/tests/test_debian_oval\.py$ + | ^vulnerabilities/tests/test_default_improver\.py$ + | ^vulnerabilities/tests/test_example\.py$ + | ^vulnerabilities/tests/test_github_osv\.py$ + | ^vulnerabilities/tests/test_import_command\.py$ + | ^vulnerabilities/tests/test_import_runner\.py$ + | ^vulnerabilities/tests/test_improve_command\.py$ + | ^vulnerabilities/tests/test_improve_runner\.py$ + | ^vulnerabilities/tests/test_models\.py$ + | ^vulnerabilities/tests/test_openssl\.py$ + | ^vulnerabilities/tests/test_oss_fuzz\.py$ + | ^vulnerabilities/tests/test_osv\.py$ + | ^vulnerabilities/tests/test_retiredotnet\.py$ + | ^vulnerabilities/tests/test_rust\.py$ + | ^vulnerabilities/tests/test_suse\.py$ + | ^vulnerabilities/tests/test_suse_oval\.py$ + | ^vulnerabilities/tests/test_tasks\.py$ + | ^vulnerabilities/tests/test_utils\.py$ + + # vulntotal data sources not yet typed + | ^vulntotal/datasources/deps\.py$ + | ^vulntotal/datasources/github\.py$ + | ^vulntotal/datasources/gitlab\.py$ + | ^vulntotal/datasources/oss_index\.py$ + | ^vulntotal/datasources/osv\.py$ + | ^vulntotal/datasources/snyk\.py$ + | ^vulntotal/datasources/vulnerablecode\.py$ + + # vulntotal core not yet typed + | ^vulntotal/validator\.py$ + | ^vulntotal/vulntotal_cli\.py$ + + # Django settings not yet typed + | ^vulnerablecode/settings\.py$ + ) \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index 75966c993..cf4ce08ab 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -80,6 +80,7 @@ MarkupSafe==3.0.3 matplotlib-inline==0.2.1 mdurl==0.1.2 more-itertools==10.8.0 +mypy==2.1.0 mypy_extensions==1.1.0 nh3==0.3.4 packageurl-python==0.17.6 diff --git a/setup.cfg b/setup.cfg index 611219c3b..837f2882c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -127,6 +127,8 @@ dev = # Tests pytest==7.1.1 pytest-django==4.5.2 + # Type checking + mypy==2.1.0 # misc ipython==8.10.0