diff --git a/README.md b/README.md index f2f0fbfb..b11109cc 100644 --- a/README.md +++ b/README.md @@ -132,12 +132,20 @@ ut-run-tests . This launcher calls `docker/run_tests.py`, which runs tests in a Docker container (headless), streams output to stdout/stderr and keeps a cache -volume so repeated runs are fast. +volume so repeated runs are fast. By default it runs Python unit tests, +syntax tests and syntax compatibility checks. + +`--file` chooses the runner from the selected file: Python files run as unit +tests, `syntax_test*` files run as syntax tests, and `.sublime-syntax` files +run compatibility checks. Useful options: - `--file tests/test_foo.py` - `--pattern test_foo.py --tests-dir tests/subdir` +- `--no-unit-tests` +- `--no-syntax-tests` +- `--no-syntax-compatibility-checks` - `--coverage` - `--failfast` - `--reload-package-on-testing` (default: off) diff --git a/docker/README.md b/docker/README.md index c55e549b..150489f5 100644 --- a/docker/README.md +++ b/docker/README.md @@ -26,52 +26,63 @@ By default it: - builds `unittesting-local` image from `./docker` if missing - mounts your repo as `/project` +- runs Python unit tests, syntax tests and syntax compatibility checks - runs UnitTesting through the same CI shell entrypoints - stores Sublime install/cache in docker volume `unittesting-home` - synchronizes only changed files into `Packages/` using `rsync` +- excludes files ignored by Git, including repository-local and global rules -## Manual docker usage +A category with no matching resources is reported and succeeds. Disable +categories that are not needed with: ```sh -# build from UnitTesting/docker -docker build -t unittesting-local . +ut-run-tests . --no-unit-tests +ut-run-tests . --no-syntax-tests +ut-run-tests . --no-syntax-compatibility-checks +``` -# run from package root -docker run --rm -it \ - -e PACKAGE=$PACKAGE \ - -v $PWD:/project \ - -v unittesting-home:/root \ - unittesting-local run_tests +`--file` selects its category automatically. Python files run as unit tests, +files whose names start with `syntax_test` run as syntax tests, and +`.sublime-syntax` files run compatibility checks: + +```sh +ut-run-tests . --file tests/test_example.py +ut-run-tests . --file syntax_test_example +ut-run-tests . --file Example.sublime-syntax ``` +`--pattern` and `--tests-dir` try every enabled category. Use the +`--no-*` options to avoid running unrelated categories when desired. + ## Fast reruns The container entrypoint writes a marker in `/root/.cache/unittesting`. With `-v unittesting-home:/root`, bootstrap/install runs once and later runs only refresh your package files and execute tests. -## Serialized runs +## Concurrent runs and races The shared cache volume contains the Sublime data directory, including -`Packages`, `Lib`, UnitTesting schedules and test output files. Concurrent -runs against the same volume are serialized by default to avoid races while +`Packages`, `Lib`, UnitTesting schedules and test output files. *Concurrent +runs against the same volume are serialized by default* to avoid races while copying packages, writing schedules and syncing Package Control libraries. +This is a speed-versus-space trade-off: tests are likely to run quickly, +keeping wait times low, and a shared volume uses less disk space than +multiple volumes. Use `--lock-timeout SECONDS` to control how long a runner waits for the cache volume lock. Use `--no-lock` only if you know the selected cache volume is not shared by another runner. -## Concurrent runs - -You can control concurrency by choosing how many cache volumes you use. The -default single volume serializes all runs. A stable volume per package allows -different packages to run concurrently while still keeping warm caches: +You can increase concurrency by choosing how many cache volumes you use. For +example, a stable volume per package allows different packages to run +concurrently while still keeping warm caches: ```sh ut-run-tests . --cache-volume unittesting-home-gitsavvy ``` -To maximize concurrency, use a stable volume per checkout directory. For +To *maximize* concurrency, use a stable volume per checkout directory. For example, in a POSIX shell: ```sh @@ -120,7 +131,21 @@ Use `--color` to control ANSI colors in test output: ut-run-tests . --color always ``` -## Run a single test file +## Manual docker usage + +```sh +# build from UnitTesting/docker +docker build -t unittesting-local . + +# run from package root +docker run --rm -it \ + -e PACKAGE=$PACKAGE \ + -v $PWD:/project \ + -v unittesting-home:/root \ + unittesting-local run_tests +``` + +Run a single test file ```sh docker run --rm -it \ diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 05dfa2d6..27f260b3 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -99,7 +99,11 @@ fi if [ -d "$UNITTESTING_SOURCE/sbin" ]; then # Ensure UnitTesting comes from the local checkout running this script, # so first runs do not depend on tagged upstream releases. - (cd "$UNITTESTING_SOURCE" && PACKAGE=UnitTesting /docker.sh copy_tested_package overwrite) + ( + cd "$UNITTESTING_SOURCE" + PACKAGE=UnitTesting UNITTESTING_IGNORE_MANIFEST= \ + /docker.sh copy_tested_package overwrite + ) # Normalize CRLF in shell scripts copied from Windows workspaces. if [ -d "$ST_PACKAGES_DIR/UnitTesting/sbin" ]; then diff --git a/docker/run_tests.py b/docker/run_tests.py index 23ad2418..fa09e953 100644 --- a/docker/run_tests.py +++ b/docker/run_tests.py @@ -23,6 +23,14 @@ DEFAULT_IMAGE = "unittesting-local" DEFAULT_CACHE_VOLUME = "unittesting-home" DEFAULT_LOCK_TIMEOUT = 3600 +UNIT_TESTS = "unit-tests" +SYNTAX_TESTS = "syntax-tests" +SYNTAX_COMPATIBILITY_CHECKS = "syntax-compatibility-checks" +ALL_TEST_CATEGORIES = ( + UNIT_TESTS, + SYNTAX_TESTS, + SYNTAX_COMPATIBILITY_CHECKS, +) DOCKER_CONTEXT_HASH_LABEL = "org.sublimetext.unittesting.context-hash" DOCKER_CONTEXT_INPUTS = ( "Dockerfile", @@ -74,9 +82,10 @@ def main(argv: list[str] | None = None) -> int: return 2 package_name = args.package_name or package_root.name - tests_dir, pattern = resolve_test_target( + tests_dir, pattern, selected_file = resolve_test_target( package_root, args.file, args.tests_dir, args.pattern ) + test_categories = resolve_test_categories(args, selected_file) maybe_build_image(image, refresh=False) @@ -85,47 +94,54 @@ def main(argv: list[str] | None = None) -> int: lock_enabled = should_lock_cache(args) runner_name = docker_cache_runner_name(args.cache_volume) if lock_enabled else None - command = build_docker_run_command( - package_root=package_root, - unit_testing_root=unit_testing_root, - package_name=package_name, - image=image, - cache_volume=args.cache_volume, - container_name=runner_name, - scheduler_delay_ms=args.scheduler_delay_ms, - coverage=args.coverage, - failfast=args.failfast, - reload_package_on_testing=args.reload_package_on_testing, - dry_run=args.dry_run, - color=args.color, - tests_dir=tests_dir, - pattern=pattern, - ) + ignore_manifest = make_git_ignore_manifest(package_root, selected_file) + with ignore_manifest: + command = build_docker_run_command( + package_root=package_root, + unit_testing_root=unit_testing_root, + package_name=package_name, + image=image, + cache_volume=args.cache_volume, + container_name=runner_name, + ignore_manifest=ignore_manifest, + test_categories=test_categories, + scheduler_delay_ms=args.scheduler_delay_ms, + coverage=args.coverage, + failfast=args.failfast, + reload_package_on_testing=args.reload_package_on_testing, + dry_run=args.dry_run, + color=args.color, + tests_dir=tests_dir, + pattern=pattern, + ) + + print(f"Package root: {package_root}") + print(f"Package name: {package_name}") + print(f"Docker image: {image}") + print(f"Scheduler delay: {args.scheduler_delay_ms}ms") + if args.refresh_image: + print("Image refresh: enabled") + if args.cache_volume: + print(f"Cache volume: {args.cache_volume}") + if lock_enabled: + print("Cache lock: enabled") + if args.refresh_cache: + print("Cache refresh: enabled") + print(f"Test categories: {', '.join(test_categories)}") + if tests_dir and pattern: + print(f"Test target: {tests_dir}/{pattern}") + if ignore_manifest: + print("Package sync: Git-ignored files excluded") - print(f"Package root: {package_root}") - print(f"Package name: {package_name}") - print(f"Docker image: {image}") - print(f"Scheduler delay: {args.scheduler_delay_ms}ms") - if args.refresh_image: - print("Image refresh: enabled") - if args.cache_volume: - print(f"Cache volume: {args.cache_volume}") if lock_enabled: - print("Cache lock: enabled") - if args.refresh_cache: - print("Cache refresh: enabled") - if tests_dir and pattern: - print(f"Test target: {tests_dir}/{pattern}") - - if lock_enabled: - with CacheVolumeLock(args.cache_volume, args.lock_timeout): - wait_for_cache_volume_idle(args.cache_volume, args.lock_timeout) - ensure_runner_container_name_available(args.cache_volume, args.lock_timeout) - return call_docker_run_with_name_retry( - command, args.cache_volume, args.lock_timeout - ) + with CacheVolumeLock(args.cache_volume, args.lock_timeout): + wait_for_cache_volume_idle(args.cache_volume, args.lock_timeout) + ensure_runner_container_name_available(args.cache_volume, args.lock_timeout) + return call_docker_run_with_name_retry( + command, args.cache_volume, args.lock_timeout + ) - return subprocess.call(command) + return subprocess.call(command) def parse_args(argv: list[str] | None) -> argparse.Namespace: @@ -142,10 +158,29 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: ) test_group = parser.add_argument_group("test options") - test_group.add_argument("--file", help="Run only tests from this file.") + test_group.add_argument( + "--file", + action="append", + help="Run only tests from this file (may be specified once).", + ) test_group.add_argument("--pattern", help="Custom unittest discovery pattern.") test_group.add_argument("--tests-dir", help="Custom tests directory.") test_group.add_argument("--package-name", help="Override package name.") + test_group.add_argument( + "--no-unit-tests", + action="store_true", + help="Do not run Python unit tests.", + ) + test_group.add_argument( + "--no-syntax-tests", + action="store_true", + help="Do not run syntax tests.", + ) + test_group.add_argument( + "--no-syntax-compatibility-checks", + action="store_true", + help="Do not run syntax compatibility checks.", + ) test_group.add_argument("--coverage", action="store_true", help="Enable coverage.") test_group.add_argument("--failfast", action="store_true", help="Stop on first failure.") test_group.add_argument( @@ -227,9 +262,27 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: args = parser.parse_args(argv) + if args.file and len(args.file) > 1: + parser.error("--file may only be specified once") + args.file = args.file[0] if args.file else None + if args.file and args.pattern: parser.error("--file and --pattern are mutually exclusive") + if args.file and args.tests_dir: + parser.error("--file and --tests-dir are mutually exclusive") + + category_options = ( + args.no_unit_tests, + args.no_syntax_tests, + args.no_syntax_compatibility_checks, + ) + if args.file and any(category_options): + parser.error("--file cannot be combined with --no-* test category options") + + if all(category_options): + parser.error("all test categories are disabled") + if args.refresh_cache and not args.cache_volume: parser.error("--refresh-cache requires a cache volume (omit --no-cache-volume)") @@ -354,9 +407,9 @@ def resolve_test_target( test_file: str | None, tests_dir: str | None, pattern: str | None, -) -> tuple[str | None, str | None]: +) -> tuple[str | None, str | None, str | None]: if not test_file: - return tests_dir, pattern + return tests_dir, pattern, None file_path = Path(test_file) if not file_path.is_absolute(): @@ -374,7 +427,114 @@ def resolve_test_target( rel_parent = rel_file_path.parent.as_posix() resolved_tests_dir = rel_parent if rel_parent else "." resolved_pattern = rel_file_path.name - return resolved_tests_dir, resolved_pattern + return resolved_tests_dir, resolved_pattern, rel_file_path.as_posix() + + +def resolve_test_categories( + args: argparse.Namespace, selected_file: str | None +) -> tuple[str, ...]: + if selected_file: + return (test_category_for_file(selected_file),) + + disabled_categories = { + UNIT_TESTS: args.no_unit_tests, + SYNTAX_TESTS: args.no_syntax_tests, + SYNTAX_COMPATIBILITY_CHECKS: args.no_syntax_compatibility_checks, + } + return tuple( + category + for category in ALL_TEST_CATEGORIES + if not disabled_categories[category] + ) + + +def test_category_for_file(test_file: str) -> str: + file_name = Path(test_file).name + if file_name.startswith("syntax_test"): + return SYNTAX_TESTS + if file_name.endswith(".sublime-syntax"): + return SYNTAX_COMPATIBILITY_CHECKS + if file_name.endswith(".py"): + return UNIT_TESTS + raise SystemExit(f"Error: unsupported test file type: {test_file}") + + +class GitIgnoreManifest: + def __init__(self, contents: bytes | None = None) -> None: + self.path: Path | None = None + if contents is not None: + with tempfile.NamedTemporaryFile( + prefix="unittesting-ignore-", suffix=".files", delete=False + ) as manifest: + manifest.write(contents) + self.path = Path(manifest.name) + + def __bool__(self) -> bool: + return self.path is not None + + def __enter__(self) -> "GitIgnoreManifest": + return self + + def __exit__(self, exc_type, exc_value, traceback) -> None: + if self.path: + self.path.unlink(missing_ok=True) + self.path = None + + +def make_git_ignore_manifest( + package_root: Path, selected_file: str | None = None +) -> GitIgnoreManifest: + if not shutil.which("git"): + return GitIgnoreManifest() + + result = subprocess.run( + [ + "git", + "-C", + str(package_root), + "ls-files", + "--others", + "--ignored", + "--exclude-standard", + "--directory", + "-z", + ], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + if result.returncode != 0: + return GitIgnoreManifest() + + return GitIgnoreManifest(make_rsync_ignore_filter(result.stdout, selected_file)) + + +def make_rsync_ignore_filter(paths: bytes, selected_file: str | None) -> bytes: + entries = [path for path in paths.split(b"\0") if path] + rules: list[bytes] = [] + + if selected_file: + selected_path = os.fsencode(selected_file) + ignored_directories = [ + path + for path in entries + if path.endswith(b"/") and selected_path.startswith(path) + ] + if selected_path in entries or ignored_directories: + parts = selected_path.split(b"/") + # rsync uses the first matching rule and does not descend into an + # excluded directory. Include the selected file and its ancestors + # first... + rules.extend( + b"+ /" + b"/".join(parts[:i]) + b"/" + for i in range(1, len(parts)) + ) + rules.append(b"+ /" + selected_path) + # ... then exclude everything else under the ignored ancestor + # directories (`/***`). + rules.extend(b"- /" + path + b"***" for path in ignored_directories) + + rules.extend(b"- /" + path for path in entries) + return b"\0".join(rules) + (b"\0" if rules else b"") def build_docker_run_command( @@ -384,6 +544,8 @@ def build_docker_run_command( image: str, cache_volume: str | None, container_name: str | None, + ignore_manifest: GitIgnoreManifest, + test_categories: tuple[str, ...], scheduler_delay_ms: int, coverage: bool, failfast: bool, @@ -409,11 +571,18 @@ def build_docker_run_command( command.extend(["-v", f"{package_root}:/project"]) command.extend(["-v", f"{unit_testing_root}:/unittesting"]) + if ignore_manifest.path: + manifest_target = "/tmp/unittesting-ignore.files" + command.extend(["-e", f"UNITTESTING_IGNORE_MANIFEST={manifest_target}"]) + command.extend(["-v", f"{ignore_manifest.path}:{manifest_target}:ro"]) + if cache_volume: command.extend(["-v", f"{cache_volume}:/root"]) command.append(image) - command.append("run_tests") + command.append("run_test_categories") + command.extend(f"--{category}" for category in test_categories) + command.append("--") if coverage: command.append("--coverage") diff --git a/docker/tests/test_run_tests.py b/docker/tests/test_run_tests.py new file mode 100644 index 00000000..74663315 --- /dev/null +++ b/docker/tests/test_run_tests.py @@ -0,0 +1,125 @@ +import importlib.util +import io +import unittest +from contextlib import redirect_stderr +from pathlib import Path + + +RUNNER_PATH = Path(__file__).resolve().parents[1] / "run_tests.py" +SPEC = importlib.util.spec_from_file_location("docker_run_tests", RUNNER_PATH) +runner = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(runner) + + +class ParseArgsTests(unittest.TestCase): + def test_rejects_repeated_file(self): + with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error: + runner.parse_args(["--file", "a.py", "--file", "b.py"]) + + self.assertEqual(error.exception.code, 2) + + def test_rejects_disabling_every_category(self): + with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error: + runner.parse_args( + [ + "--no-unit-tests", + "--no-syntax-tests", + "--no-syntax-compatibility-checks", + ] + ) + + self.assertEqual(error.exception.code, 2) + + +class TestCategoryTests(unittest.TestCase): + def test_runs_every_category_by_default(self): + args = runner.parse_args([]) + + self.assertEqual( + runner.resolve_test_categories(args, None), + runner.ALL_TEST_CATEGORIES, + ) + + def test_skips_disabled_categories(self): + args = runner.parse_args(["--no-syntax-tests"]) + + self.assertEqual( + runner.resolve_test_categories(args, None), + (runner.UNIT_TESTS, runner.SYNTAX_COMPATIBILITY_CHECKS), + ) + + def test_discovery_options_apply_to_every_category(self): + args = runner.parse_args(["--tests-dir", "specs", "--pattern", "spec*"]) + + self.assertEqual( + runner.resolve_test_categories(args, None), + runner.ALL_TEST_CATEGORIES, + ) + + def test_discovery_options_apply_to_enabled_categories(self): + args = runner.parse_args( + [ + "--tests-dir", + "syntax/test", + "--no-unit-tests", + "--no-syntax-compatibility-checks", + ] + ) + + self.assertEqual( + runner.resolve_test_categories(args, None), + (runner.SYNTAX_TESTS,), + ) + + def test_infers_category_from_file(self): + args = runner.parse_args([]) + cases = { + "tests/test_example.py": runner.UNIT_TESTS, + "syntax/syntax_test_example": runner.SYNTAX_TESTS, + "syntaxes/Example.sublime-syntax": runner.SYNTAX_COMPATIBILITY_CHECKS, + } + + for test_file, category in cases.items(): + with self.subTest(test_file=test_file): + self.assertEqual( + runner.resolve_test_categories(args, test_file), + (category,), + ) + + def test_rejects_unsupported_file_type(self): + args = runner.parse_args([]) + + with self.assertRaisesRegex(SystemExit, "unsupported test file type"): + runner.resolve_test_categories(args, "tests/example.txt") + + +class DockerCommandTests(unittest.TestCase): + def test_passes_selected_categories_to_container_runner(self): + command = runner.build_docker_run_command( + package_root=Path("/package"), + unit_testing_root=Path("/unittesting"), + package_name="Example", + image="image", + cache_volume=None, + container_name=None, + ignore_manifest=runner.GitIgnoreManifest(), + test_categories=(runner.UNIT_TESTS, runner.SYNTAX_TESTS), + scheduler_delay_ms=0, + coverage=False, + failfast=False, + reload_package_on_testing=False, + dry_run=False, + color="never", + tests_dir=None, + pattern=None, + ) + + image_index = command.index("image") + self.assertEqual( + command[image_index + 1 : image_index + 5], + ["run_test_categories", "--unit-tests", "--syntax-tests", "--"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/sbin/ci.sh b/sbin/ci.sh index b0ae90a8..aadfefa7 100644 --- a/sbin/ci.sh +++ b/sbin/ci.sh @@ -123,7 +123,12 @@ CopyTestedPackage() { if [ -n "$OverwriteExisting" ] && command -v rsync >/dev/null 2>&1; then echo "sync package into sublime package directory" - rsync -a --delete --exclude .git ./ "$STP/$PACKAGE/" + if [ -n "${UNITTESTING_IGNORE_MANIFEST:-}" ] && [ -f "$UNITTESTING_IGNORE_MANIFEST" ]; then + rsync -a --delete --delete-excluded --from0 --exclude .git \ + --exclude-from="$UNITTESTING_IGNORE_MANIFEST" ./ "$STP/$PACKAGE/" + else + rsync -a --delete --exclude .git ./ "$STP/$PACKAGE/" + fi return fi @@ -175,20 +180,74 @@ InstallPackageControl() { sh "$STP/UnitTesting/sbin/install_package_control.sh" "--st" "$SUBLIME_TEXT_VERSION" } +RunTestCategories() { + local RunUnitTests=false + local RunSyntaxTests=false + local RunSyntaxCompatibilityChecks=false + + while [ "$#" -gt 0 ]; do + case "$1" in + "--unit-tests") + RunUnitTests=true + ;; + "--syntax-tests") + RunSyntaxTests=true + ;; + "--syntax-compatibility-checks") + RunSyntaxCompatibilityChecks=true + ;; + "--") + shift + break + ;; + *) + echo "Unknown test category: $1" >&2 + return 2 + ;; + esac + shift + done + + if [ "$RunUnitTests" = false ] && [ "$RunSyntaxTests" = false ] && \ + [ "$RunSyntaxCompatibilityChecks" = false ]; then + echo "No test categories selected" >&2 + return 2 + fi + + local CategoryOptions=() + if [ "$RunUnitTests" = true ]; then + CategoryOptions+=("--unit-test") + fi + if [ "$RunSyntaxTests" = true ]; then + CategoryOptions+=("--syntax-test") + fi + if [ "$RunSyntaxCompatibilityChecks" = true ]; then + CategoryOptions+=("--syntax-compatibility") + fi + if [ "$RunSyntaxTests" = true ] || \ + [ "$RunSyntaxCompatibilityChecks" = true ]; then + CategoryOptions+=("--no-fail-if-no-resources") + fi + + RunTests "${CategoryOptions[@]}" "$@" +} + RunTests() { # if [ -n "$(echo "$@" | grep -e '--coverage\b')" ] && [ "$SUBLIME_TEXT_VERSION" -eq 4 ]; then # echo "Coverage is not yet supported in Sublime Text 4" # exit 1 # fi + local Status=0 if [ -z "$1" ]; then - python "$STP/UnitTesting/sbin/run_tests.py" "$PACKAGE" + python "$STP/UnitTesting/sbin/run_tests.py" "$PACKAGE" || Status=$? else - python "$STP/UnitTesting/sbin/run_tests.py" "$@" "$PACKAGE" + python "$STP/UnitTesting/sbin/run_tests.py" "$@" "$PACKAGE" || Status=$? fi pkill "[Ss]ubl" || true pkill 'plugin_host' || true sleep 1 + return "$Status" } @@ -217,6 +276,9 @@ case $COMMAND in "run_tests") RunTests "$@" ;; + "run_test_categories") + RunTestCategories "$@" + ;; "run_syntax_tests") RunTests "--syntax-test" "$@" ;; diff --git a/sbin/run_tests.py b/sbin/run_tests.py index 742151e9..d2584074 100644 --- a/sbin/run_tests.py +++ b/sbin/run_tests.py @@ -26,6 +26,7 @@ UT_SBIN_PATH = os.path.realpath(os.path.join(PACKAGES_DIR_PATH, 'UnitTesting', 'sbin')) SCHEDULE_RUNNER_SOURCE = os.path.join(UT_SBIN_PATH, "run_scheduler.py") SCHEDULE_RUNNER_TARGET = os.path.join(UT_DIR_PATH, "zzz_run_scheduler.py") +DONE_MESSAGE = "UnitTesting: Done.\n" RX_RESULT = re.compile(r'^(?POK|FAILED|ERROR)', re.MULTILINE) RX_DONE = re.compile(r'^UnitTesting: Done\.$', re.MULTILINE) RX_TEST_STATUS = re.compile(r'\.\.\. (ok|FAIL|ERROR|skipped)(\b.*)$') @@ -56,7 +57,7 @@ def copy_file_if_not_exists(source, target): shutil.copyfile(source, target) -def create_schedule(package, output_file, default_schedule): +def create_schedules(package, named_schedules): schedule = [] try: @@ -65,16 +66,8 @@ def create_schedule(package, output_file, default_schedule): except Exception: pass - print('Schedule:') - for k, v in default_schedule.items(): - print(' %s: %s' % (k, v)) - - for idx, item in enumerate(schedule): - if item.get('package') == package: - schedule[idx] = default_schedule - break - else: - schedule.append(default_schedule) + schedule = [item for item in schedule if item.get('package') != package] + schedule.extend(default_schedule for _, default_schedule in named_schedules) with open(SCHEDULE_FILE_PATH, 'w') as f: f.write(json.dumps(schedule, ensure_ascii=False, indent=True)) @@ -83,7 +76,6 @@ def create_schedule(package, output_file, default_schedule): def wait_for_output(path, schedule, timeout=10, poll_interval=0.2): start_time = time.time() last_dot = 0 - needs_newline = False def check_has_timed_out(): return time.time() - start_time > timeout @@ -99,7 +91,6 @@ def check_is_output_available(): if now - last_dot >= 1: print(".", end="") sys.stdout.flush() - needs_newline = True last_dot = now if check_has_timed_out(): @@ -109,8 +100,7 @@ def check_is_output_available(): time.sleep(poll_interval) else: - if needs_newline: - print() + print() def start_sublime_text(): @@ -122,7 +112,7 @@ def kill_sublime_text(): subprocess.Popen("pkill plugin_host || true", shell=True) -def read_output(path, color='auto'): +def read_output(path, color='auto', show_done=True): # todo: use notification instead of polling success = None use_color = should_use_color(color) @@ -143,11 +133,12 @@ def check_is_done(result): result = f.read() if result: + display_result = result if show_done else result.replace(DONE_MESSAGE, "") if use_color: - rendered, pending = colorize_output_chunk(result, pending) + rendered, pending = colorize_output_chunk(display_result, pending) print(rendered, end="") else: - print(result, end="") + print(display_result, end="") sys.stdout.flush() # Keep checking while we don't have a definite result. @@ -307,34 +298,34 @@ def detect_package_control_version(): return str(version) if version else None -def main(default_schedule_info, dry_run=False, color='auto'): - package_under_test = default_schedule_info['package'] +def main(named_schedules, dry_run=False, color='auto'): + package_under_test = named_schedules[0][1]['package'] output_dir = os.path.join(UT_OUTPUT_DIR_PATH, package_under_test) - output_file = os.path.join(output_dir, "result") coverage_file = os.path.join(output_dir, "coverage") - - default_schedule_info['output'] = output_file + output_files = configure_schedule_outputs(named_schedules, output_dir) print_runtime_metadata() + print_schedules(named_schedules) if dry_run: create_dir_if_not_exists(output_dir) - delete_file_if_exists(output_file) + delete_files(output_files) delete_file_if_exists(coverage_file) - create_schedule(package_under_test, output_file, default_schedule_info) + create_schedules(package_under_test, named_schedules) return for i in range(3): create_dir_if_not_exists(output_dir) - delete_file_if_exists(output_file) + delete_files(output_files) delete_file_if_exists(coverage_file) - create_schedule(package_under_test, output_file, default_schedule_info) + create_schedules(package_under_test, named_schedules) delete_file_if_exists(SCHEDULE_RUNNER_TARGET) copy_file_if_not_exists(SCHEDULE_RUNNER_SOURCE, SCHEDULE_RUNNER_TARGET) start_sublime_text() try: - print("Wait for tests output...", end="") - wait_for_output(output_file, SCHEDULE_RUNNER_TARGET) + for name, output_file in output_files: + print("Wait for %s output..." % name, end="") + wait_for_output(output_file, SCHEDULE_RUNNER_TARGET) break except ValueError: if i == 2: @@ -343,24 +334,117 @@ def main(default_schedule_info, dry_run=False, color='auto'): "is being written to the wrong file.") delete_file_if_exists(SCHEDULE_RUNNER_TARGET) sys.exit(1) + print("Retrying after Sublime Text did not produce test output.") kill_sublime_text() time.sleep(2) - print("Start to read output...") - if not read_output(output_file, color=color): + success = True + show_category_done = len(output_files) == 1 + for name, output_file in output_files: + print("=== %s OUTPUT ===" % name.upper()) + if not read_output(output_file, color=color, show_done=show_category_done): + success = False + + if not show_category_done: + print(DONE_MESSAGE, end="") + + if not success: sys.exit(1) restore_coverage_file(coverage_file, package_under_test) delete_file_if_exists(SCHEDULE_RUNNER_TARGET) +def print_schedules(named_schedules): + for name, schedule in named_schedules: + heading = 'Schedule:' if len(named_schedules) == 1 else 'Schedule (%s):' % name + print(heading) + for key, value in schedule.items(): + print(' %s: %s' % (key, value)) + + +def configure_schedule_outputs(named_schedules, output_dir): + output_files = [] + for name, schedule in named_schedules: + output_name = ( + "result" + if len(named_schedules) == 1 + else "result-" + name.replace(" ", "-") + ) + output_file = os.path.join(output_dir, output_name) + schedule['output'] = output_file + output_files.append((name, output_file)) + return output_files + + +def delete_files(named_files): + for _, path in named_files: + delete_file_if_exists(path) + + +def build_named_schedules(options, package): + schedule_options = { + 'package': package, + 'coverage': options.coverage, + 'reload_package_on_testing': bool(options.reload_package_on_testing), + } + + if options.pattern: + schedule_options['pattern'] = options.pattern + if options.tests_dir: + schedule_options['tests_dir'] = options.tests_dir + if not options.fail_if_no_resources: + schedule_options['fail_if_no_resources'] = False + if options.failfast: + schedule_options['failfast'] = True + + named_schedules = [] + explicit_category = any( + ( + options.unit_test, + options.syntax_test, + options.syntax_compatibility, + options.color_scheme_test, + ) + ) + if options.syntax_test: + named_schedules.append( + ('syntax tests', dict(schedule_options, syntax_test=True)) + ) + if options.syntax_compatibility: + named_schedules.append( + ( + 'syntax compatibility checks', + dict(schedule_options, syntax_compatibility=True), + ) + ) + if options.color_scheme_test: + named_schedules.append( + ('color scheme tests', dict(schedule_options, color_scheme_test=True)) + ) + + # Unit tests may continue through deferred callbacks after their command + # returns, so keep them last to avoid overlapping another category. + if options.unit_test or not explicit_category: + named_schedules.append(('unit tests', schedule_options)) + + return named_schedules + + if __name__ == '__main__': parser = optparse.OptionParser() + parser.add_option('--unit-test', action='store_true') parser.add_option('--syntax-test', action='store_true') parser.add_option('--syntax-compatibility', action='store_true') parser.add_option('--color-scheme-test', action='store_true') parser.add_option('--coverage', action='store_true') parser.add_option('--pattern') parser.add_option('--tests-dir') + parser.add_option( + '--no-fail-if-no-resources', + action='store_false', + dest='fail_if_no_resources', + default=True, + ) parser.add_option('--failfast', action='store_true') parser.add_option('--reload-package-on-testing', action='store_true') parser.add_option('--dry-run', action='store_true') @@ -374,31 +458,6 @@ def main(default_schedule_info, dry_run=False, color='auto'): options, remainder = parser.parse_args() - syntax_test = options.syntax_test - syntax_compatibility = options.syntax_compatibility - color_scheme_test = options.color_scheme_test - coverage = options.coverage package_under_test = remainder[0] if len(remainder) > 0 else "UnitTesting" - - default_schedule_info = { - 'package': package_under_test, - 'syntax_test': syntax_test, - 'syntax_compatibility': syntax_compatibility, - 'color_scheme_test': color_scheme_test, - 'coverage': coverage, - 'reload_package_on_testing': False, - } - - if options.pattern: - default_schedule_info['pattern'] = options.pattern - - if options.tests_dir: - default_schedule_info['tests_dir'] = options.tests_dir - - if options.failfast: - default_schedule_info['failfast'] = True - - if options.reload_package_on_testing: - default_schedule_info['reload_package_on_testing'] = True - - main(default_schedule_info, dry_run=options.dry_run, color=options.color) + named_schedules = build_named_schedules(options, package_under_test) + main(named_schedules, dry_run=options.dry_run, color=options.color) diff --git a/sbin/tests/test_sbin_runner.py b/sbin/tests/test_sbin_runner.py new file mode 100644 index 00000000..226547bd --- /dev/null +++ b/sbin/tests/test_sbin_runner.py @@ -0,0 +1,146 @@ +import importlib.util +import io +import json +import os +import tempfile +import unittest +from contextlib import redirect_stdout +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + + +RUNNER_PATH = Path(__file__).resolve().parents[1] / "run_tests.py" +SPEC = importlib.util.spec_from_file_location("sbin_run_tests", RUNNER_PATH) +runner = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(runner) + + +def options(**overrides): + values = { + "color_scheme_test": False, + "coverage": False, + "fail_if_no_resources": True, + "failfast": False, + "pattern": None, + "reload_package_on_testing": False, + "syntax_compatibility": False, + "syntax_test": False, + "tests_dir": None, + "unit_test": False, + } + values.update(overrides) + return SimpleNamespace(**values) + + +class BuildSchedulesTests(unittest.TestCase): + def test_defaults_to_unit_tests(self): + self.assertEqual( + runner.build_named_schedules(options(), "Example"), + [ + ( + "unit tests", + { + "package": "Example", + "coverage": False, + "reload_package_on_testing": False, + }, + ) + ], + ) + + def test_builds_synchronous_categories_before_unit_tests(self): + schedules = runner.build_named_schedules( + options( + unit_test=True, + syntax_test=True, + syntax_compatibility=True, + fail_if_no_resources=False, + pattern="selected*", + tests_dir="syntax/test", + ), + "Example", + ) + + self.assertEqual( + [name for name, _ in schedules], + ["syntax tests", "syntax compatibility checks", "unit tests"], + ) + for _, schedule in schedules: + self.assertEqual(schedule["pattern"], "selected*") + self.assertEqual(schedule["tests_dir"], "syntax/test") + self.assertFalse(schedule["fail_if_no_resources"]) + + def test_assigns_distinct_outputs_to_multiple_schedules(self): + schedules = runner.build_named_schedules( + options(unit_test=True, syntax_test=True), "Example" + ) + + output_files = runner.configure_schedule_outputs(schedules, "/output") + + self.assertEqual( + output_files, + [ + ("syntax tests", os.path.join("/output", "result-syntax-tests")), + ("unit tests", os.path.join("/output", "result-unit-tests")), + ], + ) + + +class OutputTests(unittest.TestCase): + def test_wait_heading_ends_with_newline_when_output_already_exists(self): + with tempfile.NamedTemporaryFile() as output: + output.write(b"ready") + output.flush() + rendered = io.StringIO() + with redirect_stdout(rendered): + print("Wait for output...", end="") + runner.wait_for_output(output.name, "unused") + + self.assertEqual(rendered.getvalue(), "Wait for output...\n") + + def test_can_hide_category_done_message(self): + with tempfile.NamedTemporaryFile(mode="w", delete=False) as output: + output.write("OK\n\n" + runner.DONE_MESSAGE) + output_path = output.name + + try: + rendered = io.StringIO() + with redirect_stdout(rendered): + success = runner.read_output( + output_path, color="never", show_done=False + ) + finally: + Path(output_path).unlink() + + self.assertTrue(success) + self.assertEqual(rendered.getvalue(), "OK\n\n") + + +class CreateSchedulesTests(unittest.TestCase): + def test_replaces_package_with_every_selected_schedule(self): + schedules = runner.build_named_schedules( + options(unit_test=True, syntax_test=True), "Example" + ) + + with tempfile.TemporaryDirectory() as temp_dir: + schedule_file = Path(temp_dir) / "schedule.json" + schedule_file.write_text( + json.dumps( + [ + {"package": "Other"}, + {"package": "Example", "stale": True}, + ] + ) + ) + with mock.patch.object(runner, "SCHEDULE_FILE_PATH", str(schedule_file)): + with redirect_stdout(io.StringIO()): + runner.create_schedules("Example", schedules) + saved = json.loads(schedule_file.read_text()) + + self.assertEqual(saved[0], {"package": "Other"}) + self.assertEqual(saved[1:], [schedule for _, schedule in schedules]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_3141596.py b/tests/test_3141596.py index d7b253c9..6954f68f 100644 --- a/tests/test_3141596.py +++ b/tests/test_3141596.py @@ -39,7 +39,8 @@ def cleanup_package(package): def with_package(package, output=None, syntax_test=False, syntax_compatibility=False, - color_scheme_test=False, wait_timeout=5000): + color_scheme_test=False, wait_timeout=5000, pattern=None, + tests_dir=None, fail_if_no_resources=None): def wrapper(func): @wraps(func) def real_wrapper(self): @@ -56,6 +57,12 @@ def real_wrapper(self): yield AWAIT_WORKER kwargs = {"package": package} + if pattern is not None: + kwargs["pattern"] = pattern + if tests_dir is not None: + kwargs["tests_dir"] = tests_dir + if fail_if_no_resources is not None: + kwargs["fail_if_no_resources"] = fail_if_no_resources if outfile: # Command kwargs have the highest precedence. Passing down # 'None' is not what we want, the intention is to omit it @@ -181,10 +188,24 @@ def test_fail_syntax(self, txt): def test_success_syntax(self, txt): self.assertOk(txt) + @with_package("_Syntax_Success", syntax_test=True, pattern="missing*") + def test_syntax_pattern(self, txt): + self.assertRegexContains(txt, r'^ERROR: No syntax_test') + + @with_package("_Syntax_Success", syntax_test=True, tests_dir="missing") + def test_syntax_tests_dir(self, txt): + self.assertRegexContains(txt, r'^ERROR: No syntax_test') + @with_package("_Syntax_Error", syntax_test=True) def test_error_syntax(self, txt): self.assertRegexContains(txt, r'^ERROR: No syntax_test') + @with_package( + "_Syntax_Error", syntax_test=True, fail_if_no_resources=False + ) + def test_empty_syntax_allowed(self, txt): + self.assertOk(txt) + @with_package("_Syntax_Compat_Failure", syntax_compatibility=True) def test_fail_syntax_compatibility(self, txt): self.assertRegexContains(txt, r'^FAILED: 3 errors in 1 of 1 syntax$') @@ -193,6 +214,26 @@ def test_fail_syntax_compatibility(self, txt): def test_success_syntax_compatibility(self, txt): self.assertOk(txt) + @with_package( + "_Syntax_Compat_Success", syntax_compatibility=True, pattern="missing*" + ) + def test_syntax_compatibility_pattern(self, txt): + self.assertRegexContains(txt, r'^ERROR: No sublime-syntax') + + @with_package( + "_Syntax_Compat_Success", syntax_compatibility=True, tests_dir="missing" + ) + def test_syntax_compatibility_tests_dir(self, txt): + self.assertRegexContains(txt, r'^ERROR: No sublime-syntax') + + @with_package( + "_Syntax_Error", + syntax_compatibility=True, + fail_if_no_resources=False, + ) + def test_empty_syntax_compatibility_allowed(self, txt): + self.assertOk(txt) + def has_colorschemeunit(): return ( diff --git a/unittesting/scheduler.py b/unittesting/scheduler.py index 871c2171..0dc03efc 100644 --- a/unittesting/scheduler.py +++ b/unittesting/scheduler.py @@ -42,6 +42,11 @@ def save(self, data, indent=4): class Unit: + SYNTAX_TESTING_OPTION_KEYS = ( + "fail_if_no_resources", + "pattern", + "tests_dir", + ) UNIT_TESTING_OPTION_KEYS = ( "capture_console", "condition_timeout", @@ -64,6 +69,9 @@ def __init__(self, s): self.syntax_compatibility = s.get("syntax_compatibility", False) self.color_scheme_test = s.get("color_scheme_test", False) self.coverage = s.get("coverage", False) + self.syntax_testing_options = { + key: s[key] for key in self.SYNTAX_TESTING_OPTION_KEYS if key in s + } self.unit_testing_options = { key: s[key] for key in self.UNIT_TESTING_OPTION_KEYS if key in s } @@ -76,16 +84,20 @@ def run(self): ) elif self.syntax_test: sublime.active_window().run_command( - "unit_testing_syntax", {"package": self.package, "output": self.output} + "unit_testing_syntax", self.syntax_testing_args() ) elif self.syntax_compatibility: sublime.active_window().run_command( - "unit_testing_syntax_compatibility", - {"package": self.package, "output": self.output}, + "unit_testing_syntax_compatibility", self.syntax_testing_args() ) else: sublime.active_window().run_command("unit_testing", self.unit_testing_args()) + def syntax_testing_args(self): + args = {"package": self.package, "output": self.output} + args.update(self.syntax_testing_options) + return args + def unit_testing_args(self): args = { "package": self.package, diff --git a/unittesting/syntax.py b/unittesting/syntax.py index 20f60bce..d15a0f9d 100644 --- a/unittesting/syntax.py +++ b/unittesting/syntax.py @@ -20,10 +20,13 @@ def run(self, package=None, **kwargs): failed_assertions = 0 try: - tests = sublime.find_resources("syntax_test*") - tests = [t for t in tests if t.startswith("Packages/%s/" % package)] + tests = find_package_resources( + package, + kwargs.get("pattern", "syntax_test*"), + kwargs.get("tests_dir"), + ) - if not tests: + if not tests and kwargs.get("fail_if_no_resources", True): raise RuntimeError("No syntax_test files are found in %s!" % package) for t in tests: assertions, test_output_lines = sublime_api.run_syntax_test(t) @@ -33,7 +36,7 @@ def run(self, package=None, **kwargs): for line in test_output_lines: stream.write(line + "\n") - file_noun = "files" if len(tests) > 1 else "file" + file_noun = "files" if len(tests) != 1 else "file" if failed_assertions > 0: stream.write( "FAILED: %d of %d assertions in %d %s failed\n" @@ -66,10 +69,13 @@ def run(self, package=None, **kwargs): stream = self.load_stream(package, settings) try: - syntaxes = sublime.find_resources("*.sublime-syntax") - syntaxes = [s for s in syntaxes if s.startswith("Packages/%s/" % package)] + syntaxes = find_package_resources( + package, + kwargs.get("pattern", "*.sublime-syntax"), + kwargs.get("tests_dir"), + ) - if not syntaxes: + if not syntaxes and kwargs.get("fail_if_no_resources", True): raise RuntimeError("No sublime-syntax files found in %s!" % package) total_errors = 0 @@ -87,7 +93,7 @@ def run(self, package=None, **kwargs): total_failed_syntaxes += 1 error_noun = "errors" if total_errors > 1 else "error" - syntax_noun = "syntaxes" if len(syntaxes) > 1 else "syntax" + syntax_noun = "syntaxes" if len(syntaxes) != 1 else "syntax" if total_errors: stream.write( "FAILED: %d %s in %d of %d %s\n" @@ -109,3 +115,17 @@ def run(self, package=None, **kwargs): stream.write("\n") stream.write(DONE_MESSAGE) stream.close() + + +def find_package_resources(package, pattern, tests_dir=None): + resource_prefix = "Packages/%s/" % package + if tests_dir: + tests_dir = tests_dir.replace("\\", "/").strip("/") + if tests_dir != ".": + resource_prefix += tests_dir + "/" + + return [ + resource + for resource in sublime.find_resources(pattern) + if resource.startswith(resource_prefix) + ]