diff --git a/CHANGELOG.md b/CHANGELOG.md index dc810923..a3ca52ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - `--verbose` warns on Bash 3.x that coverage does not count lines run inside a subshell, so a percentage that reads lower there than on Bash 4+ explains itself (#1112) ### Changed +- Docs: `assert_match_snapshot` warns that a `@data_provider` test shares one snapshot across all its values — the filename comes from the test function, so the first value creates it and the rest fail against its content. `assert_match_named_snapshot "$1"` gives each value its own (#1185) - A test file that fails to source without writing to stderr now reports its size and says there was no stderr, so a truncated file can be told apart from one whose last command returned non-zero (#1137) - Performance: cold start makes two fewer forks — `check_os::init` ran twice, once at source time and again from the entrypoint, and the root directory came from `$(dirname …)` — worth about 4ms of a 65ms startup, on every invocation (#1124) - Performance: `--coverage` is roughly 5x faster and `--coverage-report-html` roughly 19x — a run over this repo went from 16.2s to 2.9s, and a 128-file HTML report from 58.7s to 3.1s. The report phase emits each format in one awk invocation per run instead of Bash loops and forks per file and per row, and the capture path writes records straight to disk, normalizes a path with one fork instead of four, and reads each cache once (#1092, #1096, #1098, #1099, #1102, #1104, #1110, #1117) diff --git a/docs/assertions.md b/docs/assertions.md index 9eeac0ba..67860f04 100644 --- a/docs/assertions.md +++ b/docs/assertions.md @@ -2050,6 +2050,30 @@ Reports an error if `actual` differs from the stored snapshot. On the first run Pass `snapshot_file` to share one snapshot between tests; by default each test gets its own, named after the test function. +::: warning A `@data_provider` test shares one snapshot +The filename comes from the test **function**, and a provider runs that function +once per value — so every value compares against the same file. The first value +to run creates it; the rest fail with `Expected to match the snapshot` even +though nothing is wrong with them: + +```bash +function provide_values() { echo "alpha"; echo "beta"; } + +# @data_provider provide_values +function test_shared() { + assert_match_snapshot "value is $1" # one file for both values +} + +# @data_provider provide_values +function test_per_value() { + assert_match_named_snapshot "$1" "value is $1" # one file per value +} +``` + +Use [assert_match_named_snapshot](#assert-match-named-snapshot) with the value +as the name to give each data set its own. +::: + See [Snapshots](/snapshots) for the full workflow, including how to update a snapshot after an intentional change. ::: code-group diff --git a/tests/acceptance/bashunit_snapshot_provider_test.sh b/tests/acceptance/bashunit_snapshot_provider_test.sh new file mode 100644 index 00000000..11c48bd3 --- /dev/null +++ b/tests/acceptance/bashunit_snapshot_provider_test.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +set -euo pipefail + +# A test driven by a @data_provider runs once per data set, but an unnamed +# snapshot is named after the test *function* -- so every data set shares one +# file. The first one to run creates it and the rest compare against its +# content and fail with "Expected to match the snapshot", which describes the +# symptom and hides the cause: the snapshot is shared, not wrong (#1185). +# +# The behaviour itself is deliberate -- the filename is documented as the test +# function's, and `assert_match_named_snapshot` gives one file per value. What +# was missing is any mention of the interaction, so this pins the behaviour and +# docs/assertions.md now warns about it. + +function set_up_before_script() { + BASHUNIT_BIN="$(pwd)/bashunit" +} + +function set_up() { + WORKDIR="$(mktemp -d)" + { + printf '%s\n' '#!/usr/bin/env bash' + printf '%s\n' 'function provide_values() { echo "alpha"; echo "beta"; }' + printf '%s\n' '# @data_provider provide_values' + printf '%s\n' 'function test_shared_snapshot() { assert_match_snapshot "value is $1"; }' + } >"$WORKDIR/p_test.sh" +} + +function tear_down() { + rm -rf "$WORKDIR" +} + +function _run() { + (cd "$WORKDIR" && "$BASHUNIT_BIN" --no-parallel p_test.sh 2>&1) || true +} + +# The behaviour, pinned: the second data set fails against the first one's +# snapshot. Documented in docs/assertions.md, because the failure renderer has +# no signal for "this test ran with provider arguments" -- exposing one is a +# design change, not a message fix. +function test_a_second_data_set_fails_against_the_first_ones_snapshot() { + _run >/dev/null # first run creates the snapshot for the first data set + + local output + output="$(_run)" + + assert_contains "Expected to match the snapshot" "$output" + assert_contains "1 passed" "$output" +} + +# Only one file, which is the documented behaviour and stays that way: giving +# each data set its own would orphan every snapshot already on disk. +function test_one_snapshot_file_is_still_written_for_all_data_sets() { + _run >/dev/null + + local count + count=$(find "$WORKDIR/snapshots" -name '*.snapshot' | wc -l | tr -d ' ') + + assert_same 1 "$count" +} + +# A test without a provider must not gain the hint. +function test_a_plain_test_mismatch_says_nothing_about_providers() { + { + printf '%s\n' '#!/usr/bin/env bash' + printf '%s\n' 'function test_plain() { assert_match_snapshot "$SNAP_VALUE"; }' + } >"$WORKDIR/q_test.sh" + + (cd "$WORKDIR" && SNAP_VALUE=one "$BASHUNIT_BIN" --no-parallel q_test.sh >/dev/null 2>&1) || true + + local output + output="$( (cd "$WORKDIR" && SNAP_VALUE=two "$BASHUNIT_BIN" --no-parallel q_test.sh 2>&1) || true)" + + assert_contains "snapshot" "$output" + assert_not_contains "data provider" "$output" +} diff --git a/tests/acceptance/snapshots/bashunit_test_sh.test_bashunit_should_display_all_assert_docs.snapshot b/tests/acceptance/snapshots/bashunit_test_sh.test_bashunit_should_display_all_assert_docs.snapshot index 5c91e6e8..adaacac3 100644 --- a/tests/acceptance/snapshots/bashunit_test_sh.test_bashunit_should_display_all_assert_docs.snapshot +++ b/tests/acceptance/snapshots/bashunit_test_sh.test_bashunit_should_display_all_assert_docs.snapshot @@ -813,7 +813,11 @@ Reports an error if `actual` differs from the stored snapshot. On the first run Pass `snapshot_file` to share one snapshot between tests; by default each test gets its own, named after the test function. -See Snapshots(/snapshots) for the full workflow, including how to update a snapshot after an intentional change. +::: warning A `@data_provider` test shares one snapshot +The filename comes from the test **function**, and a provider runs that function +once per value — so every value compares against the same file. The first value +to run creates it; the rest fail with `Expected to match the snapshot` even +though nothing is wrong with them: ## assert_match_snapshot_ignore_colors