diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c9762e7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,21 @@ +name: ci + +on: + push: + branches: ['**'] + pull_request: + +jobs: + test: + strategy: + fail-fast: false + matrix: + runner: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version-file: go.mod + - run: go test -race ./... + - run: go vet ./... diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 21aa86f..4948e6d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,15 +12,46 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 with: go-version-file: go.mod - - run: go test ./... + - run: go test -race ./... + - run: go vet ./... - release: + validate-release: needs: test runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - { goos: linux, goarch: amd64, cgo: '0', runner: ubuntu-latest } + - { goos: linux, goarch: arm64, cgo: '0', runner: ubuntu-latest } + - { goos: linux, goarch: arm, goarm: '7', cgo: '0', runner: ubuntu-latest } + - { goos: linux, goarch: '386', cgo: '0', runner: ubuntu-latest } + - { goos: darwin, goarch: amd64, cgo: '1', runner: macos-26-intel } + - { goos: darwin, goarch: arm64, cgo: '1', runner: macos-latest } + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version-file: go.mod + - name: Test native platform + if: matrix.goos == 'darwin' || matrix.goarch == 'amd64' + run: go test ./... + - name: Validate release build + env: + CGO_ENABLED: ${{ matrix.cgo }} + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + GOARM: ${{ matrix.goarm }} + MACOSX_DEPLOYMENT_TARGET: '12.0' + run: go build -trimpath -o /tmp/snailrace ./cmd/snailrace + + release: + needs: validate-release + runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: @@ -32,8 +63,8 @@ jobs: - { goos: darwin, goarch: amd64, cgo: '1', runner: macos-26-intel, label: darwin-amd64 } - { goos: darwin, goarch: arm64, cgo: '1', runner: macos-latest, label: darwin-arm64 } steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 with: go-version-file: go.mod - name: Test Darwin @@ -45,7 +76,7 @@ jobs: GOOS: ${{ matrix.goos }} GOARCH: ${{ matrix.goarch }} GOARM: ${{ matrix.goarm }} - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos_target || '12.0' }} + MACOSX_DEPLOYMENT_TARGET: '12.0' run: | go build -trimpath \ -ldflags="-s -w -X github.com/shellcell/snailrace/internal/app.Version=${GITHUB_REF_NAME}" \ @@ -72,7 +103,7 @@ jobs: nfpm package -f /tmp/nfpm.yaml -p "$fmt" -t pkgs done - name: Upload to release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: files: | ${{ env.ARCHIVE }} diff --git a/README.md b/README.md index 7e20701..a129be3 100644 --- a/README.md +++ b/README.md @@ -60,12 +60,14 @@ Compare shell commands: ./snailrace -n 20 -c 'grep needle data.txt' -c 'rg needle data.txt' ``` -Save an HTML report: +An HTML report is saved to the current directory by default. Choose another directory: ```sh ./snailrace -format html -output ./reports -- ./my-program --flag ``` +Use `-no-save` when only stdout output is wanted. + Measure an interactive TUI: ```sh @@ -85,8 +87,9 @@ Measure an interactive TUI: | `-interval` | Sampling interval; default 10 ms | | `-index` | Index dimensions: `time,cpu,ram,disk` | | `-baseline` | 1-based baseline; `0` selects the winner | -| `-f`, `-format` | `html`, `svg`, `markdown`, `json`, or `text` | -| `-o`, `-output` | Report directory | +| `-f`, `-format` | Saved format; default `html` | +| `-o`, `-output` | Report directory; default current directory | +| `-no-save` | Do not save report files | | `-verbose` | Full statistical tables on stdout | | `-show-output` | Forward command output to stderr | | `tui` | Run inside a pseudo-terminal | @@ -106,6 +109,8 @@ Measure an interactive TUI: Raw runs and summary statistics are available in JSON. Reports include mean, sample standard deviation, median, p95, range, and a 95% Student's t interval. +Every format identifies the dimensions actually included in the balanced index +and records sampling or metric-availability caveats. ## Method @@ -113,21 +118,32 @@ sample standard deviation, median, p95, range, and a 95% Student's t interval. - The default index combines normalized time, CPU, and RAM costs. - RAM cost combines mean and peak RSS. - Sampling-limited RAM is excluded from the index. +- Commands with non-zero measured exits continue running but are excluded from rankings. +- If none of the selected dimensions is usable, reports mark the balanced ranking unavailable. - Ctrl+C keeps completed comparison rounds. - Command output is discarded unless `-show-output` is set. +- Native executables run through the inspected artifact and are identity/hash-checked after measurement. +- Scripts preserve their original invocation path so `$0`-relative behavior is unchanged. + +Saved reports are staged before publication and receive a numeric suffix rather +than overwriting an existing report with the same timestamp. ## Platform Notes -- Linux reads `/proc` and follows each process's children. +- Linux reads `/proc` and measures every member of the command's process group, + including descendants reparented while the benchmark is running. - macOS uses cgo with `proc_listpgrppids`, `proc_pidinfo`, and `proc_pid_rusage`. - macOS physical footprint estimates memory pressure charged to each process. +- Physical-footprint validity is tracked per run; unavailable samples are not treated as zero. - Tree RSS can double-count shared pages. - Sampled peaks can miss activity shorter than the interval. - macOS does not report file descriptor counts. ## Disk Footprint +- For a simple external `-c` command, disk footprint describes its leading executable; + shell builtins describe the shell itself. - Disk-backed `@rpath`, `@loader_path`, and `@executable_path` libraries are resolved recursively. - macOS dyld shared-cache libraries are listed but excluded from byte totals. diff --git a/completions/_snailrace b/completions/_snailrace index fbfb435..a56dd9f 100644 --- a/completions/_snailrace +++ b/completions/_snailrace @@ -13,6 +13,7 @@ _snailrace() { '-baseline[set 1-based baseline]:index:' \ '*'{-f,-format}'[select report format]:format:(html svg markdown json text)' \ '(-o -output)'{-o,-output}'[set report directory]:directory:_directories' \ + '-no-save[do not save report files]' \ '-verbose[print full statistical tables]' \ '-show-output[forward command output]' \ '(-d -duration)'{-d,-duration}'[set fixed TUI duration]:duration:' \ diff --git a/completions/snailrace.bash b/completions/snailrace.bash index af2dbe5..f902e05 100644 --- a/completions/snailrace.bash +++ b/completions/snailrace.bash @@ -2,7 +2,7 @@ _snailrace() { local cur prev options command_seen expect_value after_separator word i cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" - options="-c -command -label -prepare -n -runs -warmups -interval -index -baseline -f -format -o -output -verbose -show-output -d -duration -width -height -v -version -h -help" + options="-c -command -label -prepare -n -runs -warmups -interval -index -baseline -f -format -o -output -no-save -verbose -show-output -d -duration -width -height -v -version -h -help" command_seen=0 expect_value=0 diff --git a/completions/snailrace.fish b/completions/snailrace.fish index 910554d..d44b441 100644 --- a/completions/snailrace.fish +++ b/completions/snailrace.fish @@ -8,6 +8,7 @@ complete -c snailrace -l index -r -a 'time cpu ram disk' -d 'Set index dimension complete -c snailrace -l baseline -r -d 'Set 1-based baseline' complete -c snailrace -s f -l format -r -a 'html svg markdown json text' -d 'Select report format' complete -c snailrace -s o -l output -r -a '(__fish_complete_directories)' -d 'Set report directory' +complete -c snailrace -l no-save -d 'Do not save report files' complete -c snailrace -l verbose -d 'Print full statistical tables' complete -c snailrace -l show-output -d 'Forward command output' complete -c snailrace -s d -l duration -r -d 'Set fixed TUI duration' diff --git a/docs/snailrace.1 b/docs/snailrace.1 index 7b1a744..a1c6c07 100644 --- a/docs/snailrace.1 +++ b/docs/snailrace.1 @@ -29,6 +29,15 @@ Commands are given either after a separator or with repeated .Fl c flags. +.Pp +Commands with non-zero measured exits continue through all configured rounds, +but are reported as failures and excluded from rankings. +If none of the selected dimensions is usable, the balanced ranking is marked +unavailable instead of selecting a winner. +.Pp +For simple external shell commands, executable metadata and disk footprint +describe the leading command. Shell builtins describe the shell itself. +Scripts retain their original invocation path. The .Cm tui subcommand renders a live terminal view while measuring. @@ -71,8 +80,14 @@ Saved report format: .Ql markdown , .Ql json , .Ql text . +The default is +.Ql html . .It Fl o , Fl output Ar string -Directory for a saved report. +Directory for a saved report; defaults to the current directory. +Reports are staged before publication and receive a numeric suffix instead of +overwriting an existing report with the same name. +.It Fl no-save +Do not save report files; write the text report to standard output only. .It Fl verbose Print full statistical tables to stdout. .It Fl show-output @@ -92,7 +107,7 @@ TUI rows; default inherits the terminal. .It Fl v , Fl version Print version and exit. .It Fl h , Fl help -Show help. +Show help and exit successfully. .El .Sh EXAMPLES Benchmark one command with warmups: diff --git a/go.mod b/go.mod index caa53ef..f69df60 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,12 @@ module github.com/shellcell/snailrace -go 1.22 +go 1.25.0 + +toolchain go1.25.12 require ( github.com/creack/pty v1.1.24 - golang.org/x/term v0.21.0 + golang.org/x/term v0.45.0 ) -require golang.org/x/sys v0.21.0 // indirect +require golang.org/x/sys v0.47.0 diff --git a/go.sum b/go.sum index 7fb5fea..59d547f 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,6 @@ github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= diff --git a/internal/analysis/ranking.go b/internal/analysis/ranking.go index 1d90b9c..97c2af0 100644 --- a/internal/analysis/ranking.go +++ b/internal/analysis/ranking.go @@ -8,18 +8,20 @@ import ( ) type Ranking struct { - Rows []RankingRow - RAMAvailable bool - RAMPresent bool - BestOverall float64 - FixedTUI bool - PrimaryRatio bool - CPURatio bool - FootprintRatio bool - IndexPrimary bool - IndexCPU bool - IndexRAM bool - IndexFootprint bool + Rows []RankingRow + Available bool + UnavailableReason string + RAMAvailable bool + RAMPresent bool + BestOverall float64 + FixedTUI bool + PrimaryRatio bool + CPURatio bool + FootprintRatio bool + IndexPrimary bool + IndexCPU bool + IndexRAM bool + IndexFootprint bool } type RankingRow struct { @@ -42,16 +44,22 @@ type RankingRow struct { func Calculate(config model.Config, benchmarks []model.Benchmark) Ranking { count := len(benchmarks) - result := Ranking{Rows: make([]RankingRow, count)} + result := Ranking{} if count == 0 { + result.UnavailableReason = "no benchmarks" return result } - result.FixedTUI = config.Mode == "tui" && config.DurationSeconds > 0 - result.RAMAvailable = samplesReliable(config, benchmarks) + result.FixedTUI = config.FixedDurationTUI() + eligible := make([]bool, count) + eligibleCount := 0 primary, cpu := make([]float64, count), make([]float64, count) meanRAM, peakRAM := make([]float64, count), make([]float64, count) footprint := make([]float64, count) for index, benchmark := range benchmarks { + eligible[index] = benchmark.EligibleForRanking() + if eligible[index] { + eligibleCount++ + } primary[index] = benchmark.Summary.WallSeconds.Mean cpu[index] = benchmark.Summary.CPUTotalSeconds.Mean if result.FixedTUI { @@ -62,24 +70,36 @@ func Calculate(config model.Config, benchmarks []model.Benchmark) Ranking { peakRAM[index] = benchmark.Summary.PeakResidentBytes.Mean footprint[index] = float64(benchmark.Tool.DiskFootprintBytes) } - if !hasPositive(meanRAM) || !hasPositive(peakRAM) { + result.RAMAvailable = samplesReliable(config, benchmarks, eligible) + if !hasPositive(meanRAM, eligible) || !hasPositive(peakRAM, eligible) { result.RAMAvailable = false } - result.RAMPresent = allPositive(meanRAM) && allPositive(peakRAM) + result.RAMPresent = allPositive(meanRAM, eligible) && allPositive(peakRAM, eligible) result.RAMAvailable = result.RAMAvailable && result.RAMPresent - result.PrimaryRatio = allPositive(primary) - result.CPURatio = allPositive(cpu) - result.FootprintRatio = allPositive(footprint) - primaryScore, cpuScore := normalized(primary), normalized(cpu) - ramScore := normalizedPair(meanRAM, peakRAM, result.RAMAvailable) - footprintScore := normalized(footprint) + result.PrimaryRatio = allPositive(primary, eligible) + result.CPURatio = allPositive(cpu, eligible) + result.FootprintRatio = allPositive(footprint, eligible) + primaryScore, cpuScore := normalized(primary, eligible), normalized(cpu, eligible) + ramScore := normalizedPair(meanRAM, peakRAM, eligible, result.RAMAvailable) + footprintScore := normalized(footprint, eligible) included := indexSet(config.IndexDimensions) result.IndexPrimary = included["time"] && !result.FixedTUI && result.PrimaryRatio result.IndexCPU = (included["cpu"] || (result.FixedTUI && included["time"])) && result.CPURatio result.IndexRAM = included["ram"] && result.RAMAvailable result.IndexFootprint = included["disk"] && result.FootprintRatio - for index := range result.Rows { + result.Available = eligibleCount > 0 && (result.IndexPrimary || result.IndexCPU || + result.IndexRAM || result.IndexFootprint) + if eligibleCount == 0 { + result.UnavailableReason = "no successful measured runs" + } else if !result.Available { + result.UnavailableReason = "none of the selected dimensions has usable positive values" + } + result.Rows = make([]RankingRow, 0, eligibleCount) + for index := range benchmarks { + if !eligible[index] { + continue + } var scores []float64 if result.IndexPrimary { scores = append(scores, primaryScore[index]) @@ -97,39 +117,45 @@ func Calculate(config model.Config, benchmarks []model.Benchmark) Ranking { if result.RAMPresent { ramValue = geometricMean([]float64{meanRAM[index], peakRAM[index]}) } - result.Rows[index] = RankingRow{ - Benchmark: index, OverallScore: geometricMean(scores), + overallScore := 0.0 + if result.Available { + overallScore = geometricMean(scores) + } + result.Rows = append(result.Rows, RankingRow{ + Benchmark: index, OverallScore: overallScore, PrimaryScore: primaryScore[index], CPUScore: cpuScore[index], RAMScore: ramScore[index], FootprintScore: footprintScore[index], PrimaryValue: primary[index], CPUValue: cpu[index], RAMValue: ramValue, FootprintValue: footprint[index], - } + }) } - applyRanks(result.Rows, primaryScore, cpuScore, ramScore, footprintScore) - result.BestOverall = math.Inf(1) - for _, row := range result.Rows { - result.BestOverall = math.Min(result.BestOverall, row.OverallScore) + applyRanks( + result.Rows, eligible, result.Available, + primaryScore, cpuScore, ramScore, footprintScore, + ) + if result.Available { + result.BestOverall = math.Inf(1) + for _, row := range result.Rows { + result.BestOverall = math.Min(result.BestOverall, row.OverallScore) + } } return result } -func AutomaticBaseline(config model.Config, benchmarks []model.Benchmark) int { - ranking := Calculate(config, benchmarks) - if len(ranking.Rows) == 0 { - return 1 - } - return ranking.Rows[0].Benchmark + 1 -} +var indexDimensions = [...]string{"time", "cpu", "ram", "disk"} +var defaultIndexDimensions = [...]string{"time", "cpu", "ram"} -// IndexDimensions are the cost categories that may compose the balanced index. -var IndexDimensions = []string{"time", "cpu", "ram", "disk"} +func IndexDimensions() []string { + return append([]string(nil), indexDimensions[:]...) +} -// DefaultIndexDimensions is the balanced index used when none is configured. -var DefaultIndexDimensions = []string{"time", "cpu", "ram"} +func DefaultIndexDimensions() []string { + return append([]string(nil), defaultIndexDimensions[:]...) +} // ValidIndexDimension reports whether token names a known index dimension. func ValidIndexDimension(token string) bool { - for _, dimension := range IndexDimensions { + for _, dimension := range indexDimensions { if token == dimension { return true } @@ -139,7 +165,7 @@ func ValidIndexDimension(token string) bool { func indexSet(dimensions []string) map[string]bool { if len(dimensions) == 0 { - dimensions = DefaultIndexDimensions + dimensions = defaultIndexDimensions[:] } set := make(map[string]bool, len(dimensions)) for _, dimension := range dimensions { @@ -157,19 +183,41 @@ func BalancedIndexes(config model.Config, benchmarks []model.Benchmark) []float6 return indexes } -func applyRanks(rows []RankingRow, primary, cpu, ram, footprint []float64) { - overall := make([]float64, len(rows)) - for index := range rows { - overall[index] = rows[index].OverallScore +func applyRanks( + rows []RankingRow, + eligible []bool, + overallAvailable bool, + primary, cpu, ram, footprint []float64, +) { + overall := make([]float64, len(eligible)) + for index := range overall { + overall[index] = math.Inf(1) } + for _, row := range rows { + overall[row.Benchmark] = row.OverallScore + } + overallRanks := make([]int, len(eligible)) + if overallAvailable { + overallRanks = ranks(overall, eligible) + } + primaryRanks := ranks(primary, eligible) + cpuRanks := ranks(cpu, eligible) + ramRanks := ranks(ram, eligible) + footprintRanks := ranks(footprint, eligible) for index := range rows { - rows[index].OverallRank = rankOf(overall, index) - rows[index].PrimaryRank = rankOf(primary, index) - rows[index].CPURank = rankOf(cpu, index) - rows[index].RAMRank = rankOf(ram, index) - rows[index].FootprintRank = rankOf(footprint, index) + benchmark := rows[index].Benchmark + if overallAvailable { + rows[index].OverallRank = overallRanks[benchmark] + } + rows[index].PrimaryRank = primaryRanks[benchmark] + rows[index].CPURank = cpuRanks[benchmark] + rows[index].RAMRank = ramRanks[benchmark] + rows[index].FootprintRank = footprintRanks[benchmark] } sort.SliceStable(rows, func(i, j int) bool { + if !overallAvailable { + return rows[i].Benchmark < rows[j].Benchmark + } return rows[i].OverallRank < rows[j].OverallRank }) } diff --git a/internal/analysis/ranking_math.go b/internal/analysis/ranking_math.go index c1a7a59..1b32900 100644 --- a/internal/analysis/ranking_math.go +++ b/internal/analysis/ranking_math.go @@ -2,14 +2,18 @@ package analysis import ( "math" + "sort" "github.com/shellcell/snailrace/internal/model" ) -func normalized(values []float64) []float64 { +func normalized(values []float64, eligible []bool) []float64 { minimum := math.Inf(1) hasZero := false - for _, value := range values { + for index, value := range values { + if !eligible[index] { + continue + } hasZero = hasZero || value == 0 if value > 0 && value < minimum { minimum = value @@ -18,6 +22,8 @@ func normalized(values []float64) []float64 { result := make([]float64, len(values)) for index, value := range values { switch { + case !eligible[index]: + result[index] = math.Inf(1) case value == 0: result[index] = 1 case math.IsInf(minimum, 1) || value < 0: @@ -31,7 +37,7 @@ func normalized(values []float64) []float64 { return result } -func normalizedPair(first, second []float64, available bool) []float64 { +func normalizedPair(first, second []float64, eligible []bool, available bool) []float64 { result := make([]float64, len(first)) if !available { for index := range result { @@ -39,11 +45,11 @@ func normalizedPair(first, second []float64, available bool) []float64 { } return result } - one, two := normalized(first), normalized(second) + one, two := normalized(first, eligible), normalized(second, eligible) for index := range result { result[index] = geometricMean([]float64{one[index], two[index]}) } - return normalized(result) + return normalized(result, eligible) } func geometricMean(values []float64) float64 { @@ -60,49 +66,66 @@ func geometricMean(values []float64) float64 { return math.Exp(total / float64(count)) } -func samplesReliable(config model.Config, benchmarks []model.Benchmark) bool { +func samplesReliable(config model.Config, benchmarks []model.Benchmark, eligible []bool) bool { minimum := config.IntervalMS / 1000 * 2 if minimum <= 0 { return true } - for _, benchmark := range benchmarks { - for _, run := range benchmark.Runs { - if run.WallSeconds < minimum || run.SampleCount < 2 || - run.SampleCoverageSeconds < config.IntervalMS/1000 { - return false - } + for index, benchmark := range benchmarks { + if !eligible[index] { + continue + } + if !model.SamplingReliable(benchmark, config.IntervalMS/1000) { + return false } } return true } -func hasPositive(values []float64) bool { - for _, value := range values { - if value > 0 { +func hasPositive(values []float64, eligible []bool) bool { + for index, value := range values { + if eligible[index] && value > 0 { return true } } return false } -func allPositive(values []float64) bool { - if len(values) == 0 { - return false - } - for _, value := range values { +func allPositive(values []float64, eligible []bool) bool { + found := false + for index, value := range values { + if !eligible[index] { + continue + } + found = true if value <= 0 || math.IsInf(value, 0) || math.IsNaN(value) { return false } } - return true + return found } -func rankOf(values []float64, target int) int { - rank := 1 +func ranks(values []float64, eligible []bool) []int { + type item struct { + index int + value float64 + } + items := make([]item, 0, len(values)) for index, value := range values { - if index != target && value < values[target] { - rank++ + if eligible[index] && !math.IsInf(value, 0) && !math.IsNaN(value) { + items = append(items, item{index: index, value: value}) + } + } + sort.SliceStable(items, func(left, right int) bool { + return items[left].value < items[right].value + }) + result := make([]int, len(values)) + rank := 0 + for position, current := range items { + if position == 0 || current.value != items[position-1].value { + rank = position + 1 } + result[current.index] = rank } - return rank + return result } diff --git a/internal/analysis/ranking_test.go b/internal/analysis/ranking_test.go index 2af36c8..62c75fd 100644 --- a/internal/analysis/ranking_test.go +++ b/internal/analysis/ranking_test.go @@ -1,6 +1,7 @@ package analysis import ( + "math" "testing" "github.com/shellcell/snailrace/internal/model" @@ -12,15 +13,62 @@ func TestDefaultIndexExcludesDiskFootprint(t *testing.T) { indexBenchmark("fast-but-fat", 1, 1, 100, 100, 1_000_000), } // Default index (time, cpu, ram) ranks the fast tool first despite its size. - if got := AutomaticBaseline(model.Config{Mode: "command"}, benchmarks); got != 2 { - t.Fatalf("default baseline = %d, want fast tool (2)", got) + ranking := Calculate(model.Config{Mode: "command"}, benchmarks) + if !ranking.Available || ranking.Rows[0].Benchmark+1 != 2 { + t.Fatalf("default baseline = %d, want fast tool (2)", ranking.Rows[0].Benchmark+1) } // Including disk lets the huge footprint sink the fast tool. withDisk := model.Config{ Mode: "command", IndexDimensions: []string{"time", "cpu", "ram", "disk"}, } - if got := AutomaticBaseline(withDisk, benchmarks); got != 1 { - t.Fatalf("disk baseline = %d, want compact tool (1)", got) + ranking = Calculate(withDisk, benchmarks) + if !ranking.Available || ranking.Rows[0].Benchmark+1 != 1 { + t.Fatalf("disk baseline = %d, want compact tool (1)", ranking.Rows[0].Benchmark+1) + } +} + +func TestUnavailableRankingUsesExplicitState(t *testing.T) { + benchmarks := []model.Benchmark{indexBenchmark("short", 1, 1, 100, 100, 10)} + config := model.Config{ + Mode: "command", IntervalMS: 10, IndexDimensions: []string{"ram"}, + } + ranking := Calculate(config, benchmarks) + if ranking.Available || ranking.UnavailableReason == "" { + t.Fatalf("ranking = %+v, want explicit unavailable state", ranking) + } + if len(ranking.Rows) != 1 || ranking.Rows[0].OverallRank != 0 || + math.IsInf(ranking.Rows[0].OverallScore, 0) || math.IsNaN(ranking.Rows[0].OverallScore) { + t.Fatalf("unavailable row contains invalid ranking values: %+v", ranking.Rows) + } +} + +func TestFailedBenchmarkIsExcludedFromRanking(t *testing.T) { + failed := indexBenchmark("failed", 0.001, 0.001, 100, 100, 10) + failed.Runs[0].ExitCode = 7 + failed.Summary = model.Summarize(failed.Runs) + success := indexBenchmark("success", 1, 1, 200, 200, 20) + config := model.Config{ + Mode: "command", IndexDimensions: []string{"time", "cpu"}, + } + ranking := Calculate(config, []model.Benchmark{failed, success}) + if !ranking.Available || len(ranking.Rows) != 1 || ranking.Rows[0].Benchmark != 1 { + t.Fatalf("failed tool was not excluded: %+v", ranking) + } + if ranking.Rows[0].Benchmark+1 != 2 { + t.Fatalf("baseline = %d, want successful tool 2", ranking.Rows[0].Benchmark+1) + } +} + +func TestDimensionDefaultsCannotBeMutatedByCallers(t *testing.T) { + dimensions := DefaultIndexDimensions() + dimensions[0] = "disk" + if got := DefaultIndexDimensions()[0]; got != "time" { + t.Fatalf("mutated default dimension = %q", got) + } + all := IndexDimensions() + all[0] = "disk" + if got := IndexDimensions()[0]; got != "time" { + t.Fatalf("mutated known dimension = %q", got) } } diff --git a/internal/app/app.go b/internal/app/app.go index e4f4d88..1a16b9f 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -3,10 +3,10 @@ package app import ( "context" "errors" + "flag" "fmt" "io" "os" - "os/exec" "os/signal" "runtime" "syscall" @@ -26,6 +26,9 @@ var Version = "dev" func Run(arguments []string, stdout, stderr io.Writer) error { options, err := parseOptions(arguments, stderr) if err != nil { + if errors.Is(err, flag.ErrHelp) { + return nil + } return err } if options.version { @@ -85,7 +88,7 @@ func Run(arguments []string, stdout, stderr io.Writer) error { WarmupOrder: warmupOrder, }, runner.Options{ ShowOutput: options.showOutput, TUI: options.tui, - Output: stderr, + Output: stderr, Input: os.Stdin, TerminalOut: os.Stdout, Duration: options.duration, Width: width, Height: height, Interactive: interactiveTUI, FollowResize: followResize, @@ -100,10 +103,29 @@ func Run(arguments []string, stdout, stderr io.Writer) error { if err != nil && !interrupted { return err } + rankingNote := "" + if config.Baseline == 0 { + ranking := analysis.Calculate(config, benchmarks) + if ranking.Available && len(ranking.Rows) > 0 { + config.Baseline = ranking.Rows[0].Benchmark + 1 + } else { + config.Baseline = firstEligibleBenchmark(benchmarks) + 1 + config.BaselineAutomatic = false + rankingNote = fmt.Sprintf( + "Balanced ranking unavailable: %s. %s is used only as the comparison baseline.", + ranking.UnavailableReason, + benchmarks[config.Baseline-1].Tool.Name, + ) + } + } notes := platformNotes( options.tui, options.duration, len(options.specs) > 1, config.BaselineAutomatic, config.OrderSeed, config.OutputMode, ) + if rankingNote != "" { + fmt.Fprintln(stderr, rankingNote) + notes = append([]string{rankingNote}, notes...) + } if interrupted { completed := 0 if len(benchmarks) > 0 { @@ -125,21 +147,31 @@ func Run(arguments []string, stdout, stderr io.Writer) error { fmt.Fprintln(stderr, hint) notes = append(notes, hint) } - if config.Baseline == 0 { - config.Baseline = analysis.AutomaticBaseline(config, benchmarks) - } _, host.MemoryAfterBytes = platform.Memory() result := model.Report{ MeasuredAt: measuredAt, Config: config, Host: host, Benchmarks: benchmarks, Verbose: options.verbose, Notes: notes, } + outputDirectory := options.output + if options.noSave { + outputDirectory = "" + } if err := writeResult( - stdout, stderr, options.output, options.formats, result, + stdout, stderr, outputDirectory, options.formats, result, ); err != nil { return err } - return checkExitCodes(benchmarks) + return nil +} + +func firstEligibleBenchmark(benchmarks []model.Benchmark) int { + for index, benchmark := range benchmarks { + if benchmark.EligibleForRanking() { + return index + } + } + return 0 } func oneBasedOrder(order [][]int) [][]int { @@ -170,9 +202,7 @@ func prepare(ctx context.Context, command string, output io.Writer) error { if command == "" { return nil } - cmd := exec.CommandContext(ctx, "/bin/sh", "-c", command) - cmd.Stdout, cmd.Stderr = output, output - if err := cmd.Run(); err != nil { + if err := runner.RunPreparation(ctx, command, output); err != nil { return fmt.Errorf("prepare command: %w", err) } return nil @@ -184,27 +214,15 @@ func writeResult( formats []string, result model.Report, ) error { - if err := report.Write(stdout, "text", result); err != nil { - return err - } - if directory == "" { - return nil - } - return saveReportFormats(stderr, directory, formats, result) -} - -func checkExitCodes(benchmarks []model.Benchmark) error { - for _, benchmark := range benchmarks { - for _, run := range benchmark.Runs { - if run.ExitCode != 0 && run.StopReason != "duration" { - return fmt.Errorf( - "%q exited unsuccessfully in one or more runs", - benchmark.Tool.Name, - ) - } - } + renderer := report.NewRenderer(result) + var saveErr error + if directory != "" { + saveErr = saveReportFormats( + stderr, directory, formats, renderer, + ) } - return nil + textErr := renderer.Write(stdout, "text") + return errors.Join(saveErr, textErr) } func modeName(tui bool) string { diff --git a/internal/app/filename.go b/internal/app/filename.go index c6b3b0e..f5c93a3 100644 --- a/internal/app/filename.go +++ b/internal/app/filename.go @@ -4,39 +4,45 @@ import ( "fmt" "path/filepath" "strings" + "time" "unicode" "github.com/shellcell/snailrace/internal/model" + "github.com/shellcell/snailrace/internal/report" ) -func reportPath(directory, format string, report model.Report) string { +func reportPathWithStem(directory, format, stem string) string { extension := strings.ToLower(format) - switch extension { - case "text": - extension = "txt" - case "markdown": - extension = "md" + if info, ok := report.LookupFormat(format); ok { + extension = info.Extension } - return filepath.Join(directory, reportStem(report)+"."+extension) + return filepath.Join(directory, stem+"."+extension) } func reportStem(report model.Report) string { - tool := reportLabel(report.Benchmarks) - timestamp := report.MeasuredAt.Format("20060102-150405") + names := make([]string, len(report.Benchmarks)) + for index, benchmark := range report.Benchmarks { + names[index] = benchmark.Tool.Name + } + return reportStemFor(report.MeasuredAt, names) +} + +func reportStemFor(measuredAt time.Time, toolNames []string) string { + tool := reportLabelNames(toolNames) + timestamp := measuredAt.Format("20060102-150405") return fmt.Sprintf("snail-%s-%s", tool, timestamp) } -func reportLabel(benchmarks []model.Benchmark) string { - if len(benchmarks) == 0 { +func reportLabelNames(names []string) string { + if len(names) == 0 { return "unknown" } - if len(benchmarks) == 1 { - return slug(benchmarks[0].Tool.Name) + if len(names) == 1 { + return slug(names[0]) } - label := slug(benchmarks[0].Tool.Name) + "-vs-" + - slug(benchmarks[1].Tool.Name) - if len(benchmarks) > 2 { - label += fmt.Sprintf("-and-%d", len(benchmarks)-2) + label := slug(names[0]) + "-vs-" + slug(names[1]) + if len(names) > 2 { + label += fmt.Sprintf("-and-%d", len(names)-2) } return label } diff --git a/internal/app/filename_test.go b/internal/app/filename_test.go index 6c636fb..490c7b4 100644 --- a/internal/app/filename_test.go +++ b/internal/app/filename_test.go @@ -15,7 +15,7 @@ func TestReportPathContainsToolAndMeasurementTime(t *testing.T) { Tool: model.ToolInfo{Name: "My Tool!"}, }}, } - got := reportPath("reports", "markdown", report) + got := reportPathWithStem("reports", "markdown", reportStem(report)) want := filepath.Join("reports", "snail-my-tool-20260711-140509.md") if got != want { t.Fatalf("path = %q, want %q", got, want) @@ -31,7 +31,7 @@ func TestComparisonReportPathNamesMeasuredTools(t *testing.T) { {Tool: model.ToolInfo{Name: "awk"}}, }, } - got := filepath.Base(reportPath(".", "json", report)) + got := filepath.Base(reportPathWithStem(".", "json", reportStem(report))) want := "snail-grep-vs-ripgrep-and-1-20260711-140509.json" if got != want { t.Fatalf("name = %q, want %q", got, want) diff --git a/internal/app/formats.go b/internal/app/formats.go index 050df14..1fe4b1b 100644 --- a/internal/app/formats.go +++ b/internal/app/formats.go @@ -1,6 +1,10 @@ package app -import "strings" +import ( + "strings" + + "github.com/shellcell/snailrace/internal/report" +) type formatValues struct { values []string @@ -23,6 +27,9 @@ func (formats *formatValues) Set(value string) error { for _, item := range strings.Split(value, ",") { item = strings.ToLower(strings.TrimSpace(item)) if item != "" { + if info, ok := report.LookupFormat(item); ok { + item = string(info.Name) + } formats.values = append(formats.values, item) } } diff --git a/internal/app/formats_test.go b/internal/app/formats_test.go index 9d334c1..f2dcd89 100644 --- a/internal/app/formats_test.go +++ b/internal/app/formats_test.go @@ -30,4 +30,19 @@ func TestDefaultFormatIsHTML(t *testing.T) { if !reflect.DeepEqual(options.formats, []string{"html"}) { t.Fatalf("formats = %#v, want html", options.formats) } + if options.output != "." { + t.Fatalf("output = %q, want current directory", options.output) + } +} + +func TestEmptyOutputRequiresNoSave(t *testing.T) { + if _, err := parseOptions([]string{"-o", "", "--", "true"}, io.Discard); err == nil { + t.Fatal("empty output should require -no-save") + } + options, err := parseOptions( + []string{"-no-save", "-o", "", "--", "true"}, io.Discard, + ) + if err != nil || !options.noSave { + t.Fatalf("no-save options = %+v, error = %v", options, err) + } } diff --git a/internal/app/index.go b/internal/app/index.go index e59b874..35dd732 100644 --- a/internal/app/index.go +++ b/internal/app/index.go @@ -13,7 +13,7 @@ type indexValues struct { } func newIndexValues() indexValues { - return indexValues{values: append([]string(nil), analysis.DefaultIndexDimensions...)} + return indexValues{values: analysis.DefaultIndexDimensions()} } func (index *indexValues) String() string { @@ -33,7 +33,7 @@ func (index *indexValues) Set(value string) error { if !analysis.ValidIndexDimension(item) { return fmt.Errorf( "unknown index dimension %q; choose from %s", - item, strings.Join(analysis.IndexDimensions, ", "), + item, strings.Join(analysis.IndexDimensions(), ", "), ) } if !contains(index.values, item) { diff --git a/internal/app/legend.go b/internal/app/legend.go index e0e0f9f..8b65fa6 100644 --- a/internal/app/legend.go +++ b/internal/app/legend.go @@ -10,6 +10,7 @@ import ( "golang.org/x/term" "github.com/shellcell/snailrace/internal/runner" + "github.com/shellcell/snailrace/internal/style" ) // printCommandLegend lists each tool's label and full command before measurement @@ -45,7 +46,7 @@ func printCommandLegend(writer io.Writer, specs []runner.Spec) { func specCommand(spec runner.Spec) string { if spec.Shell != "" { - return spec.Shell + return style.CommandText([]string{spec.Shell}, true) } - return strings.Join(spec.Args, " ") + return style.CommandText(spec.Args, false) } diff --git a/internal/app/options.go b/internal/app/options.go index a640097..7e8937f 100644 --- a/internal/app/options.go +++ b/internal/app/options.go @@ -31,6 +31,7 @@ type options struct { index []string verbose bool showOutput bool + noSave bool version bool tui bool width, height uint16 @@ -80,8 +81,9 @@ func parseOptions(arguments []string, stderr io.Writer) (options, error) { "saved format; repeat or comma-separate: html, svg, markdown, json, text", ) flags.Var(&formats, "f", "saved format (shorthand)") - flags.StringVar(&result.output, "output", "", "directory for a saved report") - flags.StringVar(&result.output, "o", "", "directory for a saved report (shorthand)") + flags.StringVar(&result.output, "output", ".", "report directory; default current directory") + flags.StringVar(&result.output, "o", ".", "report directory (shorthand)") + flags.BoolVar(&result.noSave, "no-save", false, "do not save report files") flags.BoolVar(&result.verbose, "verbose", false, "print full statistical tables to stdout") flags.BoolVar(&result.showOutput, "show-output", false, "show command output") // TUI. @@ -150,7 +152,8 @@ func printUsage(stderr io.Writer) { }}, {"Output", []string{ "-f, -format list saved format: html, svg, markdown, json, text", - "-o, -output string directory for a saved report", + "-o, -output string report directory; default current directory", + "-no-save do not save report files", "-verbose print full statistical tables to stdout", "-show-output forward command output to stderr", }}, diff --git a/internal/app/options_test.go b/internal/app/options_test.go index 252e076..8d4a159 100644 --- a/internal/app/options_test.go +++ b/internal/app/options_test.go @@ -2,6 +2,7 @@ package app import ( "io" + "strings" "testing" ) @@ -21,6 +22,16 @@ func TestParseDirectCommand(t *testing.T) { } } +func TestHelpReturnsSuccess(t *testing.T) { + var output strings.Builder + if err := Run([]string{"-h"}, io.Discard, &output); err != nil { + t.Fatal(err) + } + if !strings.Contains(output.String(), "Usage: snailrace") { + t.Fatal("help output is missing usage") + } +} + func TestRejectsLabelCountMismatch(t *testing.T) { _, err := parseOptions( []string{"-label", "x", "-c", "true", "-c", "false"}, @@ -31,6 +42,25 @@ func TestRejectsLabelCountMismatch(t *testing.T) { } } +func TestRejectsEmptyAndDuplicateLabels(t *testing.T) { + for _, arguments := range [][]string{ + {"-label", " ", "--", "true"}, + {"-label", "same", "-label", "same", "-c", "true", "-c", "false"}, + } { + if _, err := parseOptions(arguments, io.Discard); err == nil { + t.Fatalf("arguments %q should reject ambiguous labels", arguments) + } + } +} + +func TestRejectsEmptyShellCommand(t *testing.T) { + for _, command := range []string{"", " \t\n"} { + if _, err := parseOptions([]string{"-c", command}, io.Discard); err == nil { + t.Fatalf("command %q should be rejected", command) + } + } +} + func TestSingleLabelNamesPositionalCommand(t *testing.T) { options, err := parseOptions( []string{"-label", "build", "--", "sleep", "0.1"}, diff --git a/internal/app/output_bundle.go b/internal/app/output_bundle.go index f97f570..c7858b1 100644 --- a/internal/app/output_bundle.go +++ b/internal/app/output_bundle.go @@ -1,12 +1,14 @@ package app import ( + "errors" "fmt" "io" "os" "path/filepath" + "strings" + "time" - "github.com/shellcell/snailrace/internal/model" "github.com/shellcell/snailrace/internal/report" ) @@ -14,56 +16,162 @@ func saveReportFormats( stderr io.Writer, directory string, formats []string, - result model.Report, + renderer *report.Renderer, ) error { if err := os.MkdirAll(directory, 0o755); err != nil { return err } formats = uniqueFormats(formats) - needsCharts := containsFormat(formats, "svg") || - containsFormat(formats, "markdown") || containsFormat(formats, "md") + needsCharts := false + includeCommandCharts := false + for _, format := range formats { + info, _ := report.LookupFormat(format) + needsCharts = needsCharts || info.NeedsCharts + includeCommandCharts = includeCommandCharts || info.Name == report.FormatSVG + } + metadata := renderer.Metadata() + stem, release, err := reserveReportStem( + directory, reportStemFor(metadata.MeasuredAt, metadata.ToolNames), + ) + if err != nil { + return err + } + defer release() + staging, err := os.MkdirTemp(directory, "."+stem+".tmp-") + if err != nil { + return err + } + defer os.RemoveAll(staging) + var charts []report.ChartArtifact - bundleDirectory := filepath.Join(directory, reportStem(result)) + stagedBundle := filepath.Join(staging, stem) + finalBundle := filepath.Join(directory, stem) if needsCharts { - chartDirectory := filepath.Join(bundleDirectory, "charts") - var err error - charts, err = report.WriteChartFiles( - chartDirectory, result, containsFormat(formats, "svg"), + chartDirectory := filepath.Join(stagedBundle, "charts") + charts, err = renderer.WriteChartFiles( + chartDirectory, includeCommandCharts, ) if err != nil { return err } } + type stagedOutput struct{ staged, final string } + var outputs []stagedOutput + var announcements []string for _, format := range formats { switch format { case "svg": - if err := announce(stderr, filepath.Join(bundleDirectory, "charts")); err != nil { - return err - } - case "markdown", "md": - path := filepath.Join(bundleDirectory, "report.md") - if err := writeMarkdownFile(path, result, charts); err != nil { - return err - } - if err := announce(stderr, path); err != nil { + announcements = append(announcements, filepath.Join(finalBundle, "charts")) + case "markdown": + path := filepath.Join(stagedBundle, "report.md") + if err := writeMarkdownFile(path, renderer, charts); err != nil { return err } + announcements = append(announcements, filepath.Join(finalBundle, "report.md")) default: - path := reportPath(directory, format, result) - if err := writeReportFile(path, format, result); err != nil { + stagedPath := reportPathWithStem(staging, format, stem) + if err := writeReportFile(stagedPath, format, renderer); err != nil { return err } - if err := announce(stderr, path); err != nil { - return err + finalPath := reportPathWithStem(directory, format, stem) + outputs = append(outputs, stagedOutput{staged: stagedPath, final: finalPath}) + announcements = append(announcements, finalPath) + } + } + if needsCharts { + outputs = append(outputs, stagedOutput{staged: stagedBundle, final: finalBundle}) + } + var published []string + for _, output := range outputs { + if err := renameNoReplace(output.staged, output.final); err != nil { + for _, path := range published { + os.RemoveAll(path) } + return err + } + published = append(published, output.final) + } + for _, path := range announcements { + if err := announce(stderr, path); err != nil { + return err } } return nil } +const maxReportNameAttempts = 10000 + +func reserveReportStem( + directory string, + base string, +) (string, func(), error) { + for suffix := 1; suffix <= maxReportNameAttempts; suffix++ { + stem := base + if suffix > 1 { + stem = fmt.Sprintf("%s-%d", base, suffix) + } + release, reserved, err := tryReserveStem(directory, stem) + if err != nil { + return "", nil, err + } + if reserved { + return stem, release, nil + } + } + return "", nil, fmt.Errorf("no available report name for %q in %s", base, directory) +} + +func tryReserveStem(directory, stem string) (func(), bool, error) { + lockPath := filepath.Join(directory, "."+stem+".lock") + for attempt := range 2 { + lock, err := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if errors.Is(err, os.ErrExist) { + // Reclaim a lock leaked by a crashed run, then retry this stem once. + if attempt == 0 && staleLockFile(lockPath) && os.Remove(lockPath) == nil { + continue + } + return nil, false, nil + } + if err != nil { + return nil, false, err + } + if err := lock.Close(); err != nil { + os.Remove(lockPath) + return nil, false, err + } + exists, err := reportStemExists(directory, stem) + if err != nil || exists { + os.Remove(lockPath) + return nil, false, err + } + return func() { os.Remove(lockPath) }, true, nil + } + return nil, false, nil +} + +// Locks are held only while a report is being written, so anything old enough +// to predate the current run by an hour was leaked by a crashed process. +func staleLockFile(path string) bool { + info, err := os.Stat(path) + return err == nil && time.Since(info.ModTime()) > time.Hour +} + +func reportStemExists(directory, stem string) (bool, error) { + entries, err := os.ReadDir(directory) + if err != nil { + return false, err + } + for _, entry := range entries { + if entry.Name() == stem || strings.HasPrefix(entry.Name(), stem+".") { + return true, nil + } + } + return false, nil +} + func writeMarkdownFile( path string, - result model.Report, + renderer *report.Renderer, charts []report.ChartArtifact, ) error { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { @@ -73,31 +181,39 @@ func writeMarkdownFile( if err != nil { return err } - writeErr := report.WriteMarkdownWithCharts(file, result, charts, "charts") - closeErr := file.Close() - if writeErr != nil { - return writeErr + writeErr := renderer.WriteMarkdownWithCharts(file, charts, "charts") + syncErr := error(nil) + if writeErr == nil { + syncErr = file.Sync() } - return closeErr + closeErr := file.Close() + return errors.Join(writeErr, syncErr, closeErr) } -func writeReportFile(path, format string, result model.Report) error { +func writeReportFile(path, format string, renderer *report.Renderer) error { file, err := os.Create(path) if err != nil { return err } - writeErr := report.Write(file, format, result) - closeErr := file.Close() - if writeErr != nil { - return writeErr + writeErr := renderer.Write(file, format) + syncErr := error(nil) + if writeErr == nil { + syncErr = file.Sync() } - return closeErr + closeErr := file.Close() + return errors.Join(writeErr, syncErr, closeErr) } func uniqueFormats(formats []string) []string { seen := make(map[string]bool) result := make([]string, 0, len(formats)) for _, format := range formats { + info, ok := report.LookupFormat(format) + if !ok { + format = strings.ToLower(strings.TrimSpace(format)) + } else { + format = string(info.Name) + } if !seen[format] { seen[format] = true result = append(result, format) @@ -106,15 +222,6 @@ func uniqueFormats(formats []string) []string { return result } -func containsFormat(formats []string, target string) bool { - for _, format := range formats { - if format == target { - return true - } - } - return false -} - func announce(writer io.Writer, path string) error { _, err := fmt.Fprintf(writer, "Report saved to %s\n", path) return err diff --git a/internal/app/output_test.go b/internal/app/output_test.go index 0b12ed9..defb697 100644 --- a/internal/app/output_test.go +++ b/internal/app/output_test.go @@ -2,11 +2,17 @@ package app import ( "bytes" + "errors" + "io" "os" + "path/filepath" "strings" + "sync" "testing" + "time" "github.com/shellcell/snailrace/internal/model" + reportpkg "github.com/shellcell/snailrace/internal/report" ) func TestSavingAlsoWritesCompleteReportToStdout(t *testing.T) { @@ -39,6 +45,29 @@ func TestSavingAlsoWritesCompleteReportToStdout(t *testing.T) { } } +func TestStdoutFailureDoesNotPreventSaving(t *testing.T) { + directory := t.TempDir() + report := model.Report{ + Config: model.Config{Mode: "command", Baseline: 1}, + Benchmarks: []model.Benchmark{{Tool: model.ToolInfo{Name: "tool"}}}, + } + if err := writeResult( + failingWriter{}, io.Discard, directory, []string{"html"}, report, + ); err == nil { + t.Fatal("stdout failure should be returned") + } + entries, err := os.ReadDir(directory) + if err != nil || len(entries) != 1 { + t.Fatalf("saved entries = %v, error = %v", entries, err) + } +} + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { + return 0, errors.New("write failed") +} + func TestSavedFormatDoesNotReplaceTextStdout(t *testing.T) { directory := t.TempDir() var stdout, stderr bytes.Buffer @@ -80,7 +109,7 @@ func TestMarkdownAndSVGSavedAsChartBundle(t *testing.T) { {Tool: model.ToolInfo{Name: "second"}, Runs: runs, Summary: model.Summarize(runs)}, }, } - if err := saveReportFormats( + if err := saveReportFormatsForTest( &stderr, directory, []string{"markdown", "svg"}, result, ); err != nil { t.Fatal(err) @@ -103,3 +132,248 @@ func TestMarkdownAndSVGSavedAsChartBundle(t *testing.T) { t.Fatalf("chart files = %v, error = %v", charts, err) } } + +func TestRunIgnoresNonZeroCommandExit(t *testing.T) { + t.Chdir(t.TempDir()) + var stdout, stderr bytes.Buffer + if err := Run( + []string{"-n", "2", "-warmups", "0", "-c", "exit 7"}, + &stdout, &stderr, + ); err != nil { + t.Fatal(err) + } + if !strings.Contains(stdout.String(), "FAILED") || + !strings.Contains(stdout.String(), "excluded from rankings") { + t.Fatal("stdout does not report the failed command") + } +} + +func TestRunSavesDefaultHTMLInCurrentDirectory(t *testing.T) { + directory := t.TempDir() + t.Chdir(directory) + var stdout, stderr bytes.Buffer + if err := Run( + []string{"-n", "1", "-warmups", "0", "--", "true"}, + &stdout, &stderr, + ); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".html") { + t.Fatalf("saved entries = %v, want one HTML report", entries) + } + if !strings.Contains(stderr.String(), "Report saved to") { + t.Fatal("stderr does not announce the default report path") + } +} + +func TestRunNoSaveLeavesCurrentDirectoryEmpty(t *testing.T) { + directory := t.TempDir() + t.Chdir(directory) + if err := Run( + []string{"-no-save", "-n", "1", "-warmups", "0", "--", "true"}, + io.Discard, io.Discard, + ); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(directory) + if err != nil || len(entries) != 0 { + t.Fatalf("no-save entries = %v, error = %v", entries, err) + } +} + +func TestSavingSameReportUsesUniqueNames(t *testing.T) { + directory := t.TempDir() + result := model.Report{ + Config: model.Config{Mode: "command", Baseline: 1}, + Benchmarks: []model.Benchmark{{ + Tool: model.ToolInfo{Name: "same"}, + }}, + } + const saves = 6 + errors := make(chan error, saves) + var wait sync.WaitGroup + for index := 0; index < saves; index++ { + wait.Add(1) + go func() { + defer wait.Done() + errors <- saveReportFormatsForTest(io.Discard, directory, []string{"html"}, result) + }() + } + wait.Wait() + close(errors) + for err := range errors { + if err != nil { + t.Fatal(err) + } + } + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatal(err) + } + if len(entries) != saves { + t.Fatalf("saved entries = %d, want %d unique reports", len(entries), saves) + } +} + +func TestReserveReportStemFailsInUnreadableDirectory(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("directory permissions are not enforced for root") + } + directory := t.TempDir() + if err := os.Chmod(directory, 0o333); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(directory, 0o755) }) + if _, _, err := reserveReportStem(directory, "report"); err == nil { + t.Fatal("unreadable directory should fail instead of retrying forever") + } +} + +func TestReserveReportStemReclaimsStaleLock(t *testing.T) { + directory := t.TempDir() + lockPath := filepath.Join(directory, ".report.lock") + if err := os.WriteFile(lockPath, nil, 0o600); err != nil { + t.Fatal(err) + } + leaked := time.Now().Add(-2 * time.Hour) + if err := os.Chtimes(lockPath, leaked, leaked); err != nil { + t.Fatal(err) + } + stem, release, err := reserveReportStem(directory, "report") + if err != nil { + t.Fatal(err) + } + defer release() + if stem != "report" { + t.Fatalf("stem = %q, want the stale lock reclaimed for %q", stem, "report") + } +} + +func TestReserveReportStemSkipsHeldLock(t *testing.T) { + directory := t.TempDir() + if err := os.WriteFile(filepath.Join(directory, ".report.lock"), nil, 0o600); err != nil { + t.Fatal(err) + } + stem, release, err := reserveReportStem(directory, "report") + if err != nil { + t.Fatal(err) + } + defer release() + if stem != "report-2" { + t.Fatalf("stem = %q, want a fresh lock to skip to %q", stem, "report-2") + } +} + +func TestSavingDifferentFormatsDoesNotReuseExistingStem(t *testing.T) { + directory := t.TempDir() + result := model.Report{ + Config: model.Config{Mode: "command", Baseline: 1}, + Benchmarks: []model.Benchmark{{Tool: model.ToolInfo{Name: "same"}}}, + } + if err := saveReportFormatsForTest(io.Discard, directory, []string{"html"}, result); err != nil { + t.Fatal(err) + } + if err := saveReportFormatsForTest(io.Discard, directory, []string{"markdown"}, result); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatal(err) + } + if len(entries) != 2 || entries[0].Name() == entries[1].Name() { + t.Fatalf("saved entries reused a report stem: %v", entries) + } +} + +func TestFailedSavePublishesNoPartialReports(t *testing.T) { + directory := t.TempDir() + result := model.Report{ + Config: model.Config{Mode: "command", Baseline: 1}, + Benchmarks: []model.Benchmark{{Tool: model.ToolInfo{Name: "tool"}}}, + } + if err := saveReportFormatsForTest( + io.Discard, directory, []string{"html", "unknown"}, result, + ); err == nil { + t.Fatal("invalid staged format should fail") + } + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("failed save left partial entries: %v", entries) + } +} + +func TestPublishingNeverReplacesExistingDestination(t *testing.T) { + directory := t.TempDir() + staged := directory + "/staged" + final := directory + "/final" + if err := os.WriteFile(staged, []byte("new"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(final, []byte("existing"), 0o600); err != nil { + t.Fatal(err) + } + if err := renameNoReplace(staged, final); err == nil { + t.Fatal("publication replaced an existing destination") + } + data, err := os.ReadFile(final) + if err != nil { + t.Fatal(err) + } + if string(data) != "existing" { + t.Fatalf("destination content = %q, want existing", data) + } +} + +func TestSavedSVGBundleIncludesCommandFailures(t *testing.T) { + directory := t.TempDir() + failedRun := model.Run{ExitCode: 7, StopReason: "exited", WallSeconds: 0.001} + result := model.Report{ + Config: model.Config{Mode: "command", Baseline: 1}, + Benchmarks: []model.Benchmark{{ + Tool: model.ToolInfo{Name: "failed"}, + Runs: []model.Run{failedRun}, Summary: model.Summarize([]model.Run{failedRun}), + }}, + } + if err := saveReportFormatsForTest(io.Discard, directory, []string{"svg"}, result); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(directory) + if err != nil || len(entries) != 1 { + t.Fatalf("bundle entries = %v, error = %v", entries, err) + } + charts, err := os.ReadDir(directory + "/" + entries[0].Name() + "/charts") + if err != nil { + t.Fatal(err) + } + found := false + for _, chart := range charts { + if strings.Contains(chart.Name(), "command-failures") { + found = true + } + } + if !found { + t.Fatalf("SVG charts do not include command failures: %v", charts) + } +} + +func TestFormatAliasesAreCanonicalizedBeforeSaving(t *testing.T) { + formats := uniqueFormats([]string{"text", "txt", "markdown", "md"}) + if len(formats) != 2 || formats[0] != "text" || formats[1] != "markdown" { + t.Fatalf("canonical formats = %v", formats) + } +} + +func saveReportFormatsForTest( + stderr io.Writer, directory string, formats []string, result model.Report, +) error { + return saveReportFormats( + stderr, directory, formats, reportpkg.NewRenderer(result), + ) +} diff --git a/internal/app/progress.go b/internal/app/progress.go index 527ba8c..71fe75c 100644 --- a/internal/app/progress.go +++ b/internal/app/progress.go @@ -13,6 +13,7 @@ import ( "golang.org/x/term" "github.com/shellcell/snailrace/internal/runner" + "github.com/shellcell/snailrace/internal/style" ) func progressRenderer( @@ -23,7 +24,7 @@ func progressRenderer( return nil } var mutex sync.Mutex - renderedLines := 0 + var renderedLines []string cursorHidden := false return func(event runner.ProgressEvent) { mutex.Lock() @@ -32,20 +33,20 @@ func progressRenderer( if cursorHidden { fmt.Fprint(writer, "\x1b[?25h\r\n") } - renderedLines = 0 + renderedLines = nil return } if !cursorHidden { fmt.Fprint(writer, "\x1b[?25l") cursorHidden = true } - clearProgress(writer, renderedLines) width, _, _ := term.GetSize(int(file.Fd())) + clearProgress(writer, progressRenderedRows(renderedLines, width)) lines := append( []string{progressStatus(event, nameWidth)}, progressRaceLines(event, width)..., ) fmt.Fprint(writer, strings.Join(lines, "\n")) - renderedLines = len(lines) + renderedLines = lines } } @@ -121,6 +122,51 @@ func clearProgress(writer io.Writer, lines int) { } } +func progressRenderedRows(lines []string, terminalWidth int) int { + if terminalWidth <= 0 { + return len(lines) + } + rows := 0 + for _, line := range lines { + width := progressDisplayWidth(line) + lineRows := (width + terminalWidth - 1) / terminalWidth + if lineRows == 0 { + lineRows = 1 + } + rows += lineRows + } + return rows +} + +func progressDisplayWidth(value string) int { + width := 0 + for index := 0; index < len(value); { + if value[index] == '\x1b' { + next := progressANSIEscapeEnd(value[index:]) + if next > 0 { + index += next + continue + } + } + character, size := utf8.DecodeRuneInString(value[index:]) + index += size + width += style.RuneWidth(character) + } + return width +} + +func progressANSIEscapeEnd(value string) int { + if len(value) < 2 || value[0] != '\x1b' || value[1] != '[' { + return 0 + } + for index := 2; index < len(value); index++ { + if value[index] >= '@' && value[index] <= '~' { + return index + 1 + } + } + return 0 +} + func progressDuration(seconds float64) string { if seconds < 0.001 { return fmt.Sprintf("%.3g us", seconds*1e6) diff --git a/internal/app/progress_race.go b/internal/app/progress_race.go index 4d3c044..2741273 100644 --- a/internal/app/progress_race.go +++ b/internal/app/progress_race.go @@ -78,14 +78,14 @@ func progressScores(event runner.ProgressEvent) []float64 { benchmarks := make([]model.Benchmark, 0, len(event.Estimates)) positions := make([]int, 0, len(event.Estimates)) for index, tool := range event.Estimates { - if !tool.HasEstimate { + if !tool.HasEstimate || tool.Estimate == nil { continue } benchmarks = append(benchmarks, model.Benchmark{ Tool: model.ToolInfo{ Name: tool.ToolName, DiskFootprintBytes: tool.DiskFootprintBytes, }, - Runs: tool.Runs, Summary: tool.Estimate, + Runs: tool.Runs, Summary: *tool.Estimate, }) positions = append(positions, index) } @@ -105,7 +105,7 @@ func progressTrailCharacters(tools []runner.ProgressEstimate) []string { minimum := math.Inf(1) values := make([]float64, len(tools)) for index, tool := range tools { - if !tool.HasEstimate { + if !tool.HasEstimate || tool.Estimate == nil { continue } mean := tool.Estimate.MeanResidentBytes.Mean diff --git a/internal/app/progress_race_test.go b/internal/app/progress_race_test.go index 2b95f2b..5b3deb9 100644 --- a/internal/app/progress_race_test.go +++ b/internal/app/progress_race_test.go @@ -1,12 +1,15 @@ package app import ( + "bytes" "math" "regexp" "strings" "testing" "time" + "github.com/creack/pty" + "github.com/shellcell/snailrace/internal/model" "github.com/shellcell/snailrace/internal/runner" ) @@ -15,7 +18,7 @@ var progressColorSequence = regexp.MustCompile(`\x1b\[[0-9;]*m`) func TestProgressRaceAdvancesBetterExpectedTool(t *testing.T) { event := runner.ProgressEvent{ - Completed: 2, Total: 4, Elapsed: time.Second, ETA: time.Second, + Completed: 2, Total: 4, ETA: time.Second, Estimates: []runner.ProgressEstimate{ progressEstimate("better", 1), progressEstimate("worse", 2), @@ -135,12 +138,71 @@ func TestProgressStatusAlignsNumbersAndName(t *testing.T) { } } +func TestProgressRenderedRowsCountsWrappedLines(t *testing.T) { + lines := []string{"12345678901", "short"} + if got := progressRenderedRows(lines, 10); got != 3 { + t.Fatalf("rendered rows = %d, want 3", got) + } +} + +func TestProgressRenderedRowsIgnoresANSIAndCountsKnownEmojiWidth(t *testing.T) { + line := "\x1b[38;2;136;192;208m🐌\x1b[0m123456789" + if got := progressRenderedRows([]string{line}, 10); got != 2 { + t.Fatalf("rendered rows = %d, want 2", got) + } +} + +func TestProgressRendererRecalculatesRowsAfterResize(t *testing.T) { + primary, terminal, err := pty.Open() + if err != nil { + t.Fatal(err) + } + defer primary.Close() + defer terminal.Close() + if err := pty.Setsize(terminal, &pty.Winsize{Cols: 200, Rows: 24}); err != nil { + t.Fatal(err) + } + output := progressTerminalBuffer{fd: terminal.Fd()} + render := progressRenderer(&output, true, 0) + if render == nil { + t.Fatal("progress renderer disabled for PTY") + } + event := runner.ProgressEvent{ + ToolName: "first", Tool: 1, ToolCount: 1, + Iteration: 1, Iterations: 10, Total: 10, + } + render(event) + redrawStart := output.Len() + if err := pty.Setsize(terminal, &pty.Winsize{Cols: 40, Rows: 24}); err != nil { + t.Fatal(err) + } + render(event) + + previous := []string{progressStatus(event, 0)} + wantRows := progressRenderedRows(previous, 40) + redraw := output.String()[redrawStart:] + if got := strings.Count(redraw, "\x1b[1A") + 1; got != wantRows { + t.Fatalf("cleared rows after resize = %d, want %d", got, wantRows) + } +} + +type progressTerminalBuffer struct { + bytes.Buffer + fd uintptr +} + +func (buffer *progressTerminalBuffer) Fd() uintptr { return buffer.fd } + func progressEstimate(name string, scale float64) runner.ProgressEstimate { stats := model.Stats{Mean: scale} return runner.ProgressEstimate{ ToolName: name, Completed: 1, Total: 2, HasEstimate: true, + Runs: []model.Run{{ + WallSeconds: scale, CPUUserSeconds: scale, + MeanResidentBytes: scale, PeakResidentBytes: scale, + }}, DiskFootprintBytes: int64(scale * 100), - Estimate: model.Summary{ + Estimate: &model.Summary{ WallSeconds: stats, CPUTotalSeconds: stats, MeanResidentBytes: stats, PeakResidentBytes: stats, }, diff --git a/internal/app/publish_darwin.go b/internal/app/publish_darwin.go new file mode 100644 index 0000000..3f897dd --- /dev/null +++ b/internal/app/publish_darwin.go @@ -0,0 +1,9 @@ +//go:build darwin + +package app + +import "golang.org/x/sys/unix" + +func renameNoReplace(oldPath, newPath string) error { + return unix.RenamexNp(oldPath, newPath, unix.RENAME_EXCL) +} diff --git a/internal/app/publish_linux.go b/internal/app/publish_linux.go new file mode 100644 index 0000000..481c18d --- /dev/null +++ b/internal/app/publish_linux.go @@ -0,0 +1,11 @@ +//go:build linux + +package app + +import "golang.org/x/sys/unix" + +func renameNoReplace(oldPath, newPath string) error { + return unix.Renameat2( + unix.AT_FDCWD, oldPath, unix.AT_FDCWD, newPath, unix.RENAME_NOREPLACE, + ) +} diff --git a/internal/app/specs.go b/internal/app/specs.go index d125890..a171cc8 100644 --- a/internal/app/specs.go +++ b/internal/app/specs.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/shellcell/snailrace/internal/runner" + "github.com/shellcell/snailrace/internal/style" ) func makeSpecs(commands, arguments []string, labels []string) []runner.Spec { @@ -14,6 +15,7 @@ func makeSpecs(commands, arguments []string, labels []string) []runner.Spec { if len(labels) > 0 { name = labels[0] } + name = style.SafeText(name) return []runner.Spec{{Name: name, Args: arguments}} } defaultLabels := make([]string, len(commands)) @@ -32,6 +34,7 @@ func makeSpecs(commands, arguments []string, labels []string) []runner.Spec { seen[label]++ label += " #" + fmt.Sprint(seen[label]) } + label = style.SafeText(label) result = append(result, runner.Spec{Name: label, Shell: command}) } return result diff --git a/internal/app/validation.go b/internal/app/validation.go index 271758a..968608f 100644 --- a/internal/app/validation.go +++ b/internal/app/validation.go @@ -3,6 +3,7 @@ package app import ( "errors" "fmt" + "strings" "time" "github.com/shellcell/snailrace/internal/report" @@ -27,6 +28,9 @@ func validateOptions(result options, commands []string, positional int) error { if len(result.formats) == 0 { return errors.New("at least one output format is required") } + if !result.noSave && strings.TrimSpace(result.output) == "" { + return errors.New("output directory cannot be empty; use -no-save") + } for _, format := range result.formats { if !report.ValidFormat(format) { return fmt.Errorf("unknown report format %q", format) @@ -41,6 +45,11 @@ func validateOptions(result options, commands []string, positional int) error { if len(commands) == 0 && positional == 0 { return errors.New("no command specified") } + for _, command := range commands { + if strings.TrimSpace(command) == "" { + return errors.New("command cannot be empty") + } + } commandCount := len(commands) if positional > 0 { commandCount = 1 @@ -74,6 +83,16 @@ func validateLabels(labels, commands []string) error { if len(labels) == 0 { return nil } + seen := make(map[string]bool, len(labels)) + for _, label := range labels { + if strings.TrimSpace(label) == "" { + return errors.New("labels cannot be empty") + } + if seen[label] { + return fmt.Errorf("duplicate label %q", label) + } + seen[label] = true + } if len(commands) > 0 { if len(labels) != len(commands) { return errors.New("label count must match command count") diff --git a/internal/model/model.go b/internal/model/model.go index 167a18e..c664ed4 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -22,6 +22,17 @@ type Config struct { OutputMode string `json:"output_mode"` } +const ( + ModeCommand = "command" + ModeTUI = "tui" +) + +func (config Config) IsTUI() bool { return config.Mode == ModeTUI } + +func (config Config) FixedDurationTUI() bool { + return config.IsTUI() && config.DurationSeconds > 0 +} + type HostInfo struct { OS string `json:"os"` Architecture string `json:"architecture"` @@ -38,6 +49,9 @@ type HostInfo struct { type ToolInfo struct { Name string `json:"name"` Command []string `json:"command"` + ShellCommand bool `json:"shell_command"` + ShellTarget bool `json:"-"` + ProvenanceVerified bool `json:"provenance_verified"` Executable string `json:"executable,omitempty"` SizeBytes int64 `json:"size_bytes,omitempty"` SHA256 string `json:"sha256,omitempty"` @@ -61,6 +75,7 @@ type Run struct { AverageCPUPercent float64 `json:"average_cpu_percent"` PeakResidentBytes float64 `json:"peak_resident_bytes"` PeakPhysicalFootprintBytes float64 `json:"peak_physical_footprint_bytes,omitempty"` + PhysicalFootprintValid bool `json:"physical_footprint_valid"` OSMaxRSSBytes float64 `json:"os_max_rss_bytes"` MeanResidentBytes float64 `json:"mean_resident_bytes"` PeakVirtualBytes float64 `json:"peak_virtual_bytes"` @@ -72,6 +87,10 @@ type Run struct { SampleCoverageSeconds float64 `json:"sample_coverage_seconds"` } +func (run Run) Failed() bool { + return run.StopReason != "duration" && run.ExitCode != 0 +} + type Stats struct { N int `json:"n"` Min float64 `json:"min"` @@ -116,6 +135,20 @@ type Benchmark struct { Summary Summary `json:"summary"` } +func (benchmark Benchmark) FailedRunCount() int { + count := 0 + for _, run := range benchmark.Runs { + if run.Failed() { + count++ + } + } + return count +} + +func (benchmark Benchmark) EligibleForRanking() bool { + return len(benchmark.Runs) > 0 && benchmark.FailedRunCount() == 0 +} + type Report struct { MeasuredAt time.Time `json:"measured_at"` Config Config `json:"config"` diff --git a/internal/model/model_json.go b/internal/model/model_json.go new file mode 100644 index 0000000..965c6fa --- /dev/null +++ b/internal/model/model_json.go @@ -0,0 +1,50 @@ +package model + +import ( + "encoding/json" + "path/filepath" +) + +func (tool *ToolInfo) UnmarshalJSON(data []byte) error { + type plain ToolInfo + decoded := struct { + plain + ShellCommand *bool `json:"shell_command"` + ProvenanceVerified *bool `json:"provenance_verified"` + }{} + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + *tool = ToolInfo(decoded.plain) + if decoded.ShellCommand != nil { + tool.ShellCommand = *decoded.ShellCommand + } else { + tool.ShellCommand = len(tool.Command) == 1 && + tool.Command[0] != tool.Executable && + tool.Command[0] != filepath.Base(tool.Executable) + } + if decoded.ProvenanceVerified != nil { + tool.ProvenanceVerified = *decoded.ProvenanceVerified + } else { + tool.ProvenanceVerified = tool.SHA256 != "" + } + return nil +} + +func (run *Run) UnmarshalJSON(data []byte) error { + type plain Run + decoded := struct { + plain + PhysicalFootprintValid *bool `json:"physical_footprint_valid"` + }{} + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + *run = Run(decoded.plain) + if decoded.PhysicalFootprintValid != nil { + run.PhysicalFootprintValid = *decoded.PhysicalFootprintValid + } else { + run.PhysicalFootprintValid = run.PeakPhysicalFootprintBytes > 0 + } + return nil +} diff --git a/internal/model/model_json_test.go b/internal/model/model_json_test.go new file mode 100644 index 0000000..a803914 --- /dev/null +++ b/internal/model/model_json_test.go @@ -0,0 +1,50 @@ +package model + +import ( + "encoding/json" + "testing" +) + +func TestLegacyRunInfersPhysicalFootprintValidity(t *testing.T) { + var run Run + if err := json.Unmarshal( + []byte(`{"peak_physical_footprint_bytes":123}`), &run, + ); err != nil { + t.Fatal(err) + } + if !run.PhysicalFootprintValid { + t.Fatal("legacy positive physical footprint should remain valid") + } +} + +func TestLegacyToolInfersShellCommand(t *testing.T) { + var tool ToolInfo + if err := json.Unmarshal([]byte(`{"command":["grep foo file"]}`), &tool); err != nil { + t.Fatal(err) + } + if !tool.ShellCommand { + t.Fatal("legacy one-element command should retain shell rendering") + } + data, err := json.Marshal(ToolInfo{Command: []string{"/tmp/my tool"}}) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(data, &tool); err != nil { + t.Fatal(err) + } + if tool.ShellCommand { + t.Fatal("current direct command should retain argv rendering") + } +} + +func TestLegacyToolInfersDirectSingleArgumentAndVerifiedHash(t *testing.T) { + var tool ToolInfo + if err := json.Unmarshal([]byte( + `{"command":["/tmp/my tool"],"executable":"/tmp/my tool","sha256":"abc"}`, + ), &tool); err != nil { + t.Fatal(err) + } + if tool.ShellCommand || !tool.ProvenanceVerified { + t.Fatalf("legacy direct tool = %+v", tool) + } +} diff --git a/internal/model/sampling.go b/internal/model/sampling.go new file mode 100644 index 0000000..8e199a3 --- /dev/null +++ b/internal/model/sampling.go @@ -0,0 +1,18 @@ +package model + +func SamplingReliable(benchmark Benchmark, intervalSeconds float64) bool { + if len(benchmark.Runs) == 0 { + return false + } + if intervalSeconds <= 0 { + return true + } + minimumDuration := intervalSeconds * 2 + for _, run := range benchmark.Runs { + if run.WallSeconds < minimumDuration || run.SampleCount < 2 || + run.SampleCoverageSeconds < intervalSeconds { + return false + } + } + return true +} diff --git a/internal/model/sampling_test.go b/internal/model/sampling_test.go new file mode 100644 index 0000000..89f146a --- /dev/null +++ b/internal/model/sampling_test.go @@ -0,0 +1,26 @@ +package model + +import "testing" + +func TestSamplingReliableBoundaries(t *testing.T) { + interval := 0.01 + tests := []struct { + name string + runs []Run + want bool + }{ + {"no runs", nil, false}, + {"reliable", []Run{{WallSeconds: 0.02, SampleCount: 2, SampleCoverageSeconds: 0.01}}, true}, + {"short", []Run{{WallSeconds: 0.019, SampleCount: 2, SampleCoverageSeconds: 0.01}}, false}, + {"few samples", []Run{{WallSeconds: 0.02, SampleCount: 1, SampleCoverageSeconds: 0.01}}, false}, + {"low coverage", []Run{{WallSeconds: 0.02, SampleCount: 2, SampleCoverageSeconds: 0.009}}, false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + benchmark := Benchmark{Runs: test.runs} + if got := SamplingReliable(benchmark, interval); got != test.want { + t.Fatalf("reliable = %v, want %v", got, test.want) + } + }) + } +} diff --git a/internal/model/stats.go b/internal/model/stats.go index 69f8e68..6d34e9a 100644 --- a/internal/model/stats.go +++ b/internal/model/stats.go @@ -6,96 +6,104 @@ import ( ) func Summarize(runs []Run) Summary { - values := func(pick func(Run) float64) []float64 { - result := make([]float64, len(runs)) - for i, run := range runs { - result[i] = pick(run) - } - return result - } - physicalValues := values(func(r Run) float64 { return r.PeakPhysicalFootprintBytes }) - var physicalStats *Stats - for _, value := range physicalValues { - if value > 0 { - value := stats(physicalValues) - physicalStats = &value - break - } - } - return Summary{ - WallSeconds: stats(values(func(r Run) float64 { return r.WallSeconds })), - CPUTotalSeconds: stats(values(func(r Run) float64 { - return r.CPUUserSeconds + r.CPUSystemSeconds - })), - CPUUserSeconds: stats(values(func(r Run) float64 { - return r.CPUUserSeconds - })), - CPUSystemSeconds: stats(values(func(r Run) float64 { - return r.CPUSystemSeconds - })), - AverageCPUPercent: stats(values(func(r Run) float64 { - return r.AverageCPUPercent - })), - PeakResidentBytes: stats(values(func(r Run) float64 { - return r.PeakResidentBytes - })), - PeakPhysicalFootprintBytes: physicalStats, - OSMaxRSSBytes: stats(values(func(r Run) float64 { - return r.OSMaxRSSBytes - })), - MeanResidentBytes: stats(values(func(r Run) float64 { - return r.MeanResidentBytes - })), - PeakVirtualBytes: stats(values(func(r Run) float64 { - return r.PeakVirtualBytes - })), - PeakProcesses: stats(values(func(r Run) float64 { return r.PeakProcesses })), - PeakThreads: stats(values(func(r Run) float64 { return r.PeakThreads })), - PeakFileDescriptors: stats(values(func(r Run) float64 { - return r.PeakFileDescriptors - })), - ValidSampleCount: stats(values(func(r Run) float64 { - return float64(r.SampleCount) - })), - SampleCoverageSeconds: stats(values(func(r Run) float64 { - return r.SampleCoverageSeconds - })), + accumulator := NewSummaryAccumulator(len(runs)) + for _, run := range runs { + accumulator.Add(run) } + return accumulator.Snapshot() } func CalculateStats(values []float64) Stats { return stats(values) } +type RunningStats struct { + values []float64 + moments statsMoments +} + +func NewRunningStats(capacity int) RunningStats { + return RunningStats{values: make([]float64, 0, capacity)} +} + +func (running *RunningStats) Add(value float64) { + if math.IsNaN(value) || math.IsInf(value, 0) { + return + } + position := sort.SearchFloat64s(running.values, value) + running.values = append(running.values, 0) + copy(running.values[position+1:], running.values[position:]) + running.values[position] = value + running.moments.add(value) +} + +func (running RunningStats) Snapshot() Stats { + return statsFromSorted(running.values, running.moments) +} + func stats(values []float64) Stats { - if len(values) == 0 { + finite := make([]float64, 0, len(values)) + var moments statsMoments + for _, value := range values { + if !math.IsNaN(value) && !math.IsInf(value, 0) { + finite = append(finite, value) + moments.add(value) + } + } + if len(finite) == 0 { return Stats{} } - sorted := append([]float64(nil), values...) - sort.Float64s(sorted) - total := 0.0 - for _, value := range sorted { - total += value + sort.Float64s(finite) + return statsFromSorted(finite, moments) +} + +type statsMoments struct { + count int + mean float64 + m2 float64 + varianceOverflow bool +} + +func (moments *statsMoments) add(value float64) { + moments.count++ + count := float64(moments.count) + previousMean := moments.mean + moments.mean = previousMean*(1-1/count) + value/count + if math.IsInf(moments.mean, 0) { + moments.mean = math.Copysign(math.MaxFloat64, moments.mean) } - mean := total / float64(len(sorted)) - variance := 0.0 - for _, value := range sorted { - delta := value - mean - variance += delta * delta + delta := value - previousMean + term := delta * (value - moments.mean) + if math.IsInf(term, 0) || math.IsNaN(term) || math.MaxFloat64-moments.m2 < term { + moments.varianceOverflow = true + moments.m2 = math.MaxFloat64 + } else if term > 0 { + moments.m2 += term } - if len(sorted) > 1 { - variance /= float64(len(sorted) - 1) +} + +func statsFromSorted(sorted []float64, moments statsMoments) Stats { + if len(sorted) == 0 { + return Stats{} + } + standardDeviation := 0.0 + if len(sorted) > 1 && moments.varianceOverflow { + standardDeviation = math.MaxFloat64 + } else if len(sorted) > 1 { + standardDeviation = math.Sqrt(moments.m2 / float64(len(sorted)-1)) } - standardDeviation := math.Sqrt(variance) margin := 0.0 validInterval := len(sorted) > 1 if len(sorted) > 1 { - margin = tCritical95(len(sorted)-1) * standardDeviation / - math.Sqrt(float64(len(sorted))) + margin = standardDeviation / math.Sqrt(float64(len(sorted))) * + tCritical95(len(sorted)-1) + if math.IsInf(margin, 0) { + margin = math.MaxFloat64 + } } return Stats{ - N: len(sorted), Min: sorted[0], Max: sorted[len(sorted)-1], Mean: mean, + N: len(sorted), Min: sorted[0], Max: sorted[len(sorted)-1], Mean: moments.mean, StdDev: standardDeviation, Median: percentile(sorted, 0.5), - P95: percentile(sorted, 0.95), CI95Low: mean - margin, - CI95High: mean + margin, CI95Valid: validInterval, + P95: percentile(sorted, 0.95), CI95Low: finiteDifference(moments.mean, margin), + CI95High: finiteSum(moments.mean, margin), CI95Valid: validInterval, } } @@ -106,7 +114,23 @@ func percentile(sorted []float64, percentile float64) float64 { position := percentile * float64(len(sorted)-1) lower, upper := int(math.Floor(position)), int(math.Ceil(position)) weight := position - float64(lower) - return sorted[lower]*(1-weight) + sorted[upper]*weight + return finiteSum(sorted[lower]*(1-weight), sorted[upper]*weight) +} + +func finiteSum(left, right float64) float64 { + result := left + right + if math.IsInf(result, 0) { + return math.Copysign(math.MaxFloat64, result) + } + return result +} + +func finiteDifference(left, right float64) float64 { + result := left - right + if math.IsInf(result, 0) { + return math.Copysign(math.MaxFloat64, result) + } + return result } // Two-sided 95% Student's t critical values for 1..30 degrees of freedom. diff --git a/internal/model/stats_bench_test.go b/internal/model/stats_bench_test.go new file mode 100644 index 0000000..61a68b1 --- /dev/null +++ b/internal/model/stats_bench_test.go @@ -0,0 +1,22 @@ +package model + +import "testing" + +func BenchmarkSummarize1000Runs(b *testing.B) { + runs := make([]Run, 1000) + for index := range runs { + value := float64((index * 7919) % len(runs)) + runs[index] = Run{ + WallSeconds: value, CPUUserSeconds: value, CPUSystemSeconds: value, + AverageCPUPercent: value, PeakResidentBytes: value, + OSMaxRSSBytes: value, MeanResidentBytes: value, PeakVirtualBytes: value, + PeakProcesses: value, PeakThreads: value, PeakFileDescriptors: value, + SampleCount: index, SampleCoverageSeconds: value, + } + } + b.ReportAllocs() + b.ResetTimer() + for range b.N { + _ = Summarize(runs) + } +} diff --git a/internal/model/stats_json.go b/internal/model/stats_json.go index bd05ef1..3722e35 100644 --- a/internal/model/stats_json.go +++ b/internal/model/stats_json.go @@ -1,6 +1,9 @@ package model -import "encoding/json" +import ( + "encoding/json" + "math" +) func (stats Stats) MarshalJSON() ([]byte, error) { type plain Stats @@ -14,3 +17,32 @@ func (stats Stats) MarshalJSON() ([]byte, error) { CI95High *float64 `json:"ci95_high,omitempty"` }{plain(stats), low, high}) } + +func (stats *Stats) UnmarshalJSON(data []byte) error { + type plain Stats + decoded := struct { + plain + CI95Low *float64 `json:"ci95_low"` + CI95High *float64 `json:"ci95_high"` + }{} + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + *stats = Stats(decoded.plain) + if stats.N < 2 { + stats.CI95Valid = false + return nil + } + if !stats.CI95Valid { + return nil + } + if decoded.CI95Low == nil || decoded.CI95High == nil { + margin := stats.StdDev / math.Sqrt(float64(stats.N)) * tCritical95(stats.N-1) + stats.CI95Low = finiteDifference(stats.Mean, margin) + stats.CI95High = finiteSum(stats.Mean, margin) + return nil + } + stats.CI95Low = *decoded.CI95Low + stats.CI95High = *decoded.CI95High + return nil +} diff --git a/internal/model/stats_test.go b/internal/model/stats_test.go index ba32945..f6958fe 100644 --- a/internal/model/stats_test.go +++ b/internal/model/stats_test.go @@ -3,6 +3,7 @@ package model import ( "encoding/json" "math" + "reflect" "strings" "testing" ) @@ -53,6 +54,146 @@ func TestUnavailablePhysicalFootprintIsOmittedFromJSON(t *testing.T) { } } +func TestPhysicalFootprintUsesExplicitPerRunValidity(t *testing.T) { + summary := Summarize([]Run{ + {PeakPhysicalFootprintBytes: 0, PhysicalFootprintValid: true}, + {PeakPhysicalFootprintBytes: 10, PhysicalFootprintValid: true}, + {PeakPhysicalFootprintBytes: 1000, PhysicalFootprintValid: false}, + }) + if summary.PeakPhysicalFootprintBytes == nil { + t.Fatal("valid zero-valued physical footprint should be retained") + } + if got := summary.PeakPhysicalFootprintBytes; got.N != 2 || got.Mean != 5 { + t.Fatalf("physical footprint stats = %+v, want N=2 mean=5", *got) + } +} + +func TestStatsRemainFiniteForExtremeInputs(t *testing.T) { + result := CalculateStats([]float64{ + math.MaxFloat64, -math.MaxFloat64, math.NaN(), math.Inf(1), + }) + if result.N != 2 { + t.Fatalf("N = %d, want two finite observations", result.N) + } + for name, value := range map[string]float64{ + "mean": result.Mean, "stddev": result.StdDev, "median": result.Median, + "p95": result.P95, "ci low": result.CI95Low, "ci high": result.CI95High, + } { + if math.IsNaN(value) || math.IsInf(value, 0) { + t.Fatalf("%s is not finite: %v", name, value) + } + } + if _, err := json.Marshal(result); err != nil { + t.Fatal(err) + } +} + +func TestStatsJSONRoundTripPreservesConfidenceBounds(t *testing.T) { + original := CalculateStats([]float64{1, 2, 4, 8}) + data, err := json.Marshal(original) + if err != nil { + t.Fatal(err) + } + var decoded Stats + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatal(err) + } + if decoded != original { + t.Fatalf("round trip = %+v, want %+v", decoded, original) + } +} + +func TestStatsUnmarshalAcceptsLegacyConfidenceInterval(t *testing.T) { + var decoded Stats + if err := json.Unmarshal( + []byte(`{"n":2,"mean":3,"stddev":1,"ci95_valid":true}`), &decoded, + ); err != nil { + t.Fatal(err) + } + if !decoded.CI95Valid || decoded.CI95Low == 0 || decoded.CI95High == 0 { + t.Fatalf("legacy confidence interval = %+v", decoded) + } +} + +func TestStatsUnmarshalRejectsSingleObservationInterval(t *testing.T) { + var decoded Stats + if err := json.Unmarshal( + []byte(`{"n":1,"mean":3,"ci95_valid":true,"ci95_low":2,"ci95_high":4}`), + &decoded, + ); err != nil { + t.Fatal(err) + } + if decoded.CI95Valid { + t.Fatal("one observation cannot have a confidence interval") + } +} + +func TestSummaryAccumulatorMatchesBatchSummaryAtEveryPrefix(t *testing.T) { + runs := []Run{ + {WallSeconds: 1, CPUUserSeconds: 2, CPUSystemSeconds: 3, + AverageCPUPercent: 4, PeakResidentBytes: 5, OSMaxRSSBytes: 6, + MeanResidentBytes: 7, PeakVirtualBytes: 8, PeakProcesses: 9, + PeakThreads: 10, PeakFileDescriptors: 11, SampleCount: 12, + SampleCoverageSeconds: 13}, + {WallSeconds: 14, CPUUserSeconds: 15, CPUSystemSeconds: 16, + AverageCPUPercent: 17, PeakResidentBytes: 18, + PeakPhysicalFootprintBytes: 19, PhysicalFootprintValid: true, + OSMaxRSSBytes: 20, MeanResidentBytes: 21, PeakVirtualBytes: 22, + PeakProcesses: 23, PeakThreads: 24, PeakFileDescriptors: 25, + SampleCount: 26, SampleCoverageSeconds: 27}, + } + accumulator := NewSummaryAccumulator(len(runs)) + for index, run := range runs { + accumulator.Add(run) + got, want := accumulator.Snapshot(), independentSummary(runs[:index+1]) + if !reflect.DeepEqual(got, want) { + t.Fatalf("prefix %d summary mismatch:\ngot %+v\nwant %+v", index+1, got, want) + } + } +} + +func independentSummary(runs []Run) Summary { + values := func(pick func(Run) float64) []float64 { + result := make([]float64, len(runs)) + for index, run := range runs { + result[index] = pick(run) + } + return result + } + physicalValues := make([]float64, 0, len(runs)) + for _, run := range runs { + if run.PhysicalFootprintValid { + physicalValues = append(physicalValues, run.PeakPhysicalFootprintBytes) + } + } + var physical *Stats + if len(physicalValues) > 0 { + value := CalculateStats(physicalValues) + physical = &value + } + return Summary{ + WallSeconds: CalculateStats(values(func(run Run) float64 { return run.WallSeconds })), + CPUTotalSeconds: CalculateStats(values(func(run Run) float64 { + return run.CPUUserSeconds + run.CPUSystemSeconds + })), + CPUUserSeconds: CalculateStats(values(func(run Run) float64 { return run.CPUUserSeconds })), + CPUSystemSeconds: CalculateStats(values(func(run Run) float64 { return run.CPUSystemSeconds })), + AverageCPUPercent: CalculateStats(values(func(run Run) float64 { return run.AverageCPUPercent })), + PeakResidentBytes: CalculateStats(values(func(run Run) float64 { return run.PeakResidentBytes })), + PeakPhysicalFootprintBytes: physical, + OSMaxRSSBytes: CalculateStats(values(func(run Run) float64 { return run.OSMaxRSSBytes })), + MeanResidentBytes: CalculateStats(values(func(run Run) float64 { return run.MeanResidentBytes })), + PeakVirtualBytes: CalculateStats(values(func(run Run) float64 { return run.PeakVirtualBytes })), + PeakProcesses: CalculateStats(values(func(run Run) float64 { return run.PeakProcesses })), + PeakThreads: CalculateStats(values(func(run Run) float64 { return run.PeakThreads })), + PeakFileDescriptors: CalculateStats(values(func(run Run) float64 { return run.PeakFileDescriptors })), + ValidSampleCount: CalculateStats(values(func(run Run) float64 { return float64(run.SampleCount) })), + SampleCoverageSeconds: CalculateStats(values(func(run Run) float64 { + return run.SampleCoverageSeconds + })), + } +} + func assertClose(t *testing.T, got, want float64) { t.Helper() if math.Abs(got-want) > 1e-9 { diff --git a/internal/model/summary_accumulator.go b/internal/model/summary_accumulator.go new file mode 100644 index 0000000..79335b2 --- /dev/null +++ b/internal/model/summary_accumulator.go @@ -0,0 +1,66 @@ +package model + +type SummaryAccumulator struct { + wall, cpuTotal, cpuUser, cpuSystem RunningStats + averageCPU RunningStats + peakResident, physical, osMaxRSS RunningStats + meanResident, peakVirtual RunningStats + peakProcesses, peakThreads RunningStats + peakFileDescriptors RunningStats + validSampleCount, sampleCoverage RunningStats +} + +func NewSummaryAccumulator(capacity int) SummaryAccumulator { + metric := func() RunningStats { return NewRunningStats(capacity) } + return SummaryAccumulator{ + wall: metric(), cpuTotal: metric(), cpuUser: metric(), cpuSystem: metric(), + averageCPU: metric(), peakResident: metric(), physical: metric(), + osMaxRSS: metric(), meanResident: metric(), peakVirtual: metric(), + peakProcesses: metric(), peakThreads: metric(), peakFileDescriptors: metric(), + validSampleCount: metric(), sampleCoverage: metric(), + } +} + +func (summary *SummaryAccumulator) Add(run Run) { + summary.wall.Add(run.WallSeconds) + summary.cpuTotal.Add(run.CPUUserSeconds + run.CPUSystemSeconds) + summary.cpuUser.Add(run.CPUUserSeconds) + summary.cpuSystem.Add(run.CPUSystemSeconds) + summary.averageCPU.Add(run.AverageCPUPercent) + summary.peakResident.Add(run.PeakResidentBytes) + if run.PhysicalFootprintValid { + summary.physical.Add(run.PeakPhysicalFootprintBytes) + } + summary.osMaxRSS.Add(run.OSMaxRSSBytes) + summary.meanResident.Add(run.MeanResidentBytes) + summary.peakVirtual.Add(run.PeakVirtualBytes) + summary.peakProcesses.Add(run.PeakProcesses) + summary.peakThreads.Add(run.PeakThreads) + summary.peakFileDescriptors.Add(run.PeakFileDescriptors) + summary.validSampleCount.Add(float64(run.SampleCount)) + summary.sampleCoverage.Add(run.SampleCoverageSeconds) +} + +func (summary SummaryAccumulator) Snapshot() Summary { + result := Summary{ + WallSeconds: summary.wall.Snapshot(), + CPUTotalSeconds: summary.cpuTotal.Snapshot(), + CPUUserSeconds: summary.cpuUser.Snapshot(), + CPUSystemSeconds: summary.cpuSystem.Snapshot(), + AverageCPUPercent: summary.averageCPU.Snapshot(), + PeakResidentBytes: summary.peakResident.Snapshot(), + OSMaxRSSBytes: summary.osMaxRSS.Snapshot(), + MeanResidentBytes: summary.meanResident.Snapshot(), + PeakVirtualBytes: summary.peakVirtual.Snapshot(), + PeakProcesses: summary.peakProcesses.Snapshot(), + PeakThreads: summary.peakThreads.Snapshot(), + PeakFileDescriptors: summary.peakFileDescriptors.Snapshot(), + ValidSampleCount: summary.validSampleCount.Snapshot(), + SampleCoverageSeconds: summary.sampleCoverage.Snapshot(), + } + physical := summary.physical.Snapshot() + if physical.N > 0 { + result.PeakPhysicalFootprintBytes = &physical + } + return result +} diff --git a/internal/platform/dependencies_darwin.go b/internal/platform/dependencies_darwin.go index 534ea86..e79ff99 100644 --- a/internal/platform/dependencies_darwin.go +++ b/internal/platform/dependencies_darwin.go @@ -3,31 +3,40 @@ package platform import ( "bufio" "bytes" + "context" "os" "os/exec" "path/filepath" "strings" ) -func LinkedFiles(executable string) []LinkedDependency { +func LinkedFiles(ctx context.Context, executable string) ([]LinkedDependency, error) { type queuedLibrary struct { path string rpaths []string } - executableRPaths := loadRPaths(executable, executable, executable) + executableRPaths := loadRPaths(ctx, executable, executable, executable) queue := []queuedLibrary{{path: executable, rpaths: executableRPaths}} seen := map[string]bool{executable: true} var result []LinkedDependency for len(queue) > 0 { + if err := ctx.Err(); err != nil { + return nil, err + } current := queue[0] queue = queue[1:] - output, err := exec.Command("/usr/bin/otool", "-L", current.path).Output() + output, err := exec.CommandContext(ctx, "/usr/bin/otool", "-L", current.path).Output() if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } continue } - rpaths := append(loadRPaths(current.path, executable, current.path), current.rpaths...) + rpaths := append( + loadRPaths(ctx, current.path, executable, current.path), current.rpaths..., + ) for _, name := range parseOtool(output) { - dependency, ok := resolveDylib(name, executable, current.path, rpaths) + dependency, ok := resolveDylib(ctx, name, executable, current.path, rpaths) if !ok || seen[dependency.Path] { continue } @@ -38,23 +47,26 @@ func LinkedFiles(executable string) []LinkedDependency { } } } - return result + return result, nil } func parseOtool(output []byte) []string { lines := strings.Split(string(output), "\n") result := make([]string, 0, len(lines)) for _, line := range lines[1:] { - fields := strings.Fields(line) - if len(fields) > 0 { - result = append(result, fields[0]) + line = strings.TrimSpace(line) + if metadata := strings.LastIndex(line, " (compatibility version "); metadata >= 0 { + line = line[:metadata] + } + if line != "" { + result = append(result, line) } } return result } -func loadRPaths(path, executable, loader string) []string { - output, err := exec.Command("/usr/bin/otool", "-l", path).Output() +func loadRPaths(ctx context.Context, path, executable, loader string) []string { + output, err := exec.CommandContext(ctx, "/usr/bin/otool", "-l", path).Output() if err != nil { return nil } @@ -70,16 +82,21 @@ func loadRPaths(path, executable, loader string) []string { if !wantPath || !strings.HasPrefix(line, "path ") { continue } - fields := strings.Fields(line) - if len(fields) >= 2 { - result = append(result, expandDylibPath(fields[1], executable, loader)) + path := strings.TrimSpace(strings.TrimPrefix(line, "path ")) + if metadata := strings.LastIndex(path, " (offset "); metadata >= 0 { + path = path[:metadata] + } + if path != "" { + result = append(result, expandDylibPath(path, executable, loader)) } wantPath = false } return result } -func resolveDylib(name, executable, loader string, rpaths []string) (LinkedDependency, bool) { +func resolveDylib( + ctx context.Context, name, executable, loader string, rpaths []string, +) (LinkedDependency, bool) { var candidates []string if suffix, found := strings.CutPrefix(name, "@rpath/"); found { for _, rpath := range rpaths { @@ -101,7 +118,7 @@ func resolveDylib(name, executable, loader string, rpaths []string) (LinkedDepen sharedCacheCandidate = resolved } } - if sharedCacheCandidate != "" && inSharedCache(sharedCacheCandidate) { + if sharedCacheCandidate != "" && inSharedCache(ctx, sharedCacheCandidate) { return LinkedDependency{Path: sharedCacheCandidate, SharedCache: true}, true } return LinkedDependency{}, false @@ -117,6 +134,6 @@ func isSharedCachePath(path string) bool { strings.HasPrefix(path, "/System/Library/") } -func inSharedCache(path string) bool { - return exec.Command("/usr/bin/dyld_info", "-dependents", path).Run() == nil +func inSharedCache(ctx context.Context, path string) bool { + return exec.CommandContext(ctx, "/usr/bin/dyld_info", "-dependents", path).Run() == nil } diff --git a/internal/platform/dependencies_darwin_test.go b/internal/platform/dependencies_darwin_test.go index 41f9c59..9780f1a 100644 --- a/internal/platform/dependencies_darwin_test.go +++ b/internal/platform/dependencies_darwin_test.go @@ -1,6 +1,7 @@ package platform import ( + "context" "os" "path/filepath" "reflect" @@ -18,6 +19,16 @@ func TestParseOtool(t *testing.T) { } } +func TestParseOtoolPreservesPathsWithSpaces(t *testing.T) { + output := []byte("/tmp/tool:\n" + + "\t/Applications/My Tool/lib helper.dylib " + + "(compatibility version 1.0.0, current version 1.0.0)\n") + want := []string{"/Applications/My Tool/lib helper.dylib"} + if got := parseOtool(output); !reflect.DeepEqual(got, want) { + t.Fatalf("dependencies = %#v, want %#v", got, want) + } +} + func TestResolveDylibUsesRPathAndIdentifiesSharedCache(t *testing.T) { directory := t.TempDir() library := filepath.Join(directory, "liblocal.dylib") @@ -26,12 +37,14 @@ func TestResolveDylibUsesRPathAndIdentifiesSharedCache(t *testing.T) { } library, _ = filepath.EvalSymlinks(library) dependency, ok := resolveDylib( + context.Background(), "@rpath/liblocal.dylib", "/tmp/tool", "/tmp/tool", []string{directory}, ) if !ok || dependency.Path != library || dependency.SharedCache { t.Fatalf("resolved dependency = %+v, valid=%v", dependency, ok) } dependency, ok = resolveDylib( + context.Background(), "/usr/lib/libSystem.B.dylib", "/tmp/tool", "/tmp/tool", nil, ) if !ok || !dependency.SharedCache { diff --git a/internal/platform/dependencies_linux.go b/internal/platform/dependencies_linux.go index 38a20a9..9d2c449 100644 --- a/internal/platform/dependencies_linux.go +++ b/internal/platform/dependencies_linux.go @@ -1,19 +1,27 @@ package platform import ( + "context" + "fmt" "os" "os/exec" "path/filepath" "strings" ) -func LinkedFiles(executable string) []LinkedDependency { - output, _ := exec.Command("ldd", executable).CombinedOutput() +func LinkedFiles(ctx context.Context, executable string) ([]LinkedDependency, error) { + output, err := runLDD(ctx, executable) + if err != nil { + return nil, err + } files := parseLDD(output) if len(files) == 0 { - if interpreter := shebangInterpreter(executable); interpreter != "" { + for _, interpreter := range shebangExecutables(executable) { files = append(files, interpreter) - output, _ = exec.Command("ldd", interpreter).CombinedOutput() + output, err = runLDD(ctx, interpreter) + if err != nil { + return nil, err + } files = append(files, parseLDD(output)...) } } @@ -22,18 +30,35 @@ func LinkedFiles(executable string) []LinkedDependency { for index, path := range paths { result[index] = LinkedDependency{Path: path} } - return result + return result, nil +} + +func runLDD(ctx context.Context, executable string) ([]byte, error) { + output, err := exec.CommandContext(ctx, "ldd", executable).CombinedOutput() + if ctx.Err() != nil { + return nil, ctx.Err() + } + if err == nil { + return output, nil + } + message := strings.ToLower(string(output)) + if strings.Contains(message, "not a dynamic executable") || + strings.Contains(message, "statically linked") { + return output, nil + } + return nil, fmt.Errorf("inspect linked files for %q: %w: %s", + executable, err, strings.TrimSpace(string(output))) } func parseLDD(output []byte) []string { var files []string for _, line := range strings.Split(string(output), "\n") { - fields := strings.Fields(line) - candidate := "" - if len(fields) >= 3 && fields[1] == "=>" { - candidate = fields[2] - } else if len(fields) >= 1 && filepath.IsAbs(fields[0]) { - candidate = fields[0] + candidate := strings.TrimSpace(line) + if _, after, found := strings.Cut(candidate, "=>"); found { + candidate = strings.TrimSpace(after) + } + if metadata := strings.LastIndex(candidate, " ("); metadata >= 0 { + candidate = strings.TrimSpace(candidate[:metadata]) } if filepath.IsAbs(candidate) { files = append(files, candidate) @@ -42,17 +67,63 @@ func parseLDD(output []byte) []string { return files } -func shebangInterpreter(path string) string { - data, err := os.ReadFile(path) - if err != nil || !strings.HasPrefix(string(data), "#!") { - return "" +func shebangExecutables(path string) []string { + file, err := os.Open(path) + if err != nil { + return nil + } + defer file.Close() + buffer := make([]byte, 4096) + count, err := file.Read(buffer) + if err != nil && count == 0 { + return nil } - line, _, _ := strings.Cut(string(data[2:]), "\n") + data := string(buffer[:count]) + if !strings.HasPrefix(data, "#!") { + return nil + } + line, _, _ := strings.Cut(data[2:], "\n") fields := strings.Fields(line) if len(fields) == 0 || !filepath.IsAbs(fields[0]) { - return "" + return nil + } + result := []string{fields[0]} + if filepath.Base(fields[0]) != "env" { + return result } - return fields[0] + for index := 1; index < len(fields); index++ { + field := fields[index] + switch field { + case "-u", "--unset", "-C", "--chdir", "-a", "--argv0": + index++ + continue + case "-S", "--split-string", "--": + continue + } + if strings.HasPrefix(field, "-S") && len(field) > 2 { + field = field[2:] + } else if split, found := strings.CutPrefix(field, "--split-string="); found { + field = split + } else if strings.HasPrefix(field, "--unset=") || + strings.HasPrefix(field, "--chdir=") || + strings.HasPrefix(field, "--argv0=") { + continue + } + if strings.HasPrefix(field, "-") || strings.Contains(field, "=") { + continue + } + if split := strings.Fields(field); len(split) > 0 { + field = split[0] + } + if executable, err := exec.LookPath(field); err == nil { + if absolute, absErr := filepath.Abs(executable); absErr == nil { + executable = absolute + } + result = append(result, executable) + } + break + } + return result } func uniqueFiles(paths []string, executable string) []string { diff --git a/internal/platform/dependencies_linux_test.go b/internal/platform/dependencies_linux_test.go index cba74f6..7065fea 100644 --- a/internal/platform/dependencies_linux_test.go +++ b/internal/platform/dependencies_linux_test.go @@ -1,6 +1,9 @@ package platform import ( + "os" + "os/exec" + "path/filepath" "reflect" "testing" ) @@ -19,3 +22,55 @@ libmissing.so => not found t.Fatalf("files = %#v, want %#v", got, want) } } + +func TestParseLDDPreservesPathsWithSpaces(t *testing.T) { + output := []byte("libfoo.so => /tmp/lib dir/libfoo.so (0x123)\n" + + "/tmp/loader dir/ld.so (0x456)\n") + want := []string{"/tmp/lib dir/libfoo.so", "/tmp/loader dir/ld.so"} + if got := parseLDD(output); !reflect.DeepEqual(got, want) { + t.Fatalf("files = %#v, want %#v", got, want) + } +} + +func TestShebangExecutablesResolvesEnvInterpreter(t *testing.T) { + path := filepath.Join(t.TempDir(), "script") + if err := os.WriteFile(path, []byte("#!/usr/bin/env -S sh -eu\n"), 0o700); err != nil { + t.Fatal(err) + } + got := shebangExecutables(path) + sh, err := exec.LookPath("sh") + if err != nil { + t.Fatal(err) + } + sh, _ = filepath.Abs(sh) + want := []string{"/usr/bin/env", sh} + if !reflect.DeepEqual(got, want) { + t.Fatalf("interpreters = %#v, want %#v", got, want) + } +} + +func TestShebangExecutablesSkipsEnvOptionOperands(t *testing.T) { + path := filepath.Join(t.TempDir(), "script") + if err := os.WriteFile( + path, []byte("#!/usr/bin/env -u UNUSED -C /tmp sh -eu\n"), 0o700, + ); err != nil { + t.Fatal(err) + } + got := shebangExecutables(path) + if len(got) != 2 || filepath.Base(got[1]) != "sh" { + t.Fatalf("interpreters = %#v, want env and sh", got) + } +} + +func TestShebangExecutablesHandlesAttachedSplitString(t *testing.T) { + path := filepath.Join(t.TempDir(), "script") + if err := os.WriteFile( + path, []byte("#!/usr/bin/env -a custom --split-string=sh -eu\n"), 0o700, + ); err != nil { + t.Fatal(err) + } + got := shebangExecutables(path) + if len(got) != 2 || filepath.Base(got[1]) != "sh" { + t.Fatalf("interpreters = %#v, want env and sh", got) + } +} diff --git a/internal/platform/sampler_darwin.go b/internal/platform/sampler_darwin.go index e484419..394fe2e 100644 --- a/internal/platform/sampler_darwin.go +++ b/internal/platform/sampler_darwin.go @@ -13,34 +13,52 @@ static int sample_process(pid_t pid, struct proc_taskinfo *task, } if (proc_pid_rusage(pid, RUSAGE_INFO_V4, (rusage_info_t *)usage) != 0) { usage->ri_phys_footprint = 0; + return 1; } - return 1; + return 2; } */ import "C" import ( - "time" + "syscall" "unsafe" ) -func DefaultInterval() time.Duration { return 10 * time.Millisecond } +// GroupSampler owns the scratch buffers for one monitoring goroutine, so the +// per-tick sampling loop does not generate garbage while a run is measured. +// It is not safe for concurrent use. +type GroupSampler struct { + pids []C.pid_t +} + +func NewGroupSampler() *GroupSampler { return &GroupSampler{} } -func SampleImmediately() bool { return true } +func SampleProcessGroup(rootPID int) (Metrics, bool) { + return NewGroupSampler().Sample(rootPID) +} -func SampleTree(rootPID int) (Metrics, bool) { - pids := processGroupPIDs(rootPID) +func (sampler *GroupSampler) Sample(rootPID int) (Metrics, bool) { + pids := sampler.processGroupPIDs(rootPID) var total Metrics foundRoot := false + physicalFootprintValid := true for _, pid := range pids { if pid <= 0 { continue } var task C.struct_proc_taskinfo var usage C.struct_rusage_info_v4 - if C.sample_process(pid, &task, &usage) == 0 { + sampled := C.sample_process(pid, &task, &usage) + if sampled == 0 { + if syscall.Kill(int(pid), 0) == nil { + physicalFootprintValid = false + } continue } + if sampled < 2 { + physicalFootprintValid = false + } if int(pid) == rootPID { foundRoot = true } @@ -50,23 +68,34 @@ func SampleTree(rootPID int) (Metrics, bool) { total.Processes++ total.Threads += uint64(task.pti_threadnum) } + total.PhysicalFootprintValid = total.Processes > 0 && physicalFootprintValid + total.PhysicalFootprintInvalid = !total.PhysicalFootprintValid && + (foundRoot || syscall.Kill(rootPID, 0) == nil) return total, foundRoot } -func processGroupPIDs(groupID int) []C.pid_t { +func (sampler *GroupSampler) processGroupPIDs(groupID int) []C.pid_t { bytes := C.proc_listpgrppids(C.pid_t(groupID), nil, 0) if bytes <= 0 { return nil } - // Leave room for processes created between the sizing and data calls. - count := int(bytes) + 16 - pids := make([]C.pid_t, count) - count = int(C.proc_listpgrppids( - C.pid_t(groupID), unsafe.Pointer(&pids[0]), C.int(len(pids))*C.sizeof_pid_t, - )) - if count <= 0 { - return nil + // Leave room for process churn, and retry if the result fills the buffer. + capacity := int(bytes) + 16 + for range 3 { + if cap(sampler.pids) < capacity { + sampler.pids = make([]C.pid_t, capacity) + } + pids := sampler.pids[:capacity] + count := int(C.proc_listpgrppids( + C.pid_t(groupID), unsafe.Pointer(&pids[0]), C.int(len(pids))*C.sizeof_pid_t, + )) + if count <= 0 { + return nil + } + if count < len(pids) { + return pids[:count] + } + capacity *= 2 } - count = min(count, len(pids)) - return pids[:count] + return nil } diff --git a/internal/platform/sampler_darwin_test.go b/internal/platform/sampler_darwin_test.go index efcff2a..5636b10 100644 --- a/internal/platform/sampler_darwin_test.go +++ b/internal/platform/sampler_darwin_test.go @@ -6,7 +6,7 @@ import ( "testing" ) -func TestSampleTreeReadsProcessGroup(t *testing.T) { +func TestSampleProcessGroupReadsProcessGroup(t *testing.T) { cmd := exec.Command("/bin/sleep", "1") cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} if err := cmd.Start(); err != nil { @@ -17,7 +17,7 @@ func TestSampleTreeReadsProcessGroup(t *testing.T) { _ = cmd.Wait() }() - metrics, valid := SampleTree(cmd.Process.Pid) + metrics, valid := SampleProcessGroup(cmd.Process.Pid) if !valid { t.Fatal("process group leader was not sampled") } diff --git a/internal/platform/sampler_linux.go b/internal/platform/sampler_linux.go index a5c1d98..0b1e719 100644 --- a/internal/platform/sampler_linux.go +++ b/internal/platform/sampler_linux.go @@ -1,97 +1,270 @@ package platform import ( + "bytes" "errors" "io" + "math" "os" "path/filepath" + "runtime" "strconv" - "strings" - "time" -) + "unsafe" -func DefaultInterval() time.Duration { return 10 * time.Millisecond } + "golang.org/x/sys/unix" +) -func SampleImmediately() bool { return true } +// GroupSampler owns the scratch buffers for one monitoring goroutine, so the +// per-tick sampling loop does not generate garbage while a run is measured. +// It is not safe for concurrent use. +type GroupSampler struct { + statBuffer []byte + directoryBuffer []byte +} -func SampleTree(rootPID int) (Metrics, bool) { - seen := make(map[int]bool) - queue := []int{rootPID} - var total Metrics - for len(queue) > 0 { - pid := queue[0] - queue = queue[1:] - if seen[pid] { - continue - } - seen[pid] = true - record, err := readProcess(pid) - if err != nil { - if pid == rootPID { - return Metrics{}, false - } - continue - } - total.ResidentBytes += record.ResidentBytes - total.VirtualBytes += record.VirtualBytes - total.Processes++ - total.Threads += record.Threads - total.FileDescriptors += record.FileDescriptors - queue = append(queue, readChildren(pid)...) +func NewGroupSampler() *GroupSampler { + return &GroupSampler{ + statBuffer: make([]byte, 4096), + directoryBuffer: make([]byte, 32*1024), } - return total, true } -func readChildren(pid int) []int { - taskDirectory := filepath.Join("/proc", strconv.Itoa(pid), "task") - tasks, err := os.ReadDir(taskDirectory) +func SampleProcessGroup(rootPID int) (Metrics, bool) { + return NewGroupSampler().Sample(rootPID) +} + +func (sampler *GroupSampler) Sample(rootPID int) (Metrics, bool) { + directory, err := unix.Open("/proc", unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) if err != nil { - return nil + return Metrics{}, false } - seen := make(map[int]bool) - children := make([]int, 0, 2) - for _, task := range tasks { - data, readErr := os.ReadFile(filepath.Join(taskDirectory, task.Name(), "children")) + defer unix.Close(directory) + var total Metrics + rootFound := false + for { + count, readErr := unix.ReadDirent(directory, sampler.directoryBuffer) if readErr != nil { - continue + return Metrics{}, false } - for _, field := range strings.Fields(string(data)) { - child, parseErr := strconv.Atoi(field) - if parseErr == nil && !seen[child] { - seen[child] = true - children = append(children, child) + if count == 0 { + return total, rootFound + } + validDirents := scanProcDirents(sampler.directoryBuffer[:count], func(pid int) { + record, err := readProcessAt( + directory, pid, rootPID, sampler.statBuffer, + ) + if err != nil || record.GroupID != rootPID { + return + } + if record.PID == rootPID { + rootFound = true } + total.ResidentBytes += record.ResidentBytes + total.VirtualBytes += record.VirtualBytes + total.Processes++ + total.Threads += record.Threads + total.FileDescriptors += record.FileDescriptors + }) + if !validDirents { + return Metrics{}, false } } - return children } -func readProcess(pid int) (Process, error) { +func readProcess(pid int) (process, error) { base := filepath.Join("/proc", strconv.Itoa(pid)) data, err := os.ReadFile(filepath.Join(base, "stat")) if err != nil { - return Process{}, err + return process{}, err } - closeParen := strings.LastIndexByte(string(data), ')') + record, err := parseProcessStat(data, pid) + if err != nil { + return process{}, err + } + record.FileDescriptors = countDirectory(filepath.Join(base, "fd")) + return record, nil +} + +func readProcessAt( + procFD, pid, groupID int, buffer []byte, +) (process, error) { + file, err := openProcessStat(procFD, pid) + if err != nil { + return process{}, err + } + count, readErr := unix.Read(file, buffer) + _ = unix.Close(file) + if readErr != nil { + return process{}, readErr + } + if count == len(buffer) { + return process{}, errors.New("process stat exceeds buffer") + } + record, err := parseProcessStat(buffer[:count], pid) + if err != nil { + return process{}, err + } + if record.GroupID == groupID { + record.FileDescriptors = countDirectory( + "/proc/" + strconv.Itoa(pid) + "/fd", + ) + } + return record, nil +} + +func openProcessStat(procFD, pid int) (int, error) { + var storage [32]byte + path := strconv.AppendInt(storage[:0], int64(pid), 10) + path = append(path, '/', 's', 't', 'a', 't', 0) + file, _, errno := unix.Syscall6( + unix.SYS_OPENAT, + uintptr(procFD), uintptr(unsafe.Pointer(&path[0])), + uintptr(unix.O_RDONLY|unix.O_CLOEXEC), 0, 0, 0, + ) + runtime.KeepAlive(path) + if errno != 0 { + return -1, errno + } + return int(file), nil +} + +func parseProcessStat(data []byte, pid int) (process, error) { + closeParen := bytes.LastIndexByte(data, ')') if closeParen < 0 || closeParen+2 >= len(data) { - return Process{}, errors.New("malformed process stat") - } - fields := strings.Fields(string(data[closeParen+2:])) - if len(fields) < 22 { - return Process{}, errors.New("short process stat") - } - ppid, _ := strconv.Atoi(fields[1]) - threads, _ := strconv.ParseUint(fields[17], 10, 64) - virtual, _ := strconv.ParseUint(fields[20], 10, 64) - rssPages, _ := strconv.ParseInt(fields[21], 10, 64) - if rssPages < 0 { - rssPages = 0 - } - return Process{ - PID: pid, PPID: ppid, Threads: threads, VirtualBytes: virtual, - ResidentBytes: uint64(rssPages) * uint64(os.Getpagesize()), - FileDescriptors: countDirectory(filepath.Join(base, "fd")), - }, nil + return process{}, errors.New("malformed process stat") + } + fields := data[closeParen+2:] + var result process + result.PID = pid + field := 0 + for offset := 0; offset < len(fields); { + for offset < len(fields) && (fields[offset] == ' ' || fields[offset] == '\n') { + offset++ + } + start := offset + for offset < len(fields) && fields[offset] != ' ' && fields[offset] != '\n' { + offset++ + } + if start == offset { + break + } + value := fields[start:offset] + var err error + switch field { + case 1: + result.PPID, err = parsePositiveInt(value) + case 2: + result.GroupID, err = parsePositiveInt(value) + case 17: + result.Threads, err = parseUint(value) + case 20: + result.VirtualBytes, err = parseUint(value) + case 21: + var pages int64 + pages, err = parseInt64(value) + if pages > 0 { + pageSize := uint64(os.Getpagesize()) + if uint64(pages) > math.MaxUint64/pageSize { + err = errors.New("resident size overflows") + } else { + result.ResidentBytes = uint64(pages) * pageSize + } + } + } + if err != nil { + return process{}, err + } + field++ + if field > 21 { + return result, nil + } + } + return process{}, errors.New("short process stat") +} + +func scanProcDirents(data []byte, visit func(int)) bool { + // linux_dirent64 stores d_reclen at byte 16 and d_name at byte 19. + const recordHeader = 19 + for offset := 0; offset < len(data); { + if len(data)-offset < recordHeader { + return false + } + recordLength := int(data[offset+16]) | int(data[offset+17])<<8 + if recordLength < recordHeader || recordLength%8 != 0 || + recordLength > len(data)-offset { + return false + } + name := data[offset+recordHeader : offset+recordLength] + nameLength := bytes.IndexByte(name, 0) + if nameLength < 0 { + return false + } + if pid, ok := parsePIDBytes(name[:nameLength]); ok { + visit(pid) + } + offset += recordLength + } + return true +} + +func parsePIDBytes(value []byte) (int, bool) { + if len(value) == 0 { + return 0, false + } + result := 0 + for _, character := range value { + digit := character - '0' + if digit > 9 || result > (math.MaxInt-int(digit))/10 { + return 0, false + } + result = result*10 + int(digit) + } + return result, result > 0 +} + +func parsePositiveInt(value []byte) (int, error) { + parsed, err := parseUint(value) + if err != nil || parsed > uint64(math.MaxInt) { + return 0, errors.New("invalid process integer") + } + return int(parsed), nil +} + +func parseUint(value []byte) (uint64, error) { + if len(value) == 0 { + return 0, errors.New("empty process integer") + } + var result uint64 + for _, character := range value { + digit := character - '0' + if digit > 9 || result > (math.MaxUint64-uint64(digit))/10 { + return 0, errors.New("invalid process integer") + } + result = result*10 + uint64(digit) + } + return result, nil +} + +func parseInt64(value []byte) (int64, error) { + if len(value) == 0 { + return 0, errors.New("empty process integer") + } + negative := value[0] == '-' + if negative { + value = value[1:] + } + parsed, err := parseUint(value) + if err != nil || (!negative && parsed > math.MaxInt64) || + (negative && parsed > uint64(math.MaxInt64)+1) { + return 0, errors.New("invalid process integer") + } + if negative { + if parsed == uint64(math.MaxInt64)+1 { + return math.MinInt64, nil + } + return -int64(parsed), nil + } + return int64(parsed), nil } func countDirectory(path string) uint64 { diff --git a/internal/platform/sampler_linux_bench_test.go b/internal/platform/sampler_linux_bench_test.go new file mode 100644 index 0000000..9519172 --- /dev/null +++ b/internal/platform/sampler_linux_bench_test.go @@ -0,0 +1,26 @@ +package platform + +import ( + "os/exec" + "syscall" + "testing" +) + +func BenchmarkSampleProcessGroup(b *testing.B) { + command := exec.Command("sleep", "60") + command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := command.Start(); err != nil { + b.Fatal(err) + } + b.Cleanup(func() { + _ = syscall.Kill(-command.Process.Pid, syscall.SIGKILL) + _ = command.Wait() + }) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + if _, valid := SampleProcessGroup(command.Process.Pid); !valid { + b.Fatal("sampled process disappeared") + } + } +} diff --git a/internal/platform/sampler_linux_test.go b/internal/platform/sampler_linux_test.go index 493042e..035daa7 100644 --- a/internal/platform/sampler_linux_test.go +++ b/internal/platform/sampler_linux_test.go @@ -3,9 +3,15 @@ package platform import ( + "fmt" "os" + "os/exec" "path/filepath" + "strconv" + "strings" + "syscall" "testing" + "time" ) func TestCountDirectoryDoesNotRequireSortedEntries(t *testing.T) { @@ -21,7 +27,105 @@ func TestCountDirectoryDoesNotRequireSortedEntries(t *testing.T) { } func TestMissingRootIsNotAZeroSample(t *testing.T) { - if _, valid := SampleTree(1 << 30); valid { + if _, valid := SampleProcessGroup(1 << 30); valid { t.Fatal("missing root process should produce an invalid sample") } } + +func TestScanProcDirentsVisitsOnlyNumericNames(t *testing.T) { + var data []byte + for _, name := range []string{".", "self", "123", "98765"} { + length := (19 + len(name) + 1 + 7) &^ 7 + record := make([]byte, length) + record[16], record[17] = byte(length), byte(length>>8) + copy(record[19:], name) + data = append(data, record...) + } + var pids []int + if !scanProcDirents(data, func(pid int) { pids = append(pids, pid) }) { + t.Fatal("valid dirents rejected") + } + if fmt.Sprint(pids) != "[123 98765]" { + t.Fatalf("PIDs = %v", pids) + } + if scanProcDirents(data[:len(data)-1], func(int) {}) { + t.Fatal("truncated dirent should be rejected") + } + missingNUL := make([]byte, 24) + missingNUL[16] = byte(len(missingNUL)) + copy(missingNUL[19:], "12345") + if scanProcDirents(missingNUL, func(int) {}) { + t.Fatal("dirent without a NUL-terminated name should be rejected") + } + misaligned := make([]byte, 23) + misaligned[16] = byte(len(misaligned)) + if scanProcDirents(misaligned, func(int) {}) { + t.Fatal("unaligned dirent should be rejected") + } +} + +func TestSampleProcessGroupIncludesReparentedMembers(t *testing.T) { + childFile := filepath.Join(t.TempDir(), "child-pid") + cmd := exec.Command( + "sh", "-c", + "(sleep 30 & printf '%s' $! > \"$1\"); exec sleep 30", + "sh", childFile, + ) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + _ = cmd.Wait() + }) + + var child process + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + data, err := os.ReadFile(childFile) + if err == nil { + childPID, parseErr := strconv.Atoi(strings.TrimSpace(string(data))) + if parseErr == nil { + child, err = readProcess(childPID) + if err == nil && child.PPID != cmd.Process.Pid { + break + } + } + } + time.Sleep(10 * time.Millisecond) + } + if child.PID == 0 || child.PPID == cmd.Process.Pid || child.GroupID != cmd.Process.Pid { + t.Fatalf("child process was not reparented in group: %+v", child) + } + metrics, valid := SampleProcessGroup(cmd.Process.Pid) + if !valid || metrics.Processes < 2 { + t.Fatalf("group metrics = %+v, valid = %v", metrics, valid) + } +} + +func TestParseProcessStatValidatesRequiredNumbers(t *testing.T) { + fields := make([]string, 22) + for index := range fields { + fields[index] = "0" + } + fields[0], fields[1], fields[2] = "S", "42", "123" + fields[17], fields[20], fields[21] = "7", "4096", "2" + data := []byte(fmt.Sprintf("99 (name with ) parenthesis) %s", strings.Join(fields, " "))) + process, err := parseProcessStat(data, 99) + if err != nil { + t.Fatal(err) + } + if process.PPID != 42 || process.GroupID != 123 || process.Threads != 7 || + process.VirtualBytes != 4096 || process.ResidentBytes == 0 { + t.Fatalf("parsed process = %+v", process) + } + for _, field := range []int{1, 2, 17, 20, 21} { + invalid := append([]string(nil), fields...) + invalid[field] = "invalid" + data = []byte(fmt.Sprintf("99 (name) %s", strings.Join(invalid, " "))) + if _, err := parseProcessStat(data, 99); err == nil { + t.Fatalf("field %d should reject malformed input", field) + } + } +} diff --git a/internal/platform/types.go b/internal/platform/types.go index aa90041..9e8f710 100644 --- a/internal/platform/types.go +++ b/internal/platform/types.go @@ -1,20 +1,22 @@ package platform +import "time" + +func DefaultInterval() time.Duration { return 10 * time.Millisecond } + type Metrics struct { - ResidentBytes uint64 - PhysicalFootprintBytes uint64 - VirtualBytes uint64 - Processes uint64 - Threads uint64 - FileDescriptors uint64 - ResidentByteSamples uint64 - SampleCount uint64 - ResidentByteSeconds float64 - SampleCoverageSeconds float64 + ResidentBytes uint64 + PhysicalFootprintBytes uint64 + PhysicalFootprintValid bool + PhysicalFootprintInvalid bool + VirtualBytes uint64 + Processes uint64 + Threads uint64 + FileDescriptors uint64 } -type Process struct { - PID, PPID int +type process struct { + PID, PPID, GroupID int ResidentBytes, VirtualBytes uint64 Threads, FileDescriptors uint64 } diff --git a/internal/report/chart_artifacts.go b/internal/report/chart_artifacts.go index fe822c9..4e10614 100644 --- a/internal/report/chart_artifacts.go +++ b/internal/report/chart_artifacts.go @@ -4,8 +4,6 @@ import ( "fmt" "os" "path/filepath" - - "github.com/shellcell/snailrace/internal/model" ) type ChartArtifact struct { @@ -15,17 +13,20 @@ type ChartArtifact struct { Filename string } -func WriteChartFiles( - directory string, - report model.Report, - includeCommands bool, +func (renderer *Renderer) WriteChartFiles( + directory string, includeCommands bool, ) ([]ChartArtifact, error) { + report := renderer.displayReport if err := os.MkdirAll(directory, 0o755); err != nil { return nil, err } - charts := reportCharts(report) + charts := renderer.reportCharts() if includeCommands { - charts = append([]svgChart{commandLegendChart(report)}, charts...) + prefix := []svgChart{commandLegendChart(report)} + if failures := failureChart(report); failures.height > 0 { + prefix = append(prefix, failures) + } + charts = append(prefix, charts...) } artifacts := make([]ChartArtifact, 0, len(charts)) for index, chart := range charts { diff --git a/internal/report/chart_commands.go b/internal/report/chart_commands.go index 94dea41..2a23db7 100644 --- a/internal/report/chart_commands.go +++ b/internal/report/chart_commands.go @@ -20,7 +20,7 @@ func commandLegendChart(report model.Report) svgChart { y := height fmt.Fprintf( &body, `%d. %s`, - y, toolColor(report, index), index+1, + y, toolColor(index), index+1, html.EscapeString(reportToolLabel(report, index)), ) for _, line := range lines { diff --git a/internal/report/chart_delta.go b/internal/report/chart_delta.go index 6dfcfa1..6c433e4 100644 --- a/internal/report/chart_delta.go +++ b/internal/report/chart_delta.go @@ -5,8 +5,6 @@ import ( "html" "math" "strings" - - "github.com/shellcell/snailrace/internal/model" ) type forestRow struct { @@ -18,33 +16,46 @@ type forestRow struct { baseline bool } -func deltaForestChart(report model.Report, group chartGroup) svgChart { +func deltaForestChart(renderer *Renderer, group chartGroup) svgChart { + report := renderer.displayReport baselinePosition := baselineIndex(report) baseline := report.Benchmarks[baselinePosition] rows := make([]forestRow, 0) maximum := 0.0 for _, metric := range group.metrics { - row := metricRows[metric.row] - if !available(row, report.Host.OS) { + row := metric + if !availableFor(row, report.Host.OS, baseline.Summary) { rows = append(rows, forestRow{ - label: metric.name, status: "N/A on " + report.Host.OS, - class: "neutral", identity: toolColor(report, baselinePosition), + label: metric.chartName, status: "N/A on " + report.Host.OS, + class: "neutral", identity: toolColor(baselinePosition), baseline: true, }) continue } - baseStats := metric.stats(baseline) + baseStats := metric.stats(baseline.Summary) baseMean := baseStats.Mean + if baseStats.N == 0 || !finiteNumber(baseMean) { + rows = append(rows, forestRow{ + label: metric.chartName, status: "N/A", class: "neutral", + identity: toolColor(baselinePosition), baseline: true, + }) + continue + } baselineRow := forestRow{ - label: metric.name + " Β· " + reportToolLabel(report, baselinePosition), + label: metric.chartName + " Β· " + reportToolLabel(report, baselinePosition), status: metric.format(baseMean) + " Β· baseline", - class: "neutral", identity: toolColor(report, baselinePosition), + class: "neutral", identity: toolColor(baselinePosition), baseline: true, } if baseMean != 0 && baseStats.CI95Valid { baselineRow.low, baselineRow.high = percentInterval(baseStats.CI95Low, baseStats.CI95High, baseMean) - baselineRow.interval = true - maximum = math.Max(maximum, math.Max(math.Abs(baselineRow.low), math.Abs(baselineRow.high))) + baselineRow.interval = finiteNumber(baselineRow.low) && + finiteNumber(baselineRow.high) + if baselineRow.interval { + maximum = math.Max( + maximum, math.Max(math.Abs(baselineRow.low), math.Abs(baselineRow.high)), + ) + } } rows = append(rows, baselineRow) if baseMean == 0 { @@ -52,14 +63,19 @@ func deltaForestChart(report model.Report, group chartGroup) svgChart { if index == baselinePosition { continue } - value := metric.stats(candidate).Mean - delta := compareMetric( - baseline, candidate, row, report.Config.IntervalMS/1000, - ) + if !availableFor(row, report.Host.OS, candidate.Summary) { + rows = append(rows, forestRow{ + label: metric.chartName + " Β· " + candidate.Tool.Name, + status: "N/A", class: "neutral", identity: toolColor(index), + }) + continue + } + value := metric.stats(candidate.Summary).Mean + delta := renderer.comparison(index, metric.id) rows = append(rows, forestRow{ - label: metric.name + " Β· " + candidate.Tool.Name, + label: metric.chartName + " Β· " + candidate.Tool.Name, status: metric.format(value) + " Β· Ξ”% n/a Β· " + delta.status, - class: delta.class, identity: toolColor(report, index), + class: delta.class, identity: toolColor(index), baseline: true, }) } @@ -69,18 +85,32 @@ func deltaForestChart(report model.Report, group chartGroup) svgChart { if index == baselinePosition { continue } - delta := compareMetric( - baseline, candidate, row, report.Config.IntervalMS/1000, - ) - low := delta.difference.CI95Low / baseMean * 100 - high := delta.difference.CI95High / baseMean * 100 - maximum = math.Max(maximum, math.Max(math.Abs(low), math.Abs(high))) - maximum = math.Max(maximum, math.Abs(delta.percent)) + if !availableFor(row, report.Host.OS, candidate.Summary) { + rows = append(rows, forestRow{ + label: metric.chartName + " Β· " + candidate.Tool.Name, + status: "N/A", class: "neutral", identity: toolColor(index), + }) + continue + } + delta := renderer.comparison(index, metric.id) + low, high := 0.0, 0.0 + interval := delta.difference.CI95Valid + if interval { + low = delta.difference.CI95Low / baseMean * 100 + high = delta.difference.CI95High / baseMean * 100 + interval = finiteNumber(low) && finiteNumber(high) + } + if interval { + maximum = math.Max(maximum, math.Max(math.Abs(low), math.Abs(high))) + } + if delta.percentAvailable { + maximum = math.Max(maximum, math.Abs(delta.percent)) + } rows = append(rows, forestRow{ - label: metric.name + " Β· " + candidate.Tool.Name, + label: metric.chartName + " Β· " + candidate.Tool.Name, point: delta.percent, low: low, high: high, status: delta.status, class: delta.class, - identity: toolColor(report, index), interval: delta.difference.CI95Valid, + identity: toolColor(index), interval: interval, }) } } diff --git a/internal/report/chart_distribution.go b/internal/report/chart_distribution.go index 4022c85..2a2c1af 100644 --- a/internal/report/chart_distribution.go +++ b/internal/report/chart_distribution.go @@ -10,14 +10,24 @@ import ( ) func absoluteDistributionChart(report model.Report, metric chartMetric) svgChart { - if !available(metricRows[metric.row], report.Host.OS) { - return unavailableChart("RUN DISTRIBUTION Β· "+metric.name, report.Host.OS) + if !available(metric, report.Host.OS) { + return unavailableChart("RUN DISTRIBUTION Β· "+metric.chartName, report.Host.OS) + } + for _, benchmark := range report.Benchmarks { + if !availableFor(metric, report.Host.OS, benchmark.Summary) { + return unavailableChart("RUN DISTRIBUTION Β· "+metric.chartName, "metric unavailable") + } } minimum, maximum := 0.0, 0.0 for _, benchmark := range report.Benchmarks { - stats := metric.stats(benchmark) + stats := metric.stats(benchmark.Summary) + if stats.N == 0 || !finiteNumber(stats.Min) || + !finiteNumber(stats.Max) || !finiteNumber(stats.Mean) { + return svgChart{} + } maximum = math.Max(maximum, stats.Max) - if stats.CI95Valid { + if stats.CI95Valid && finiteNumber(stats.CI95Low) && + finiteNumber(stats.CI95High) { minimum = math.Min(minimum, stats.CI95Low) maximum = math.Max(maximum, stats.CI95High) } @@ -31,13 +41,18 @@ func absoluteDistributionChart(report model.Report, metric chartMetric) svgChart return left + (value-minimum)/(maximum-minimum)*plotWidth } var body strings.Builder + points := 0 + for _, benchmark := range report.Benchmarks { + points += len(benchmark.Runs) + } + body.Grow(1024 + len(report.Benchmarks)*384 + points*96) body.WriteString(svgChartStyle) fmt.Fprintf( &body, ``+ `RUN DISTRIBUTION Β· %s`+ `dots = runs Β· diamond = mean Β· whisker = mean 95%% CI`, - html.EscapeString(metric.name), + html.EscapeString(metric.chartName), ) fmt.Fprintf( &body, @@ -45,10 +60,9 @@ func absoluteDistributionChart(report model.Report, metric chartMetric) svgChart `%s`, left, metric.format(minimum), left+plotWidth, metric.format(maximum), ) - baseline := baselineIndex(report) for index, benchmark := range report.Benchmarks { y := 76 + index*rowHeight - color := chartColor(index, baseline) + color := toolColor(index) fmt.Fprintf( &body, `%s`+ @@ -57,16 +71,22 @@ func absoluteDistributionChart(report model.Report, metric chartMetric) svgChart left, y-4, left+plotWidth, ) for runIndex, run := range benchmark.Runs { + if !chartRunAvailable(metric, run) { + continue + } jitter := (runIndex%3 - 1) * 5 - fmt.Fprintf( - &body, ``, - position(metric.run(run)), y-4+jitter, color, - ) + body.WriteString(``) } - stats := metric.stats(benchmark) + stats := metric.stats(benchmark.Summary) mean := position(stats.Mean) - if stats.CI95Valid { + if stats.CI95Valid && finiteNumber(stats.CI95Low) && + finiteNumber(stats.CI95High) { low, high := position(stats.CI95Low), position(stats.CI95High) fmt.Fprintf( &body, @@ -87,8 +107,8 @@ func absoluteDistributionChart(report model.Report, metric chartMetric) svgChart ) } return svgChart{ - kind: "distribution", title: metric.name, slug: chartSlug(metric.name), - description: distributionChartDescription(metric.name), + kind: "distribution", title: metric.chartName, slug: chartSlug(metric.chartName), + description: distributionChartDescription(metric.chartName), body: body.String(), height: height, } } diff --git a/internal/report/chart_metrics.go b/internal/report/chart_metrics.go index f8cc921..fb78043 100644 --- a/internal/report/chart_metrics.go +++ b/internal/report/chart_metrics.go @@ -2,22 +2,26 @@ package report import "github.com/shellcell/snailrace/internal/model" -func reportCharts(report model.Report) []svgChart { - groups := chartGroups(report) +func buildReportCharts(renderer *Renderer) []svgChart { + report := renderer.displayReport + if len(report.Benchmarks) == 0 { + return nil + } + groups := renderer.chartGroups var charts []svgChart if len(report.Benchmarks) > 1 { - charts = append(charts, rankingCharts(report)...) + charts = append(charts, rankingCharts(report, renderer.ranking)...) for _, group := range groups { for _, metric := range group.metrics { - chart := deltaForestChart(report, chartGroup{ - name: metric.name, metrics: []chartMetric{metric}, + chart := deltaForestChart(renderer, chartGroup{ + name: metric.chartName, metrics: []chartMetric{metric}, }) if chart.height > 0 { charts = append(charts, chart) } } } - charts = append(charts, tradeoffCharts(report)...) + charts = append(charts, tradeoffCharts(report, renderer.ranking)...) } for _, group := range groups { for _, metric := range group.metrics { @@ -34,64 +38,39 @@ func reportCharts(report model.Report) []svgChart { ); chart.height > 0 { charts = append(charts, chart) } - charts = append(charts, measurementTrendCharts(report)...) + charts = append(charts, measurementTrendCharts(report, groups)...) return charts } func chartGroups(report model.Report) []chartGroup { - metrics := chartMetricDefinitions() groups := []chartGroup{ - {"PERFORMANCE", metrics[0:1]}, - {"CPU COST", metrics[1:5]}, - {"MEMORY COST", metrics[5:10]}, - {"PROCESS STRUCTURE", metrics[10:13]}, + {"PERFORMANCE", chartMetrics(metricWall)}, + {"CPU COST", chartMetrics( + metricCPUTotal, metricCPUUser, metricCPUSystem, metricAverageCPU, + )}, + {"MEMORY COST", chartMetrics( + metricMeanResident, metricPeakResident, metricOSMaxRSS, + metricPhysicalFootprint, metricPeakVirtual, + )}, + {"PROCESS STRUCTURE", chartMetrics( + metricPeakProcesses, metricPeakThreads, metricPeakFDs, + )}, } - if report.Config.Mode == "tui" && report.Config.DurationSeconds > 0 { + if report.Config.FixedDurationTUI() { groups = groups[1:] } return groups } -func chartMetricDefinitions() []chartMetric { - return []chartMetric{ - {"Wall time", formatDuration, - func(b model.Benchmark) model.Stats { return b.Summary.WallSeconds }, - func(r model.Run) float64 { return r.WallSeconds }, 0}, - {"Total CPU", formatDuration, - func(b model.Benchmark) model.Stats { return b.Summary.CPUTotalSeconds }, - func(r model.Run) float64 { return r.CPUUserSeconds + r.CPUSystemSeconds }, 1}, - {"User CPU", formatDuration, - func(b model.Benchmark) model.Stats { return b.Summary.CPUUserSeconds }, - func(r model.Run) float64 { return r.CPUUserSeconds }, 2}, - {"System CPU", formatDuration, - func(b model.Benchmark) model.Stats { return b.Summary.CPUSystemSeconds }, - func(r model.Run) float64 { return r.CPUSystemSeconds }, 3}, - {"Average CPU", formatPercent, - func(b model.Benchmark) model.Stats { return b.Summary.AverageCPUPercent }, - func(r model.Run) float64 { return r.AverageCPUPercent }, 4}, - {"Mean RSS", formatBytes, - func(b model.Benchmark) model.Stats { return b.Summary.MeanResidentBytes }, - func(r model.Run) float64 { return r.MeanResidentBytes }, 5}, - {"Peak RSS", formatBytes, - func(b model.Benchmark) model.Stats { return b.Summary.PeakResidentBytes }, - func(r model.Run) float64 { return r.PeakResidentBytes }, 6}, - {"OS-reported max RSS", formatBytes, - func(b model.Benchmark) model.Stats { return b.Summary.OSMaxRSSBytes }, - func(r model.Run) float64 { return r.OSMaxRSSBytes }, 7}, - {"Peak physical footprint", formatBytes, - func(b model.Benchmark) model.Stats { return b.Summary.PhysicalFootprintStats() }, - func(r model.Run) float64 { return r.PeakPhysicalFootprintBytes }, 8}, - {"Peak virtual memory", formatBytes, - func(b model.Benchmark) model.Stats { return b.Summary.PeakVirtualBytes }, - func(r model.Run) float64 { return r.PeakVirtualBytes }, 9}, - {"Peak processes", formatCount, - func(b model.Benchmark) model.Stats { return b.Summary.PeakProcesses }, - func(r model.Run) float64 { return r.PeakProcesses }, 10}, - {"Peak threads", formatCount, - func(b model.Benchmark) model.Stats { return b.Summary.PeakThreads }, - func(r model.Run) float64 { return r.PeakThreads }, 11}, - {"Peak FD references", formatCount, - func(b model.Benchmark) model.Stats { return b.Summary.PeakFileDescriptors }, - func(r model.Run) float64 { return r.PeakFileDescriptors }, 12}, +func chartMetrics(ids ...metricID) []chartMetric { + metrics := make([]chartMetric, len(ids)) + for index, id := range ids { + metrics[index] = metricCatalog[id] } + return metrics +} + +func chartRunAvailable(metric chartMetric, run model.Run) bool { + return (metric.id != metricPhysicalFootprint || run.PhysicalFootprintValid) && + finiteNumber(metric.run(run)) } diff --git a/internal/report/chart_ranking.go b/internal/report/chart_ranking.go index 61dc8d1..f29477a 100644 --- a/internal/report/chart_ranking.go +++ b/internal/report/chart_ranking.go @@ -4,6 +4,7 @@ import ( "fmt" "html" "math" + "sort" "strings" "github.com/shellcell/snailrace/internal/model" @@ -16,40 +17,43 @@ type rankingChartMetric struct { rank func(rankingRow) int } -func rankingCharts(report model.Report) []svgChart { - ranking := calculateRanking(report) - metrics := []rankingChartMetric{ - {"BALANCED INDEX", formatScore, - func(row rankingRow) float64 { return row.overallScore }, - func(row rankingRow) int { return row.overallRank }}, - {ranking.primaryLabel, ranking.primaryUnit, - func(row rankingRow) float64 { return row.primaryValue }, - func(row rankingRow) int { return row.primaryRank }}, - } - if !(report.Config.Mode == "tui" && report.Config.DurationSeconds > 0) { +func rankingCharts(report model.Report, ranking rankingData) []svgChart { + if len(ranking.Rows) == 0 { + return nil + } + var metrics []rankingChartMetric + if ranking.Available { + metrics = append(metrics, rankingChartMetric{"BALANCED INDEX", formatScore, + func(row rankingRow) float64 { return row.OverallScore }, + func(row rankingRow) int { return row.OverallRank }}) + } + metrics = append(metrics, rankingChartMetric{ranking.primaryLabel, ranking.primaryUnit, + func(row rankingRow) float64 { return row.PrimaryValue }, + func(row rankingRow) int { return row.PrimaryRank }}) + if !report.Config.FixedDurationTUI() { metrics = append(metrics, rankingChartMetric{ "CPU COST", formatDuration, - func(row rankingRow) float64 { return row.cpuValue }, - func(row rankingRow) int { return row.cpuRank }, + func(row rankingRow) float64 { return row.CPUValue }, + func(row rankingRow) int { return row.CPURank }, }) } - if ranking.ramPresent { - ramRank := func(row rankingRow) int { return row.ramRank } - if !ranking.ramAvailable { + if ranking.RAMPresent { + ramRank := func(row rankingRow) int { return row.RAMRank } + if !ranking.RAMAvailable { // Sampling-limited RAM has no reliable score; rank descriptively by value. - order := rankByValue(ranking.rows, func(row rankingRow) float64 { return row.ramValue }) - ramRank = func(row rankingRow) int { return order[row.benchmark] } + order := rankByValue(ranking.Rows, func(row rankingRow) float64 { return row.RAMValue }) + ramRank = func(row rankingRow) int { return order[row.Benchmark] } } metrics = append(metrics, rankingChartMetric{ "RAM AGGREGATE", formatBytes, - func(row rankingRow) float64 { return row.ramValue }, + func(row rankingRow) float64 { return row.RAMValue }, ramRank, }) } metrics = append(metrics, rankingChartMetric{ "LINKED SIZE", formatBytes, - func(row rankingRow) float64 { return row.footprintValue }, - func(row rankingRow) int { return row.footprintRank }, + func(row rankingRow) float64 { return row.FootprintValue }, + func(row rankingRow) int { return row.FootprintRank }, }) charts := make([]svgChart, 0, len(metrics)) for _, metric := range metrics { @@ -60,14 +64,16 @@ func rankingCharts(report model.Report) []svgChart { func rankByValue(rows []rankingRow, value func(rankingRow) float64) map[int]int { result := make(map[int]int, len(rows)) - for _, row := range rows { - rank := 1 - for _, other := range rows { - if value(other) < value(row) { - rank++ - } + ordered := append([]rankingRow(nil), rows...) + sort.SliceStable(ordered, func(left, right int) bool { + return value(ordered[left]) < value(ordered[right]) + }) + rank := 0 + for position, row := range ordered { + if position == 0 || value(row) != value(ordered[position-1]) { + rank = position + 1 } - result[row.benchmark] = rank + result[row.Benchmark] = rank } return result } @@ -79,17 +85,21 @@ func rankingBarChart( ) svgChart { const left, plotWidth, rowHeight = 205, 355, 30 maximum := 0.0 - for _, row := range ranking.rows { - maximum = math.Max(maximum, metric.value(row)) + for _, row := range ranking.Rows { + value := metric.value(row) + if !finiteNumber(value) || value < 0 { + return svgChart{} + } + maximum = math.Max(maximum, value) } if maximum <= 0 { maximum = 1 } - height := 62 + len(ranking.rows)*rowHeight + height := 62 + len(ranking.Rows)*rowHeight var body strings.Builder body.WriteString(svgChartStyle) subtitle := "descriptive point estimates Β· outline = category leader" - if metric.name == "RAM AGGREGATE" && ranking.ramPresent && !ranking.ramAvailable { + if metric.name == "RAM AGGREGATE" && ranking.RAMPresent && !ranking.RAMAvailable { subtitle = "sampling-limited Β· descriptive only Β· excluded from balanced index" if ranking.samplingInterval != "" { subtitle += " Β· lower -interval (now " + ranking.samplingInterval + ")" @@ -102,11 +112,15 @@ func rankingBarChart( `%s`, html.EscapeString(metric.name), html.EscapeString(subtitle), ) - for index, row := range ranking.rows { + rows := append([]rankingRow(nil), ranking.Rows...) + sort.SliceStable(rows, func(left, right int) bool { + return metric.rank(rows[left]) < metric.rank(rows[right]) + }) + for index, row := range rows { y := 62 + index*rowHeight value := metric.value(row) width := value / maximum * plotWidth - color := toolColor(report, row.benchmark) + color := toolColor(row.Benchmark) outline := "none" outlineWidth := 0 if metric.rank(row) == 1 { @@ -120,7 +134,7 @@ func rankingBarChart( `fill="%s" stroke="%s" stroke-width="%d"/>`+ `%s`, y, color, metric.rank(row), - html.EscapeString(clip(reportToolLabel(report, row.benchmark), 26)), + html.EscapeString(clip(reportToolLabel(report, row.Benchmark), 26)), left, y-11, width, color, outline, outlineWidth, y, metric.format(value), ) } @@ -149,7 +163,7 @@ func rankingChartDescription(metric string, ranking rankingData) string { "this ranking does not include uncertainty intervals." case "RAM AGGREGATE": caveat := "" - if ranking.ramPresent && !ranking.ramAvailable { + if ranking.RAMPresent && !ranking.RAMAvailable { caveat = " These values are sampling-limited (short runs or too few valid " + "samples), so they are shown for reference but excluded from the balanced index." if ranking.samplingInterval != "" { @@ -172,16 +186,16 @@ func rankingChartDescription(metric string, ranking rankingData) string { func balancedIndexCategories(ranking rankingData) string { var categories []string - if ranking.indexPrimary { + if ranking.IndexPrimary { categories = append(categories, primaryCategoryName(ranking.primaryLabel)) } - if ranking.indexCPU { + if ranking.IndexCPU { categories = append(categories, "CPU cost") } - if ranking.indexRAM { + if ranking.IndexRAM { categories = append(categories, "RAM aggregate") } - if ranking.indexFootprint { + if ranking.IndexFootprint { categories = append(categories, "linked size") } if len(categories) == 0 { diff --git a/internal/report/chart_static.go b/internal/report/chart_static.go index 8b65d8f..cfaa8ae 100644 --- a/internal/report/chart_static.go +++ b/internal/report/chart_static.go @@ -15,6 +15,9 @@ func staticCostChart( pick func(model.Benchmark) float64, format func(float64) string, ) svgChart { + if len(report.Benchmarks) == 0 { + return svgChart{} + } maximum := 0.0 for _, benchmark := range report.Benchmarks { value := pick(benchmark) @@ -40,12 +43,11 @@ func staticCostChart( `%s`, left, left+plotWidth, format(maximum), ) - baseline := baselineIndex(report) for index, benchmark := range report.Benchmarks { y := 72 + index*rowHeight value := pick(benchmark) x := left + value/maximum*plotWidth - color := chartColor(index, baseline) + color := toolColor(index) fmt.Fprintf( &body, `%s`+ diff --git a/internal/report/chart_test.go b/internal/report/chart_test.go index 46c5b6b..90bd960 100644 --- a/internal/report/chart_test.go +++ b/internal/report/chart_test.go @@ -1,12 +1,41 @@ package report import ( + "math" "strings" "testing" "github.com/shellcell/snailrace/internal/model" ) +func TestEmptyReportBuildsNoCharts(t *testing.T) { + if charts := NewRenderer(model.Report{}).reportCharts(); len(charts) != 0 { + t.Fatalf("empty report charts = %d, want 0", len(charts)) + } +} + +func TestChartsOmitNonFiniteCoordinates(t *testing.T) { + runs := []model.Run{ + {Index: 1, WallSeconds: math.Inf(1)}, + {Index: 2, WallSeconds: math.NaN()}, + } + report := model.Report{ + Config: model.Config{Baseline: 1, Mode: "command"}, + Host: model.HostInfo{OS: "linux"}, + Benchmarks: []model.Benchmark{{ + Tool: model.ToolInfo{Name: "invalid", DiskFootprintBytes: 1}, + Runs: runs, Summary: model.Summarize(runs), + }}, + } + for _, chart := range NewRenderer(report).reportCharts() { + output := chart.html() + if strings.Contains(output, "NaN") || strings.Contains(output, "+Inf") || + strings.Contains(output, "-Inf") { + t.Fatalf("chart %q contains non-finite coordinates", chart.title) + } + } +} + func TestComparisonChartsContainDeltaAndRunViews(t *testing.T) { report := model.Report{ Config: model.Config{Mode: "command", Baseline: 1}, @@ -15,7 +44,7 @@ func TestComparisonChartsContainDeltaAndRunViews(t *testing.T) { benchmarkWithWallTimes("candidate", 8, 8, 8), }, } - charts := reportCharts(report) + charts := NewRenderer(report).reportCharts() if len(charts) < 2 { t.Fatalf("charts = %d, want delta and distribution charts", len(charts)) } @@ -44,13 +73,13 @@ func TestBaselineChartsSeparateEveryMetric(t *testing.T) { }, } var titles []string - for _, chart := range reportCharts(report) { + for _, chart := range NewRenderer(report).reportCharts() { if chart.kind == "baseline" { titles = append(titles, chart.title) } } - if len(titles) != len(chartMetricDefinitions()) { - t.Fatalf("baseline charts = %d, want %d: %v", len(titles), len(chartMetricDefinitions()), titles) + if len(titles) != len(metricCatalog) { + t.Fatalf("baseline charts = %d, want %d: %v", len(titles), len(metricCatalog), titles) } for _, expected := range []string{ "Mean RSS", "Peak RSS", "OS-reported max RSS", "Peak virtual memory", @@ -73,8 +102,8 @@ func TestBaselineChartShowsBaselineConfidenceWhisker(t *testing.T) { benchmarkWithWallTimes("candidate", 8, 9, 10), }, } - chart := deltaForestChart(report, chartGroup{ - name: "Wall time", metrics: []chartMetric{chartMetricDefinitions()[0]}, + chart := deltaForestChart(NewRenderer(report), chartGroup{ + name: "Wall time", metrics: chartMetrics(metricWall), }) if !strings.Contains(chart.body, `class="baseline-ci"`) { t.Fatalf("baseline confidence whisker missing: %s", chart.body) @@ -87,16 +116,13 @@ func TestBaselineChartShowsBaselineConfidenceWhisker(t *testing.T) { func TestToolColorsAreDistinctAndStable(t *testing.T) { seen := make(map[string]bool) for index := 0; index < 10; index++ { - color := chartColor(index, 0) + color := toolColor(index) if seen[color] { t.Fatalf("tool %d repeats color %s", index, color) } seen[color] = true - if chartColor(index, 3) != color || chartColor(index, 7) != color { - t.Fatalf("tool %d color changes with baseline", index) - } } - if chartColor(0, 0) != "#88c0d0" { + if toolColor(0) != "#88c0d0" { t.Fatal("first tool should begin with the Frost color") } } @@ -110,9 +136,9 @@ func TestChartLegendsUseToolIdentityColors(t *testing.T) { }, } legend := commandLegendChart(report).body - ranking := rankingCharts(report)[0].body + ranking := rankingCharts(report, calculateRanking(report))[0].body for index := range report.Benchmarks { - color := toolColor(report, index) + color := toolColor(index) style := `style="fill:` + color + `"` if !strings.Contains(legend, style) || !strings.Contains(ranking, style) { t.Fatalf("tool %d color %s missing from chart legend or ranking", index, color) @@ -120,12 +146,45 @@ func TestChartLegendsUseToolIdentityColors(t *testing.T) { } } +func TestRankingChartsPutBestAtTopAndWorstAtBottom(t *testing.T) { + report := model.Report{ + Config: model.Config{ + Mode: "command", Baseline: 3, + IndexDimensions: []string{"time", "cpu"}, + }, + Benchmarks: []model.Benchmark{ + benchmarkWithResourceCosts("fast", 1, 9, 300, 300, 300), + benchmarkWithResourceCosts("middle", 2, 2, 200, 200, 200), + benchmarkWithResourceCosts("efficient", 3, 1, 100, 100, 100), + }, + } + expected := map[string][]string{ + "BALANCED INDEX": {"#1 efficient", "#2 middle", "#3 fast"}, + "TIME": {"#1 fast", "#2 middle", "#3 efficient"}, + "CPU COST": {"#1 efficient", "#2 middle", "#3 fast"}, + "RAM AGGREGATE": {"#1 efficient", "#2 middle", "#3 fast"}, + "LINKED SIZE": {"#1 efficient", "#2 middle", "#3 fast"}, + } + for _, chart := range rankingCharts(report, calculateRanking(report)) { + order, ok := expected[chart.title] + if !ok { + t.Fatalf("missing expected order for %s chart", chart.title) + } + best := strings.Index(chart.body, order[0]) + middle := strings.Index(chart.body, order[1]) + worst := strings.Index(chart.body, order[2]) + if best < 0 || middle <= best || worst <= middle { + t.Fatalf("%s chart does not put best at top and worst at bottom", chart.title) + } + } +} + func TestSingleRunDistributionOmitsUndefinedConfidenceWhisker(t *testing.T) { report := model.Report{ Config: model.Config{Mode: "command", Baseline: 1}, Benchmarks: []model.Benchmark{benchmarkWithWallTimes("tool", 1)}, } - chart := absoluteDistributionChart(report, chartMetricDefinitions()[0]) + chart := absoluteDistributionChart(report, metricCatalog[metricWall]) if strings.Contains(chart.body, `stroke="#eceff4"`) { t.Fatal("one observation should not render a confidence whisker") } @@ -136,7 +195,7 @@ func TestDistributionSummaryPaintsAboveRawRuns(t *testing.T) { Config: model.Config{Mode: "command", Baseline: 1}, Benchmarks: []model.Benchmark{benchmarkWithWallTimes("tool", 1, 2)}, } - body := absoluteDistributionChart(report, chartMetricDefinitions()[0]).body + body := absoluteDistributionChart(report, metricCatalog[metricWall]).body dot := strings.Index(body, `class="run-dot"`) whisker := strings.Index(body, `class="mean-ci"`) diamond := strings.Index(body, `class="mean-diamond"`) @@ -152,9 +211,14 @@ func TestFixedTUIRankingChartUsesCPUUnits(t *testing.T) { benchmarkWithWallTimes("first", 1), benchmarkWithWallTimes("second", 1), }, } - charts := rankingCharts(report) - if len(charts) < 2 || charts[1].title != "CPU" || - !strings.Contains(charts[1].body, "%") { + charts := rankingCharts(report, calculateRanking(report)) + found := false + for _, chart := range charts { + if chart.title == "CPU" && strings.Contains(chart.body, "%") { + found = true + } + } + if !found { t.Fatal("fixed TUI primary ranking chart should show average CPU percentage") } } @@ -164,7 +228,7 @@ func TestSingleToolReportOmitsRankingCharts(t *testing.T) { Config: model.Config{Mode: "command", Baseline: 1}, Benchmarks: []model.Benchmark{benchmarkWithTrendRuns("tool")}, } - for _, chart := range reportCharts(report) { + for _, chart := range NewRenderer(report).reportCharts() { if chart.kind == "ranking" { t.Fatalf("single-tool report should not include ranking chart %q", chart.title) } @@ -181,7 +245,7 @@ func TestBalancedIndexDescriptionListsIncludedCategories(t *testing.T) { Config: model.Config{Mode: "command", Baseline: 1}, Benchmarks: benchmarks, } - description := rankingCharts(report)[0].description + description := rankingCharts(report, calculateRanking(report))[0].description for _, expected := range []string{ "Included here: wall time, CPU cost, RAM aggregate.", "score = value / best value", @@ -198,7 +262,7 @@ func TestBalancedIndexDescriptionListsIncludedCategories(t *testing.T) { }, Benchmarks: benchmarks, } - description = rankingCharts(withDisk)[0].description + description = rankingCharts(withDisk, calculateRanking(withDisk))[0].description if !strings.Contains(description, "wall time, CPU cost, RAM aggregate, linked size") { t.Fatalf("disk index missing linked size: %s", description) } @@ -211,7 +275,7 @@ func TestSingleToolReportHasMetricGroupTrendCharts(t *testing.T) { } var titles []string var combined strings.Builder - for _, chart := range reportCharts(report) { + for _, chart := range NewRenderer(report).reportCharts() { if chart.kind != "trend" { continue } @@ -242,7 +306,7 @@ func TestTrendChartsAreSectionedByTool(t *testing.T) { }, } var titles []string - for _, section := range chartSections(reportCharts(report)) { + for _, section := range chartSections(NewRenderer(report).reportCharts()) { if strings.HasPrefix(section.title, "Measurement trends") { titles = append(titles, section.title) } @@ -263,7 +327,10 @@ func TestTrendLegendValuesDoNotOverlapLabels(t *testing.T) { Benchmarks: []model.Benchmark{benchmarkWithTrendRuns("tool")}, } chart := measurementTrendChart(report, 0, chartGroup{ - name: "MEMORY COST", metrics: chartMetricDefinitions()[5:9], + name: "MEMORY COST", metrics: chartMetrics( + metricMeanResident, metricPeakResident, metricOSMaxRSS, + metricPhysicalFootprint, + ), }) if strings.Contains(chart.body, `x="704"`) { t.Fatalf("trend legend values should not use overlapping right column: %s", chart.body) @@ -289,7 +356,7 @@ func TestComparisonReportHasPerToolTrendCharts(t *testing.T) { benchmarkWithTrendRuns("second"), }, } - titles := trendChartTitles(reportCharts(report)) + titles := trendChartTitles(NewRenderer(report).reportCharts()) for _, expected := range []string{ "CPU cost trends Β· first [BASELINE]", "Memory cost trends Β· first [BASELINE]", @@ -315,8 +382,8 @@ func TestChartsHaveExplicitDescriptions(t *testing.T) { Benchmarks: []model.Benchmark{benchmarkWithWallTimes("tool", 1, 2)}, } chartSets := [][]svgChart{ - append([]svgChart{commandLegendChart(report)}, reportCharts(report)...), - reportCharts(single), + append([]svgChart{commandLegendChart(report)}, NewRenderer(report).reportCharts()...), + NewRenderer(single).reportCharts(), } for _, charts := range chartSets { for _, chart := range charts { diff --git a/internal/report/chart_tradeoff.go b/internal/report/chart_tradeoff.go index 3ab7a25..d045c7c 100644 --- a/internal/report/chart_tradeoff.go +++ b/internal/report/chart_tradeoff.go @@ -6,7 +6,6 @@ import ( "math" "strings" - "github.com/shellcell/snailrace/internal/analysis" "github.com/shellcell/snailrace/internal/model" ) @@ -16,14 +15,14 @@ type tradeoffMetric struct { value func(model.Benchmark) float64 } -func tradeoffCharts(report model.Report) []svgChart { +func tradeoffCharts(report model.Report, ranking rankingData) []svgChart { if len(report.Benchmarks) < 2 { return nil } primary := tradeoffMetric{"Time", formatDuration, func(b model.Benchmark) float64 { return b.Summary.WallSeconds.Mean }} - if report.Config.Mode == "tui" && report.Config.DurationSeconds > 0 { + if report.Config.FixedDurationTUI() { primary = tradeoffMetric{"Average CPU", formatPercent, func(b model.Benchmark) float64 { return b.Summary.AverageCPUPercent.Mean }} @@ -37,13 +36,13 @@ func tradeoffCharts(report model.Report) []svgChart { if mean <= 0 || peak <= 0 { return 0 } - return math.Sqrt(mean * peak) + return math.Sqrt(mean) * math.Sqrt(peak) }} linked := tradeoffMetric{"Linked size", formatBytes, func(b model.Benchmark) float64 { return float64(b.Tool.DiskFootprintBytes) }} pairs := [][2]tradeoffMetric{{primary, cpu}, {primary, linked}} - if analysis.Calculate(report.Config, report.Benchmarks).RAMAvailable { + if ranking.RAMAvailable { pairs = append(pairs, [2]tradeoffMetric{primary, ram}, [2]tradeoffMetric{cpu, ram}) } charts := make([]svgChart, 0, len(pairs)) @@ -57,8 +56,13 @@ func tradeoffChart(report model.Report, xMetric, yMetric tradeoffMetric) svgChar const left, top, plotWidth, plotHeight = 62, 62, 390, 240 xMaximum, yMaximum := 0.0, 0.0 for _, benchmark := range report.Benchmarks { - xMaximum = math.Max(xMaximum, xMetric.value(benchmark)) - yMaximum = math.Max(yMaximum, yMetric.value(benchmark)) + xValue, yValue := xMetric.value(benchmark), yMetric.value(benchmark) + if !finiteNumber(xValue) || !finiteNumber(yValue) || + xValue < 0 || yValue < 0 { + return svgChart{} + } + xMaximum = math.Max(xMaximum, xValue) + yMaximum = math.Max(yMaximum, yValue) } if xMaximum <= 0 { xMaximum = 1 @@ -87,7 +91,7 @@ func tradeoffChart(report model.Report, xMetric, yMetric tradeoffMetric) svgChar xValue, yValue := xMetric.value(benchmark), yMetric.value(benchmark) x := left + xValue/xMaximum*plotWidth y := top + plotHeight - yValue/yMaximum*plotHeight - color := toolColor(report, index) + color := toolColor(index) legendY := 67 + index*26 fmt.Fprintf( &body, diff --git a/internal/report/chart_trend.go b/internal/report/chart_trend.go index 5513b3c..6e7d8f7 100644 --- a/internal/report/chart_trend.go +++ b/internal/report/chart_trend.go @@ -16,11 +16,10 @@ type trendLane struct { format func(float64) string } -func measurementTrendCharts(report model.Report) []svgChart { +func measurementTrendCharts(report model.Report, groups []chartGroup) []svgChart { if len(report.Benchmarks) == 0 { return nil } - groups := chartGroups(report) charts := make([]svgChart, 0, len(report.Benchmarks)*len(groups)) for benchmarkIndex, benchmark := range report.Benchmarks { if len(benchmark.Runs) < 2 { @@ -37,7 +36,7 @@ func measurementTrendCharts(report model.Report) []svgChart { } func measurementTrendChart(report model.Report, benchmarkIndex int, group chartGroup) svgChart { - lanes := trendLanes(report, group) + lanes := trendLanes(report, benchmarkIndex, group) if len(lanes) == 0 { return svgChart{} } @@ -48,8 +47,13 @@ func measurementTrendChart(report model.Report, benchmarkIndex int, group chartG return left + float64(index)/float64(len(runs)-1)*(right-left) } var body strings.Builder + metricCount := 0 + for _, lane := range lanes { + metricCount += len(lane.metrics) + } + body.Grow(1536 + len(runs)*metricCount*96) toolLabel := reportToolLabel(report, benchmarkIndex) - toolColorHex := toolColor(report, benchmarkIndex) + toolColorHex := toolColor(benchmarkIndex) body.WriteString(svgChartStyle) fmt.Fprintf( &body, @@ -88,34 +92,53 @@ func measurementTrendChart(report model.Report, benchmarkIndex int, group chartG left, bottom+18, right, bottom+18, len(runs), ) for metricIndex, metric := range lane.metrics { - values := trendValues(runs, metric) - minimum, maximum := valueRange(values) - color := style.Tool(metric.row).Hex + minimum, maximum := trendMetricRange(runs, metric) + color := style.Tool(int(metric.id)).Hex var path strings.Builder - for index, value := range values { + path.Grow(len(runs) * 20) + started := false + for index, run := range runs { + if !chartRunAvailable(metric, run) { + started = false + continue + } + value := metric.run(run) command := "L" - if index == 0 { + if !started { command = "M" } - fmt.Fprintf(&path, "%s %.1f %.1f ", command, x(index), positionY(value)) + path.WriteString(command) + path.WriteByte(' ') + writeChartFloat(&path, x(index)) + path.WriteByte(' ') + writeChartFloat(&path, positionY(value)) + path.WriteByte(' ') + started = true } fmt.Fprintf( &body, ``, path.String(), color, ) - for index, value := range values { - fmt.Fprintf( - &body, ``, - x(index), positionY(value), color, - ) + for index, run := range runs { + if !chartRunAvailable(metric, run) { + continue + } + value := metric.run(run) + body.WriteString(``) } legendY := top + 4 + metricIndex*26 fmt.Fprintf( &body, `%s`+ `%s ... %s`, - legendY, color, html.EscapeString(clip(metric.name, 24)), legendY+12, + legendY, color, html.EscapeString(clip(metric.chartName, 24)), legendY+12, metric.format(minimum), metric.format(maximum), ) } @@ -128,17 +151,13 @@ func measurementTrendChart(report model.Report, benchmarkIndex int, group chartG } } -func trendValues(runs []model.Run, metric chartMetric) []float64 { - values := make([]float64, len(runs)) - for index, run := range runs { - values[index] = metric.run(run) - } - return values -} - -func valueRange(values []float64) (float64, float64) { +func trendMetricRange(runs []model.Run, metric chartMetric) (float64, float64) { minimum, maximum := math.Inf(1), math.Inf(-1) - for _, value := range values { + for _, run := range runs { + if !chartRunAvailable(metric, run) { + continue + } + value := metric.run(run) minimum = math.Min(minimum, value) maximum = math.Max(maximum, value) } @@ -152,6 +171,9 @@ func trendLaneRange(runs []model.Run, metrics []chartMetric) (float64, float64) minimum, maximum := math.Inf(1), math.Inf(-1) for _, metric := range metrics { for _, run := range runs { + if !chartRunAvailable(metric, run) { + continue + } value := metric.run(run) minimum = math.Min(minimum, value) maximum = math.Max(maximum, value) @@ -163,39 +185,43 @@ func trendLaneRange(runs []model.Run, metrics []chartMetric) (float64, float64) return minimum, maximum } -func trendLanes(report model.Report, group chartGroup) []trendLane { - metrics := trendMetrics(report, group.metrics) - byRow := make(map[int]chartMetric, len(metrics)) +func trendLanes(report model.Report, benchmarkIndex int, group chartGroup) []trendLane { + metrics := trendMetrics(report, benchmarkIndex, group.metrics) + byRow := make(map[metricID]chartMetric, len(metrics)) for _, metric := range metrics { - byRow[metric.row] = metric + byRow[metric.id] = metric } switch group.name { case "PERFORMANCE": - return trendLaneDefinitions(byRow, trendLaneSpec{"Wall time", []int{0}}) + return trendLaneDefinitions(byRow, trendLaneSpec{"Wall time", []metricID{metricWall}}) case "CPU COST": return trendLaneDefinitions( byRow, - trendLaneSpec{"CPU time", []int{1, 2, 3}}, - trendLaneSpec{"Average CPU", []int{4}}, + trendLaneSpec{"CPU time", []metricID{metricCPUTotal, metricCPUUser, metricCPUSystem}}, + trendLaneSpec{"Average CPU", []metricID{metricAverageCPU}}, ) case "MEMORY COST": return trendLaneDefinitions( byRow, - trendLaneSpec{"Resident memory", []int{7, 6, 5}}, - trendLaneSpec{"Physical footprint", []int{8}}, - trendLaneSpec{"Virtual memory", []int{9}}, + trendLaneSpec{"Resident memory", []metricID{ + metricOSMaxRSS, metricPeakResident, metricMeanResident, + }}, + trendLaneSpec{"Physical footprint", []metricID{metricPhysicalFootprint}}, + trendLaneSpec{"Virtual memory", []metricID{metricPeakVirtual}}, ) case "PROCESS STRUCTURE": return trendLaneDefinitions( byRow, - trendLaneSpec{"Processes and threads", []int{11, 10}}, - trendLaneSpec{"FD references", []int{12}}, + trendLaneSpec{"Processes and threads", []metricID{ + metricPeakThreads, metricPeakProcesses, + }}, + trendLaneSpec{"FD references", []metricID{metricPeakFDs}}, ) default: lanes := make([]trendLane, 0, len(metrics)) for _, metric := range metrics { lanes = append(lanes, trendLane{ - name: metric.name, metrics: []chartMetric{metric}, format: metric.format, + name: metric.chartName, metrics: []chartMetric{metric}, format: metric.format, }) } return lanes @@ -204,10 +230,10 @@ func trendLanes(report model.Report, group chartGroup) []trendLane { type trendLaneSpec struct { name string - rows []int + rows []metricID } -func trendLaneDefinitions(byRow map[int]chartMetric, specs ...trendLaneSpec) []trendLane { +func trendLaneDefinitions(byRow map[metricID]chartMetric, specs ...trendLaneSpec) []trendLane { lanes := make([]trendLane, 0, len(specs)) for _, spec := range specs { metrics := make([]chartMetric, 0, len(spec.rows)) @@ -224,10 +250,16 @@ func trendLaneDefinitions(byRow map[int]chartMetric, specs ...trendLaneSpec) []t return lanes } -func trendMetrics(report model.Report, metrics []chartMetric) []chartMetric { +func trendMetrics( + report model.Report, benchmarkIndex int, metrics []chartMetric, +) []chartMetric { result := make([]chartMetric, 0, len(metrics)) for _, metric := range metrics { - if available(metricRows[metric.row], report.Host.OS) { + stats := metric.stats(report.Benchmarks[benchmarkIndex].Summary) + if stats.N > 0 && finiteNumber(stats.Mean) && availableFor( + metric, report.Host.OS, + report.Benchmarks[benchmarkIndex].Summary, + ) { result = append(result, metric) } } @@ -282,7 +314,7 @@ func trendLaneNames(lanes []trendLane) string { func trendMetricNames(metrics []chartMetric) string { names := make([]string, 0, len(metrics)) for _, metric := range metrics { - names = append(names, metric.name) + names = append(names, metric.chartName) } return strings.Join(names, ", ") } diff --git a/internal/report/chart_types.go b/internal/report/chart_types.go index 1ffc268..4d819fc 100644 --- a/internal/report/chart_types.go +++ b/internal/report/chart_types.go @@ -3,11 +3,12 @@ package report import ( "fmt" "html" + "io" "math" + "strconv" "strings" "unicode" - "github.com/shellcell/snailrace/internal/model" "github.com/shellcell/snailrace/internal/style" ) @@ -30,13 +31,7 @@ type svgChart struct { height int } -type chartMetric struct { - name string - format func(float64) string - stats func(model.Benchmark) model.Stats - run func(model.Run) float64 - row int -} +type chartMetric = metricDefinition type chartGroup struct { name string @@ -44,26 +39,48 @@ type chartGroup struct { } func (chart svgChart) html() string { - return fmt.Sprintf( + var output strings.Builder + output.Grow(len(chart.body) + 512) + chart.writeHTML(&output) + return output.String() +} + +func (chart svgChart) writeHTML(writer io.Writer) { + fmt.Fprintf( + writer, `%s%s`, - chartWidth, chart.height, chartWidth, chart.metadata(), chart.body, + `style="max-width:%dpx;font-family:monospace">`, + chartWidth, chart.height, chartWidth, ) + chart.writeMetadata(writer) + writeChartString(writer, chart.body) + writeChartString(writer, ``) } -func (chart svgChart) embedded(x, y int) string { - return fmt.Sprintf( +func (chart svgChart) writeEmbedded(writer io.Writer, x, y int) { + fmt.Fprintf( + writer, `%s%s`, + `viewBox="0 0 %d %d">`, x, y, chartWidth, chart.height, chartWidth, chart.height, - chart.metadata(), chart.body, ) + chart.writeMetadata(writer) + writeChartString(writer, chart.body) + writeChartString(writer, ``) +} + +func writeChartString(writer io.Writer, value string) { + if stringWriter, ok := writer.(io.StringWriter); ok { + _, _ = stringWriter.WriteString(value) + return + } + _, _ = fmt.Fprint(writer, value) } -func (chart svgChart) metadata() string { - return fmt.Sprintf( - `%s%s`, +func (chart svgChart) writeMetadata(writer io.Writer) { + fmt.Fprintf( + writer, `%s%s`, html.EscapeString(chart.accessibleTitle()), html.EscapeString(chart.explanation()), ) } @@ -82,11 +99,7 @@ func (chart svgChart) explanation() string { return chart.accessibleTitle() } -func chartColor(index, _ int) string { - return style.Tool(index).Hex -} - -func toolColor(_ model.Report, index int) string { +func toolColor(index int) string { return style.Tool(index).Hex } @@ -103,7 +116,20 @@ func statusColor(class string) string { } } +func writeChartFloat(output *strings.Builder, value float64) { + buffer := make([]byte, 0, 24) + output.Write(strconv.AppendFloat(buffer, value, 'f', 1, 64)) +} + +func writeChartInt(output *strings.Builder, value int) { + buffer := make([]byte, 0, 12) + output.Write(strconv.AppendInt(buffer, int64(value), 10)) +} + func niceDeltaScale(value float64) float64 { + if math.IsNaN(value) || math.IsInf(value, 0) { + return 1 + } value = math.Max(1, value) power := math.Pow(10, math.Floor(math.Log10(value))) normalized := value / power @@ -119,6 +145,10 @@ func niceDeltaScale(value float64) float64 { } } +func finiteNumber(value float64) bool { + return !math.IsNaN(value) && !math.IsInf(value, 0) +} + func balanceCharts(charts []svgChart) ([]svgChart, []svgChart) { var left, right []svgChart leftHeight, rightHeight := 0, 0 diff --git a/internal/report/command_legend.go b/internal/report/command_legend.go index 2db6f2a..b9d2672 100644 --- a/internal/report/command_legend.go +++ b/internal/report/command_legend.go @@ -1,11 +1,14 @@ package report import ( - "strings" - "github.com/shellcell/snailrace/internal/model" + "github.com/shellcell/snailrace/internal/style" ) func fullCommand(benchmark model.Benchmark) string { - return strings.Join(benchmark.Tool.Command, " ") + command := benchmark.Tool.Command + if len(command) == 0 { + return "" + } + return style.CommandText(command, benchmark.Tool.ShellCommand) } diff --git a/internal/report/command_legend_svg.go b/internal/report/command_legend_svg.go index 7f997e4..8a18a1c 100644 --- a/internal/report/command_legend_svg.go +++ b/internal/report/command_legend_svg.go @@ -14,7 +14,7 @@ func svgCommandLegend(output *strings.Builder, report model.Report, y int) int { for index, benchmark := range report.Benchmarks { fmt.Fprintf( output, `%d. %s`, - y, toolColor(report, index), index+1, + y, toolColor(index), index+1, html.EscapeString(clip(reportToolLabel(report, index), 80)), ) y += 18 diff --git a/internal/report/command_legend_test.go b/internal/report/command_legend_test.go new file mode 100644 index 0000000..5369fc3 --- /dev/null +++ b/internal/report/command_legend_test.go @@ -0,0 +1,48 @@ +package report + +import ( + "bytes" + "strings" + "testing" + + "github.com/shellcell/snailrace/internal/model" +) + +func TestFullCommandQuotesDirectArgumentsFaithfully(t *testing.T) { + benchmark := model.Benchmark{Tool: model.ToolInfo{Command: []string{ + "tool", "two words", "it's", "", "line\nbreak", + }}} + want := `tool 'two words' 'it'"'"'s' '' "line\nbreak"` + if got := fullCommand(benchmark); got != want { + t.Fatalf("command = %q, want %q", got, want) + } +} + +func TestFullCommandQuotesSingleDirectArgument(t *testing.T) { + benchmark := model.Benchmark{Tool: model.ToolInfo{Command: []string{"/tmp/my tool"}}} + if got, want := fullCommand(benchmark), `'/tmp/my tool'`; got != want { + t.Fatalf("command = %q, want %q", got, want) + } +} + +func TestRenderedLabelsContainNoTerminalControls(t *testing.T) { + run := model.Run{WallSeconds: 1, StopReason: "exited"} + report := model.Report{Verbose: true, Config: model.Config{Baseline: 1}, + Benchmarks: []model.Benchmark{{ + Tool: model.ToolInfo{Name: "safe\n\x1b[31munsafe", Command: []string{"true"}}, + Runs: []model.Run{run}, Summary: model.Summarize([]model.Run{run}), + }}} + for _, format := range []string{"text", "markdown", "html", "svg"} { + var output bytes.Buffer + if err := NewRenderer(report).Write(&output, format); err != nil { + t.Fatalf("%s: %v", format, err) + } + if strings.ContainsAny(output.String(), "\n\x1b") && + strings.Contains(output.String(), "safe\n\x1b") { + t.Fatalf("%s retained label control characters", format) + } + if !strings.Contains(output.String(), `safe\n\x1b[31munsafe`) { + t.Fatalf("%s does not contain readable escaped label: %q", format, output.String()) + } + } +} diff --git a/internal/report/comparison.go b/internal/report/comparison.go index f53898d..b943122 100644 --- a/internal/report/comparison.go +++ b/internal/report/comparison.go @@ -17,6 +17,8 @@ const ( type deltaResult struct { percent float64 percentAvailable bool + baselineMean float64 + candidateMean float64 difference model.Stats status string class string @@ -33,17 +35,32 @@ func baselineIndex(report model.Report) int { func compareMetric( baseline model.Benchmark, candidate model.Benchmark, - row metricRow, + row metricDefinition, intervalSeconds float64, ) deltaResult { - baseMean := row.stats(baseline.Summary).Mean - candidateMean := row.stats(candidate.Summary).Mean - result := deltaResult{} + return compareMetricWithBaselineIndex( + baseline, indexMetricValues(baseline.Runs, row), candidate, row, intervalSeconds, + ) +} + +func compareMetricWithBaselineIndex( + baseline model.Benchmark, + baselineValues map[int]float64, + candidate model.Benchmark, + row metricDefinition, + intervalSeconds float64, +) deltaResult { + baseMean, candidateMean, differences := pairedDifferences( + baselineValues, candidate.Runs, row, + ) + result := deltaResult{baselineMean: baseMean, candidateMean: candidateMean} if baseMean != 0 { - result.percent = (candidateMean/baseMean - 1) * 100 - result.percentAvailable = true + percent := (candidateMean/baseMean - 1) * 100 + if finiteNumber(percent) { + result.percent = percent + result.percentAvailable = true + } } - differences := pairedDifferences(baseline.Runs, candidate.Runs, row.run) result.difference = model.CalculateStats(differences) result.status, result.class = comparisonStatus(result.difference, row.direction) if row.sampled && !samplingReliable(baseline, candidate, intervalSeconds) { @@ -52,6 +69,20 @@ func compareMetric( return result } +func indexMetricValues(runs []model.Run, row metricDefinition) map[int]float64 { + values := make(map[int]float64, len(runs)) + for _, run := range runs { + if row.id == metricPhysicalFootprint && !run.PhysicalFootprintValid { + continue + } + value := row.run(run) + if finiteNumber(value) { + values[run.Index] = value + } + } + return values +} + func samplingReliable( baseline, candidate model.Benchmark, intervalSeconds float64, @@ -60,21 +91,7 @@ func samplingReliable( return true } for _, benchmark := range []model.Benchmark{baseline, candidate} { - if !benchmarkSamplesReliable(benchmark, intervalSeconds) { - return false - } - } - return true -} - -func benchmarkSamplesReliable(benchmark model.Benchmark, intervalSeconds float64) bool { - if intervalSeconds <= 0 { - return true - } - minimum := intervalSeconds * 2 - for _, run := range benchmark.Runs { - if run.WallSeconds < minimum || run.SampleCount < 2 || - run.SampleCoverageSeconds < intervalSeconds { + if !model.SamplingReliable(benchmark, intervalSeconds) { return false } } @@ -82,21 +99,39 @@ func benchmarkSamplesReliable(benchmark model.Benchmark, intervalSeconds float64 } func pairedDifferences( - baseline, candidate []model.Run, - pick func(model.Run) float64, -) []float64 { - baselineByIndex := make(map[int]float64, len(baseline)) - for _, run := range baseline { - baselineByIndex[run.Index] = pick(run) - } + baseline map[int]float64, + candidate []model.Run, + row metricDefinition, +) (float64, float64, []float64) { differences := make([]float64, 0, len(candidate)) + baseMean, candidateMean := 0.0, 0.0 + count := 0 for _, run := range candidate { - value, ok := baselineByIndex[run.Index] + if row.id == metricPhysicalFootprint && !run.PhysicalFootprintValid { + continue + } + value, ok := baseline[run.Index] if ok { - differences = append(differences, pick(run)-value) + candidateValue := row.run(run) + if !finiteNumber(candidateValue) { + continue + } + count++ + baseMean = runningMean(baseMean, value, count) + candidateMean = runningMean(candidateMean, candidateValue, count) + differences = append(differences, candidateValue-value) } } - return differences + return baseMean, candidateMean, differences +} + +func runningMean(mean, value float64, count int) float64 { + n := float64(count) + mean = mean*(1-1/n) + value/n + if math.IsInf(mean, 0) { + return math.Copysign(math.MaxFloat64, mean) + } + return mean } func comparisonStatus(stats model.Stats, direction metricDirection) (string, string) { @@ -124,7 +159,7 @@ func comparisonStatus(stats model.Stats, direction metricDirection) (string, str return "inconclusive", "uncertain" } -func formatDelta(delta deltaResult, row metricRow) string { +func formatDelta(delta deltaResult) string { change := "Ξ” n/a" if delta.percentAvailable { change = "Ξ” " + formatSignedPercent(delta.percent) @@ -132,7 +167,7 @@ func formatDelta(delta deltaResult, row metricRow) string { return change + " Β· " + delta.status } -func formatDeltaInterval(delta deltaResult, row metricRow) string { +func formatDeltaInterval(delta deltaResult, row metricDefinition) string { if !delta.difference.CI95Valid { return "insufficient paired runs" } diff --git a/internal/report/comparison_html.go b/internal/report/comparison_html.go index 6635780..14adbba 100644 --- a/internal/report/comparison_html.go +++ b/internal/report/comparison_html.go @@ -8,7 +8,8 @@ import ( "github.com/shellcell/snailrace/internal/model" ) -func writeHTMLComparison(writer io.Writer, report model.Report) { +func writeHTMLComparison(writer io.Writer, renderer *Renderer) { + report := renderer.displayReport baselinePosition := baselineIndex(report) baseline := report.Benchmarks[baselinePosition] fmt.Fprintf( @@ -35,10 +36,10 @@ func writeHTMLComparison(writer io.Writer, report model.Report) { `MetricBaseline mean`+ `Candidate meanΞ” mean`) writeHTMLStaticFootprint(writer, baseline, candidate) - for _, row := range metricRows { + for _, row := range metricCatalog { writeHTMLComparisonRow( - writer, report.Host.OS, report.Config.IntervalMS/1000, - baseline, candidate, row, + writer, report.Host.OS, baseline, candidate, row, + renderer.comparison(index, row.id), ) } fmt.Fprint(writer, "") @@ -72,23 +73,22 @@ func writeHTMLStaticFootprint( func writeHTMLComparisonRow( writer io.Writer, operatingSystem string, - intervalSeconds float64, baseline, candidate model.Benchmark, - row metricRow, + row metricDefinition, + delta deltaResult, ) { - if !available(row, operatingSystem) { + if !availableFor(row, operatingSystem, baseline.Summary, candidate.Summary) { fmt.Fprintf(writer, "%sN/A", row.name) return } - delta := compareMetric(baseline, candidate, row, intervalSeconds) fmt.Fprintf( writer, `%s%s%s`+ `%s`+ `%s`, - row.name, row.format(row.stats(baseline.Summary).Mean), delta.class, - row.format(row.stats(candidate.Summary).Mean), delta.class, - html.EscapeString(formatDelta(delta, row)), + row.name, row.format(delta.baselineMean), delta.class, + row.format(delta.candidateMean), delta.class, + html.EscapeString(formatDelta(delta)), html.EscapeString(formatDeltaInterval(delta, row)), ) } diff --git a/internal/report/comparison_markdown.go b/internal/report/comparison_markdown.go index 5da5f28..e22727a 100644 --- a/internal/report/comparison_markdown.go +++ b/internal/report/comparison_markdown.go @@ -3,11 +3,10 @@ package report import ( "fmt" "io" - - "github.com/shellcell/snailrace/internal/model" ) -func writeMarkdownComparison(writer io.Writer, report model.Report) { +func writeMarkdownComparison(writer io.Writer, renderer *Renderer) { + report := renderer.displayReport baselinePosition := baselineIndex(report) baseline := report.Benchmarks[baselinePosition] fmt.Fprintf( @@ -35,19 +34,18 @@ func writeMarkdownComparison(writer io.Writer, report model.Report) { formatBytes(float64(baseline.Tool.DiskFootprintBytes)), formatBytes(float64(candidate.Tool.DiskFootprintBytes)), staticDelta, ) - for _, row := range metricRows { - if !available(row, report.Host.OS) { + for _, row := range metricCatalog { + if !availableFor( + row, report.Host.OS, baseline.Summary, candidate.Summary, + ) { fmt.Fprintf(writer, "| %s | N/A | N/A | N/A | N/A |\n", row.name) continue } - delta := compareMetric( - baseline, candidate, row, report.Config.IntervalMS/1000, - ) + delta := renderer.comparison(index, row.id) fmt.Fprintf( writer, "| %s | %s | %s | %s | %s |\n", row.name, - row.format(row.stats(baseline.Summary).Mean), - row.format(row.stats(candidate.Summary).Mean), - formatDelta(delta, row), formatDeltaInterval(delta, row), + row.format(delta.baselineMean), row.format(delta.candidateMean), + formatDelta(delta), formatDeltaInterval(delta, row), ) } fmt.Fprintln(writer) diff --git a/internal/report/comparison_test.go b/internal/report/comparison_test.go index cb1f439..e71a84d 100644 --- a/internal/report/comparison_test.go +++ b/internal/report/comparison_test.go @@ -8,7 +8,7 @@ import ( ) func TestPairedComparisonRequiresConfidence(t *testing.T) { - row := metricRows[0] + row := metricCatalog[metricWall] baseline := benchmarkWithWallTimes("baseline", 10, 10, 10) better := benchmarkWithWallTimes("better", 8, 8, 8) delta := compareMetric(baseline, better, row, 0) @@ -27,7 +27,7 @@ func TestSinglePairIsInconclusive(t *testing.T) { delta := compareMetric( benchmarkWithWallTimes("baseline", 10), benchmarkWithWallTimes("candidate", 5), - metricRows[0], 0, + metricCatalog[metricWall], 0, ) if delta.status != "inconclusive" { t.Fatalf("status = %q, want inconclusive", delta.status) @@ -38,9 +38,9 @@ func TestZeroBaselineDeltaIsUnavailable(t *testing.T) { delta := compareMetric( benchmarkWithWallTimes("baseline", 0, 0), benchmarkWithWallTimes("candidate", 1, 1), - metricRows[0], 0, + metricCatalog[metricWall], 0, ) - if got := formatDelta(delta, metricRows[0]); got != "Ξ” n/a Β· worse" { + if got := formatDelta(delta); got != "Ξ” n/a Β· worse" { t.Fatalf("delta = %q, want unavailable percentage", got) } if got, _ := compareStaticCost(0, 100); got != "Ξ” n/a Β· worse" { @@ -57,7 +57,7 @@ func TestShortRunsMarkSampledMetricsAsLimited(t *testing.T) { } baseline.Summary = model.Summarize(baseline.Runs) candidate.Summary = model.Summarize(candidate.Runs) - delta := compareMetric(baseline, candidate, metricRows[5], 0.01) + delta := compareMetric(baseline, candidate, metricCatalog[metricMeanResident], 0.01) if delta.status != "sampling-limited" { t.Fatalf("status = %q, want sampling-limited", delta.status) } @@ -74,12 +74,46 @@ func TestTooFewValidSamplesMarksMetricsAsLimited(t *testing.T) { } baseline.Summary = model.Summarize(baseline.Runs) candidate.Summary = model.Summarize(candidate.Runs) - delta := compareMetric(baseline, candidate, metricRows[5], 0.01) + delta := compareMetric(baseline, candidate, metricCatalog[metricMeanResident], 0.01) if delta.status != "sampling-limited" { t.Fatalf("status = %q, want sampling-limited", delta.status) } } +func TestPhysicalFootprintComparisonExcludesInvalidPairs(t *testing.T) { + baselineRuns := []model.Run{ + {Index: 1, PeakPhysicalFootprintBytes: 100, PhysicalFootprintValid: true}, + {Index: 2, PeakPhysicalFootprintBytes: 1000}, + } + candidateRuns := []model.Run{ + {Index: 1, PeakPhysicalFootprintBytes: 90, PhysicalFootprintValid: true}, + {Index: 2, PeakPhysicalFootprintBytes: 1}, + } + baseline := model.Benchmark{Runs: baselineRuns, Summary: model.Summarize(baselineRuns)} + candidate := model.Benchmark{Runs: candidateRuns, Summary: model.Summarize(candidateRuns)} + result := compareMetric(baseline, candidate, metricCatalog[metricPhysicalFootprint], 0) + if result.difference.N != 1 || result.difference.Mean != -10 { + t.Fatalf("physical footprint difference = %+v", result.difference) + } + if result.baselineMean != 100 || result.candidateMean != 90 || + math.Abs(result.percent+10) > 1e-9 { + t.Fatalf("paired physical footprint means = %+v", result) + } +} + +func TestComparisonDropsPairWithNonFiniteValue(t *testing.T) { + baselineRuns := []model.Run{{Index: 1, WallSeconds: 1}} + candidateRuns := []model.Run{{Index: 1, WallSeconds: math.NaN()}} + result := compareMetric( + model.Benchmark{Runs: baselineRuns, Summary: model.Summarize(baselineRuns)}, + model.Benchmark{Runs: candidateRuns, Summary: model.Summarize(candidateRuns)}, + metricCatalog[metricWall], 0, + ) + if result.difference.N != 0 || result.percentAvailable { + t.Fatalf("non-finite pair produced a comparison: %+v", result) + } +} + func benchmarkWithWallTimes(name string, values ...float64) model.Benchmark { runs := make([]model.Run, len(values)) for index, value := range values { diff --git a/internal/report/comparison_text.go b/internal/report/comparison_text.go index 8580bc3..108ec46 100644 --- a/internal/report/comparison_text.go +++ b/internal/report/comparison_text.go @@ -3,11 +3,10 @@ package report import ( "fmt" "io" - - "github.com/shellcell/snailrace/internal/model" ) -func writeTextComparison(writer io.Writer, report model.Report) { +func writeTextComparison(writer io.Writer, renderer *Renderer) { + report := renderer.displayReport baselinePosition := baselineIndex(report) baseline := report.Benchmarks[baselinePosition] for index, candidate := range report.Benchmarks { @@ -31,19 +30,18 @@ func writeTextComparison(writer io.Writer, report model.Report) { formatBytes(float64(baseline.Tool.DiskFootprintBytes)), formatBytes(float64(candidate.Tool.DiskFootprintBytes)), staticDelta, ) - for _, row := range metricRows { - if !available(row, report.Host.OS) { + for _, row := range metricCatalog { + if !availableFor( + row, report.Host.OS, baseline.Summary, candidate.Summary, + ) { fmt.Fprintf(writer, "%s\tN/A\tN/A\tN/A\tN/A\n", row.name) continue } - delta := compareMetric( - baseline, candidate, row, report.Config.IntervalMS/1000, - ) + delta := renderer.comparison(index, row.id) fmt.Fprintf( writer, "%s\t%s\t%s\t%s\t%s\n", row.name, - row.format(row.stats(baseline.Summary).Mean), - row.format(row.stats(candidate.Summary).Mean), - formatDelta(delta, row), formatDeltaInterval(delta, row), + row.format(delta.baselineMean), row.format(delta.candidateMean), + formatDelta(delta), formatDeltaInterval(delta, row), ) } fmt.Fprintln(writer) diff --git a/internal/report/error_writer.go b/internal/report/error_writer.go new file mode 100644 index 0000000..d005df5 --- /dev/null +++ b/internal/report/error_writer.go @@ -0,0 +1,28 @@ +package report + +import "io" + +type errorWriter struct { + writer io.Writer + err error +} + +func newErrorWriter(writer io.Writer) *errorWriter { + if checked, ok := writer.(*errorWriter); ok { + return checked + } + return &errorWriter{writer: writer} +} + +func (writer *errorWriter) Write(value []byte) (int, error) { + if writer.err != nil { + return 0, writer.err + } + written, err := writer.writer.Write(value) + if err != nil { + writer.err = err + } + return written, err +} + +func (writer *errorWriter) Err() error { return writer.err } diff --git a/internal/report/error_writer_test.go b/internal/report/error_writer_test.go new file mode 100644 index 0000000..d2ddb1c --- /dev/null +++ b/internal/report/error_writer_test.go @@ -0,0 +1,42 @@ +package report + +import ( + "errors" + "testing" + + "github.com/shellcell/snailrace/internal/model" +) + +var errWriteLimit = errors.New("write limit reached") + +type limitedWriter struct { + remaining int +} + +func (writer *limitedWriter) Write(value []byte) (int, error) { + if writer.remaining <= 0 { + return 0, errWriteLimit + } + if len(value) > writer.remaining { + written := writer.remaining + writer.remaining = 0 + return written, errWriteLimit + } + writer.remaining -= len(value) + return len(value), nil +} + +func TestReportFormatsPropagateWriterErrors(t *testing.T) { + report := model.Report{ + Config: model.Config{Mode: "command", Baseline: 1}, + Benchmarks: []model.Benchmark{benchmarkWithWallTimes("tool", 1, 2)}, + } + for _, format := range []string{"text", "html", "markdown", "svg", "json"} { + t.Run(format, func(t *testing.T) { + err := NewRenderer(report).Write(&limitedWriter{}, format) + if !errors.Is(err, errWriteLimit) { + t.Fatalf("error = %v, want write failure", err) + } + }) + } +} diff --git a/internal/report/failures.go b/internal/report/failures.go new file mode 100644 index 0000000..0ed9ade --- /dev/null +++ b/internal/report/failures.go @@ -0,0 +1,125 @@ +package report + +import ( + "fmt" + "html" + "io" + "sort" + "strconv" + "strings" + + "github.com/shellcell/snailrace/internal/model" +) + +type benchmarkFailure struct { + benchmark int + failed int + total int + exitCodes []int +} + +func benchmarkFailures(report model.Report) []benchmarkFailure { + var failures []benchmarkFailure + for index, benchmark := range report.Benchmarks { + failed := benchmark.FailedRunCount() + if failed == 0 { + continue + } + seen := make(map[int]bool) + for _, run := range benchmark.Runs { + if run.Failed() { + seen[run.ExitCode] = true + } + } + codes := make([]int, 0, len(seen)) + for code := range seen { + codes = append(codes, code) + } + sort.Ints(codes) + failures = append(failures, benchmarkFailure{ + benchmark: index, failed: failed, total: len(benchmark.Runs), exitCodes: codes, + }) + } + return failures +} + +func failureExitCodes(failure benchmarkFailure) string { + values := make([]string, len(failure.exitCodes)) + for index, code := range failure.exitCodes { + values[index] = strconv.Itoa(code) + } + return strings.Join(values, ", ") +} + +func writeTextFailures(writer io.Writer, report model.Report) { + failures := benchmarkFailures(report) + for _, failure := range failures { + fmt.Fprintf( + writer, "FAILED %s: %d/%d measured runs exited non-zero (codes %s); excluded from rankings\n", + report.Benchmarks[failure.benchmark].Tool.Name, + failure.failed, failure.total, failureExitCodes(failure), + ) + } + if len(failures) > 0 { + fmt.Fprintln(writer) + } +} + +func writeHTMLFailures(writer io.Writer, report model.Report) { + failures := benchmarkFailures(report) + if len(failures) == 0 { + return + } + fmt.Fprint(writer, `

Command failures

`) + for _, failure := range failures { + fmt.Fprintf( + writer, `

%s: %d/%d measured runs exited `+ + `non-zero (codes %s); excluded from rankings.

`, + htmlToolLabel(report, failure.benchmark, true), + failure.failed, failure.total, html.EscapeString(failureExitCodes(failure)), + ) + } + fmt.Fprint(writer, `
`) +} + +func writeMarkdownFailures(writer io.Writer, report model.Report) { + failures := benchmarkFailures(report) + if len(failures) == 0 { + return + } + fmt.Fprintln(writer, "## Command Failures") + for _, failure := range failures { + fmt.Fprintf( + writer, "\n- **%s**: %d/%d measured runs exited non-zero (codes %s); excluded from rankings.\n", + escapeMarkdown(report.Benchmarks[failure.benchmark].Tool.Name), + failure.failed, failure.total, failureExitCodes(failure), + ) + } + fmt.Fprintln(writer) +} + +func failureChart(report model.Report) svgChart { + failures := benchmarkFailures(report) + if len(failures) == 0 { + return svgChart{} + } + height := 54 + len(failures)*24 + var body strings.Builder + body.WriteString(svgChartStyle) + body.WriteString(``) + body.WriteString(`COMMAND FAILURES`) + for index, failure := range failures { + fmt.Fprintf( + &body, `%s: `+ + `%d/%d runs, codes %s; excluded from rankings`, + 50+index*24, + html.EscapeString(clip(report.Benchmarks[failure.benchmark].Tool.Name, 28)), + failure.failed, failure.total, html.EscapeString(failureExitCodes(failure)), + ) + } + return svgChart{ + kind: "failure", title: "Command failures", slug: "command-failures", + description: "Lists commands with non-zero measured runs that were excluded from rankings.", + body: body.String(), height: height, + } +} diff --git a/internal/report/format.go b/internal/report/format.go index 6a19e4b..acfce64 100644 --- a/internal/report/format.go +++ b/internal/report/format.go @@ -8,8 +8,29 @@ import ( "github.com/shellcell/snailrace/internal/model" ) -type metricRow struct { +type metricID uint8 + +const ( + metricWall metricID = iota + metricCPUTotal + metricCPUUser + metricCPUSystem + metricAverageCPU + metricMeanResident + metricPeakResident + metricOSMaxRSS + metricPhysicalFootprint + metricPeakVirtual + metricPeakProcesses + metricPeakThreads + metricPeakFDs + metricCount +) + +type metricDefinition struct { + id metricID name string + chartName string stats func(model.Summary) model.Stats format func(float64) string unavailableDarwin bool @@ -19,32 +40,32 @@ type metricRow struct { sampled bool } -var metricRows = []metricRow{ - {"Wall time", func(s model.Summary) model.Stats { return s.WallSeconds }, formatDuration, false, false, +var metricCatalog = [...]metricDefinition{ + {metricWall, "Wall time", "Wall time", func(s model.Summary) model.Stats { return s.WallSeconds }, formatDuration, false, false, func(r model.Run) float64 { return r.WallSeconds }, lowerIsBetter, false}, - {"CPU total", func(s model.Summary) model.Stats { return s.CPUTotalSeconds }, formatDuration, false, false, + {metricCPUTotal, "CPU total", "Total CPU", func(s model.Summary) model.Stats { return s.CPUTotalSeconds }, formatDuration, false, false, func(r model.Run) float64 { return r.CPUUserSeconds + r.CPUSystemSeconds }, lowerIsBetter, false}, - {"CPU user", func(s model.Summary) model.Stats { return s.CPUUserSeconds }, formatDuration, false, false, + {metricCPUUser, "CPU user", "User CPU", func(s model.Summary) model.Stats { return s.CPUUserSeconds }, formatDuration, false, false, func(r model.Run) float64 { return r.CPUUserSeconds }, lowerIsBetter, false}, - {"CPU system", func(s model.Summary) model.Stats { return s.CPUSystemSeconds }, formatDuration, false, false, + {metricCPUSystem, "CPU system", "System CPU", func(s model.Summary) model.Stats { return s.CPUSystemSeconds }, formatDuration, false, false, func(r model.Run) float64 { return r.CPUSystemSeconds }, lowerIsBetter, false}, - {"Average CPU", func(s model.Summary) model.Stats { return s.AverageCPUPercent }, formatPercent, false, false, + {metricAverageCPU, "Average CPU", "Average CPU", func(s model.Summary) model.Stats { return s.AverageCPUPercent }, formatPercent, false, false, func(r model.Run) float64 { return r.AverageCPUPercent }, neutralDirection, false}, - {"Mean resident", func(s model.Summary) model.Stats { return s.MeanResidentBytes }, formatBytes, false, false, + {metricMeanResident, "Mean resident", "Mean RSS", func(s model.Summary) model.Stats { return s.MeanResidentBytes }, formatBytes, false, false, func(r model.Run) float64 { return r.MeanResidentBytes }, lowerIsBetter, true}, - {"Peak resident", func(s model.Summary) model.Stats { return s.PeakResidentBytes }, formatBytes, false, false, + {metricPeakResident, "Peak resident", "Peak RSS", func(s model.Summary) model.Stats { return s.PeakResidentBytes }, formatBytes, false, false, func(r model.Run) float64 { return r.PeakResidentBytes }, lowerIsBetter, true}, - {"OS-reported max RSS", func(s model.Summary) model.Stats { return s.OSMaxRSSBytes }, formatBytes, false, false, + {metricOSMaxRSS, "OS-reported max RSS", "OS-reported max RSS", func(s model.Summary) model.Stats { return s.OSMaxRSSBytes }, formatBytes, false, false, func(r model.Run) float64 { return r.OSMaxRSSBytes }, lowerIsBetter, false}, - {"Peak physical footprint", func(s model.Summary) model.Stats { return s.PhysicalFootprintStats() }, formatBytes, false, true, + {metricPhysicalFootprint, "Peak physical footprint", "Peak physical footprint", func(s model.Summary) model.Stats { return s.PhysicalFootprintStats() }, formatBytes, false, true, func(r model.Run) float64 { return r.PeakPhysicalFootprintBytes }, lowerIsBetter, true}, - {"Peak virtual", func(s model.Summary) model.Stats { return s.PeakVirtualBytes }, formatBytes, false, false, + {metricPeakVirtual, "Peak virtual", "Peak virtual memory", func(s model.Summary) model.Stats { return s.PeakVirtualBytes }, formatBytes, false, false, func(r model.Run) float64 { return r.PeakVirtualBytes }, lowerIsBetter, true}, - {"Peak processes", func(s model.Summary) model.Stats { return s.PeakProcesses }, formatCount, false, false, + {metricPeakProcesses, "Peak processes", "Peak processes", func(s model.Summary) model.Stats { return s.PeakProcesses }, formatCount, false, false, func(r model.Run) float64 { return r.PeakProcesses }, neutralDirection, true}, - {"Peak threads", func(s model.Summary) model.Stats { return s.PeakThreads }, formatCount, false, false, + {metricPeakThreads, "Peak threads", "Peak threads", func(s model.Summary) model.Stats { return s.PeakThreads }, formatCount, false, false, func(r model.Run) float64 { return r.PeakThreads }, neutralDirection, true}, - {"Peak FD references", func(s model.Summary) model.Stats { return s.PeakFileDescriptors }, formatCount, true, false, + {metricPeakFDs, "Peak FD references", "Peak FD references", func(s model.Summary) model.Stats { return s.PeakFileDescriptors }, formatCount, true, false, func(r model.Run) float64 { return r.PeakFileDescriptors }, lowerIsBetter, true}, } @@ -92,6 +113,9 @@ func formatSignedPercent(value float64) string { } func formatNumber(value float64) string { + if math.IsNaN(value) || math.IsInf(value, 0) { + return "N/A" + } if value == 0 { return "0" } @@ -115,7 +139,24 @@ func formatNumber(value float64) string { return result } -func available(row metricRow, operatingSystem string) bool { +func available(row metricDefinition, operatingSystem string) bool { return (operatingSystem != "darwin" || !row.unavailableDarwin) && (!row.darwinOnly || operatingSystem == "darwin") } + +func availableFor( + row metricDefinition, operatingSystem string, summaries ...model.Summary, +) bool { + if !available(row, operatingSystem) { + return false + } + if !row.darwinOnly { + return true + } + for _, summary := range summaries { + if row.stats(summary).N == 0 { + return false + } + } + return true +} diff --git a/internal/report/formats.go b/internal/report/formats.go new file mode 100644 index 0000000..649ab06 --- /dev/null +++ b/internal/report/formats.go @@ -0,0 +1,44 @@ +package report + +import "strings" + +type Format string + +const ( + FormatHTML Format = "html" + FormatSVG Format = "svg" + FormatMarkdown Format = "markdown" + FormatJSON Format = "json" + FormatText Format = "text" +) + +type FormatInfo struct { + Name Format + Extension string + NeedsCharts bool +} + +var formatCatalog = map[Format]FormatInfo{ + FormatHTML: {Name: FormatHTML, Extension: "html"}, + FormatSVG: {Name: FormatSVG, Extension: "svg", NeedsCharts: true}, + FormatMarkdown: {Name: FormatMarkdown, Extension: "md", NeedsCharts: true}, + FormatJSON: {Name: FormatJSON, Extension: "json"}, + FormatText: {Name: FormatText, Extension: "txt"}, +} + +func LookupFormat(value string) (FormatInfo, bool) { + value = strings.ToLower(strings.TrimSpace(value)) + switch value { + case "md": + value = string(FormatMarkdown) + case "txt": + value = string(FormatText) + } + info, ok := formatCatalog[Format(value)] + return info, ok +} + +func ValidFormat(format string) bool { + _, ok := LookupFormat(format) + return ok +} diff --git a/internal/report/formats_test.go b/internal/report/formats_test.go new file mode 100644 index 0000000..cd28260 --- /dev/null +++ b/internal/report/formats_test.go @@ -0,0 +1,29 @@ +package report + +import "testing" + +func TestFormatCatalogCanonicalizesAliasesAndMetadata(t *testing.T) { + tests := []struct { + input string + name Format + extension string + charts bool + }{ + {"HTML", FormatHTML, "html", false}, + {"svg", FormatSVG, "svg", true}, + {"md", FormatMarkdown, "md", true}, + {"markdown", FormatMarkdown, "md", true}, + {"json", FormatJSON, "json", false}, + {"txt", FormatText, "txt", false}, + } + for _, test := range tests { + info, ok := LookupFormat(test.input) + if !ok || info.Name != test.name || info.Extension != test.extension || + info.NeedsCharts != test.charts { + t.Fatalf("format %q = %+v, valid=%v", test.input, info, ok) + } + } + if ValidFormat("unknown") { + t.Fatal("unknown format should be invalid") + } +} diff --git a/internal/report/html.go b/internal/report/html.go index 4c43814..42ac5d6 100644 --- a/internal/report/html.go +++ b/internal/report/html.go @@ -4,7 +4,6 @@ import ( "fmt" "html" "io" - "strings" "github.com/shellcell/snailrace/internal/model" ) @@ -58,7 +57,10 @@ th:first-child,td:first-child{text-align:left}code{color:var(--accent);white-spa .command-item{grid-template-columns:1fr}}
` -func writeHTML(writer io.Writer, report model.Report) error { +func writeHTML(writer io.Writer, renderer *Renderer) error { + report := renderer.displayReport + checked := newErrorWriter(writer) + writer = checked fmt.Fprint(writer, htmlStart) fmt.Fprintf( writer, @@ -67,16 +69,27 @@ func writeHTML(writer io.Writer, report model.Report) error { html.EscapeString(report.Host.OS), html.EscapeString(report.Host.Architecture), ) writeCards(writer, report) + fmt.Fprintf( + writer, `

Balanced index dimensions actually included: %s.

`, + html.EscapeString(renderer.dimensionText), + ) + for _, caveat := range renderer.caveats { + fmt.Fprintf( + writer, `

Reliability: %s.

`, + html.EscapeString(caveat), + ) + } writeHTMLCommandLegend(writer, report) + writeHTMLFailures(writer, report) if len(report.Benchmarks) > 1 { - writeHTMLRanking(writer, report) + writeHTMLRanking(writer, report, renderer.ranking) } - charts := reportCharts(report) + charts := renderer.reportCharts() for _, section := range chartSections(charts) { writeHTMLChartSection(writer, section.title, section.charts) } if len(report.Benchmarks) > 1 { - writeHTMLComparison(writer, report) + writeHTMLComparison(writer, renderer) } fmt.Fprint(writer, "

Statistical detail

") for index, benchmark := range report.Benchmarks { @@ -89,8 +102,8 @@ func writeHTML(writer io.Writer, report model.Report) error { for _, note := range report.Notes { fmt.Fprintf(writer, "

Note: %s

", html.EscapeString(note)) } - _, err := fmt.Fprint(writer, "
") - return err + fmt.Fprint(writer, "") + return checked.Err() } type chartSection struct { @@ -165,9 +178,11 @@ func writeHTMLChartColumn(writer io.Writer, charts []svgChart) { `
`+ ``+ - `%s
`, - label, explanation, explanation, chart.html(), + ``, + label, explanation, explanation, ) + chart.writeHTML(writer) + fmt.Fprint(writer, ``) } fmt.Fprint(writer, "") } @@ -209,7 +224,7 @@ func writeHTMLBenchmark( } fmt.Fprintf( writer, `
%s
`, - html.EscapeString(strings.Join(benchmark.Tool.Command, " ")), + html.EscapeString(fullCommand(benchmark)), ) fmt.Fprintf( writer, @@ -227,7 +242,7 @@ func writeHTMLBenchmark( formatCount(benchmark.Summary.ValidSampleCount.Mean), formatDuration(benchmark.Summary.SampleCoverageSeconds.Mean), ) - if !benchmarkSamplesReliable(benchmark, report.Config.IntervalMS/1000) { + if !model.SamplingReliable(benchmark, report.Config.IntervalMS/1000) { fmt.Fprint( writer, `

Sampling quality: LIMITED `+ @@ -237,7 +252,7 @@ func writeHTMLBenchmark( fmt.Fprint(writer, `

`+ ``+ ``) - for _, row := range metricRows { + for _, row := range metricCatalog { writeHTMLRow(writer, operatingSystem, row, benchmark.Summary) } fmt.Fprint(writer, "
MetricMean Β± Οƒ95% CI meanMedianP95Range
") @@ -251,10 +266,10 @@ func writeHTMLBenchmark( func writeHTMLRow( writer io.Writer, operatingSystem string, - row metricRow, + row metricDefinition, summary model.Summary, ) { - if !available(row, operatingSystem) { + if !availableFor(row, operatingSystem, summary) { fmt.Fprintf(writer, "%sN/A", row.name) return } diff --git a/internal/report/labels_html.go b/internal/report/labels_html.go index a13a7d3..55d46be 100644 --- a/internal/report/labels_html.go +++ b/internal/report/labels_html.go @@ -10,7 +10,7 @@ import ( func htmlToolLabel(report model.Report, index int, badge bool) string { label := fmt.Sprintf( `%s`, - toolColor(report, index), + toolColor(index), html.EscapeString(report.Benchmarks[index].Tool.Name), ) if badge && index == baselineIndex(report) { diff --git a/internal/report/markdown.go b/internal/report/markdown.go index 16154e4..cc3dae1 100644 --- a/internal/report/markdown.go +++ b/internal/report/markdown.go @@ -8,16 +8,14 @@ import ( "github.com/shellcell/snailrace/internal/model" ) -func writeMarkdown(writer io.Writer, report model.Report) error { - return WriteMarkdownWithCharts(writer, report, nil, "") -} - -func WriteMarkdownWithCharts( +func (renderer *Renderer) WriteMarkdownWithCharts( writer io.Writer, - report model.Report, charts []ChartArtifact, chartDirectory string, ) error { + report := renderer.displayReport + checked := newErrorWriter(writer) + writer = checked fmt.Fprintf( writer, "# 🐌 Snailrace Report\n\nMeasured %s on `%s/%s`. "+ @@ -26,13 +24,21 @@ func WriteMarkdownWithCharts( report.Host.OS, report.Host.Architecture, report.Config.Runs, report.Config.Warmups, report.Config.Mode, ) + fmt.Fprintf( + writer, "Balanced index dimensions actually included: **%s**.\n\n", + escapeMarkdown(renderer.dimensionText), + ) + for _, caveat := range renderer.caveats { + fmt.Fprintf(writer, "> Reliability: %s.\n\n", escapeMarkdown(caveat)) + } writeMarkdownCommandLegend(writer, report) + writeMarkdownFailures(writer, report) if len(report.Benchmarks) > 1 { - writeMarkdownRanking(writer, report) + writeMarkdownRankingWith(writer, report, renderer.ranking) } writeMarkdownCharts(writer, charts, chartDirectory) if len(report.Benchmarks) > 1 { - writeMarkdownComparison(writer, report) + writeMarkdownComparison(writer, renderer) } fmt.Fprintln(writer, "## Statistical Detail") for index, benchmark := range report.Benchmarks { @@ -56,7 +62,7 @@ func WriteMarkdownWithCharts( for _, note := range report.Notes { fmt.Fprintf(writer, "> %s\n\n", escapeMarkdown(note)) } - return nil + return checked.Err() } func writeMarkdownBenchmark( @@ -76,7 +82,7 @@ func writeMarkdownBenchmark( "Disk footprint: %s executable + %s linked = %s "+ "(%d files; %d dyld-cache dependencies excluded)\n\n", escapeMarkdown(label), - strings.Join(benchmark.Tool.Command, " "), + fullCommand(benchmark), benchmark.Tool.SHA256, formatBytes(float64(benchmark.Tool.SizeBytes)), formatBytes(float64(benchmark.Tool.LinkedSizeBytes)), @@ -89,7 +95,7 @@ func writeMarkdownBenchmark( formatCount(benchmark.Summary.ValidSampleCount.Mean), formatDuration(benchmark.Summary.SampleCoverageSeconds.Mean), ) - if !benchmarkSamplesReliable(benchmark, intervalSeconds) { + if !model.SamplingReliable(benchmark, intervalSeconds) { fmt.Fprintln( writer, "> Sampling quality: **LIMITED** (fewer than two valid samples or intervals).", @@ -101,8 +107,8 @@ func writeMarkdownBenchmark( "| Metric | Mean Β± Οƒ | 95% CI mean | Median | P95 | Range |", ) fmt.Fprintln(writer, "|---|---:|---:|---:|---:|---:|") - for _, row := range metricRows { - if !available(row, operatingSystem) { + for _, row := range metricCatalog { + if !availableFor(row, operatingSystem, benchmark.Summary) { fmt.Fprintf(writer, "| %s | N/A | N/A | N/A | N/A | N/A |\n", row.name) continue } diff --git a/internal/report/metric_catalog_test.go b/internal/report/metric_catalog_test.go new file mode 100644 index 0000000..aa8e613 --- /dev/null +++ b/internal/report/metric_catalog_test.go @@ -0,0 +1,47 @@ +package report + +import ( + "testing" + + "github.com/shellcell/snailrace/internal/model" +) + +func TestMetricCatalogIdentityAndSelectors(t *testing.T) { + if len(metricCatalog) != int(metricCount) { + t.Fatalf("metric catalog length = %d, want %d", len(metricCatalog), metricCount) + } + run := model.Run{ + WallSeconds: 1, CPUUserSeconds: 2, CPUSystemSeconds: 3, + AverageCPUPercent: 4, MeanResidentBytes: 5, PeakResidentBytes: 6, + OSMaxRSSBytes: 7, PeakPhysicalFootprintBytes: 8, + PhysicalFootprintValid: true, PeakVirtualBytes: 9, + PeakProcesses: 10, PeakThreads: 11, PeakFileDescriptors: 12, + } + summary := model.Summarize([]model.Run{run}) + for index, metric := range metricCatalog { + if metric.id != metricID(index) { + t.Fatalf("metric %d has ID %d", index, metric.id) + } + if metric.name == "" || metric.chartName == "" || metric.format == nil { + t.Fatalf("metric %d has incomplete presentation metadata", metric.id) + } + if got, want := metric.stats(summary).Mean, metric.run(run); got != want { + t.Fatalf("metric %d summary = %v, run = %v", metric.id, got, want) + } + } +} + +func TestChartGroupsReferenceEveryMetricOnce(t *testing.T) { + groups := chartGroups(model.Report{Config: model.Config{Mode: "command"}}) + seen := make(map[metricID]int) + for _, group := range groups { + for _, metric := range group.metrics { + seen[metric.id]++ + } + } + for _, metric := range metricCatalog { + if seen[metric.id] != 1 { + t.Fatalf("metric %d appears in %d chart groups", metric.id, seen[metric.id]) + } + } +} diff --git a/internal/report/ranking.go b/internal/report/ranking.go index 7d1b445..bb2bc38 100644 --- a/internal/report/ranking.go +++ b/internal/report/ranking.go @@ -8,31 +8,13 @@ import ( func calculateRanking(report model.Report) rankingData { numeric := analysis.Calculate(report.Config, report.Benchmarks) result := rankingData{ - rows: make([]rankingRow, len(numeric.Rows)), - ramAvailable: numeric.RAMAvailable, ramPresent: numeric.RAMPresent, - bestOverall: numeric.BestOverall, - primaryRatio: numeric.PrimaryRatio, cpuRatio: numeric.CPURatio, - footprintRatio: numeric.FootprintRatio, - indexPrimary: numeric.IndexPrimary, indexCPU: numeric.IndexCPU, - indexRAM: numeric.IndexRAM, indexFootprint: numeric.IndexFootprint, + Ranking: numeric, } - if result.ramPresent && !result.ramAvailable { + if result.RAMPresent && !result.RAMAvailable { result.samplingInterval = currentIntervalLabel(report.Config) } - for index, row := range numeric.Rows { - result.rows[index] = rankingRow{ - benchmark: row.Benchmark, overallRank: row.OverallRank, - primaryRank: row.PrimaryRank, cpuRank: row.CPURank, - ramRank: row.RAMRank, footprintRank: row.FootprintRank, - overallScore: row.OverallScore, primaryScore: row.PrimaryScore, - cpuScore: row.CPUScore, ramScore: row.RAMScore, - footprintScore: row.FootprintScore, primaryValue: row.PrimaryValue, - cpuValue: row.CPUValue, ramValue: row.RAMValue, - footprintValue: row.FootprintValue, - } - } result.primaryLabel, result.primaryUnit = "TIME", formatDuration - if numeric.FixedTUI { + if result.FixedTUI { result.primaryLabel, result.primaryUnit = "CPU", formatPercent } result.winners = rankingWinners(report, result) diff --git a/internal/report/ranking_html.go b/internal/report/ranking_html.go index 1314c5c..159d382 100644 --- a/internal/report/ranking_html.go +++ b/internal/report/ranking_html.go @@ -9,8 +9,14 @@ import ( "github.com/shellcell/snailrace/internal/model" ) -func writeHTMLRanking(writer io.Writer, report model.Report) { - ranking := calculateRanking(report) +func writeHTMLRanking(writer io.Writer, report model.Report, ranking rankingData) { + if len(ranking.Rows) == 0 { + fmt.Fprintf( + writer, `

Ranking unavailable

%s

`, + html.EscapeString(ranking.UnavailableReason), + ) + return + } fmt.Fprint(writer, `

Category leaders (point estimates)

`) for _, winner := range ranking.winners { var names string @@ -28,6 +34,13 @@ func writeHTMLRanking(writer io.Writer, report model.Report) { html.EscapeString(winner.value), ) } + if !ranking.Available { + fmt.Fprintf( + writer, `

Overall ranking unavailable

%s

`, + html.EscapeString(ranking.UnavailableReason), + ) + return + } fmt.Fprintf( writer, `

Overall ranking (point estimates)

Comparison baseline: `+ @@ -43,20 +56,20 @@ func writeHTMLRanking(writer io.Writer, report model.Report) { rankingHeader(5, "RAM aggregate", "number")+ rankingHeader(6, "Linked size", "number")+ ``) - for _, row := range ranking.rows { + for _, row := range ranking.Rows { ramValue := "N/A" - if ranking.ramAvailable { + if ranking.RAMAvailable { ramValue = rankingHTMLCell( - formatBytes(row.ramValue), row.ramScore, row.ramRank, true, + formatBytes(row.RAMValue), row.RAMScore, row.RAMRank, true, ) - } else if ranking.ramPresent { + } else if ranking.RAMPresent { ramValue = fmt.Sprintf( `%ssampling-limited`, - html.EscapeString(formatBytes(row.ramValue)), + html.EscapeString(formatBytes(row.RAMValue)), ) } class := "" - if row.overallRank == 1 { + if row.OverallRank == 1 { class = ` class="rank-one"` } fmt.Fprintf( @@ -68,29 +81,29 @@ func writeHTMLRanking(writer io.Writer, report model.Report) { `%s`+ `%s`+ `%s`, - class, row.overallRank, row.overallRank, - html.EscapeString(report.Benchmarks[row.benchmark].Tool.Name), - rankingHTMLTool(report, row.benchmark), - row.overallScore, + class, row.OverallRank, row.OverallRank, + html.EscapeString(report.Benchmarks[row.Benchmark].Tool.Name), + htmlToolLabel(report, row.Benchmark, true), + row.OverallScore, rankingHTMLCell( - formatScore(row.overallScore), row.overallScore/ranking.bestOverall, - row.overallRank, true, + formatScore(row.OverallScore), row.OverallScore/ranking.BestOverall, + row.OverallRank, true, ), - row.primaryValue, + row.PrimaryValue, rankingHTMLCell( - ranking.primaryUnit(row.primaryValue), row.primaryScore, - row.primaryRank, ranking.primaryRatio, + ranking.primaryUnit(row.PrimaryValue), row.PrimaryScore, + row.PrimaryRank, ranking.PrimaryRatio, ), - row.cpuValue, + row.CPUValue, rankingHTMLCell( - cpuRankingValue(report, row), row.cpuScore, row.cpuRank, ranking.cpuRatio, + cpuRankingValue(report, row), row.CPUScore, row.CPURank, ranking.CPURatio, ), - row.ramValue, + row.RAMValue, ramValue, - row.footprintValue, + row.FootprintValue, rankingHTMLCell( - formatBytes(row.footprintValue), row.footprintScore, - row.footprintRank, ranking.footprintRatio, + formatBytes(row.FootprintValue), row.FootprintScore, + row.FootprintRank, ranking.FootprintRatio, ), ) } @@ -113,10 +126,6 @@ const rankingSortScript = `` -func rankingHTMLTool(report model.Report, index int) string { - return htmlToolLabel(report, index, true) -} - func rankingHTMLCell(value string, score float64, rank int, ratioAvailable bool) string { if math.IsInf(score, 1) || math.IsNaN(score) { return "N/A" diff --git a/internal/report/ranking_json.go b/internal/report/ranking_json.go index 2853a01..9e7c4ef 100644 --- a/internal/report/ranking_json.go +++ b/internal/report/ranking_json.go @@ -13,11 +13,18 @@ type jsonWinner struct { Value string `json:"formatted_value"` } +type jsonFailure struct { + Tool string `json:"tool"` + Failed int `json:"failed_runs"` + Total int `json:"total_runs"` + ExitCodes []int `json:"exit_codes"` +} + type jsonRankingRow struct { - Rank int `json:"rank"` + Rank int `json:"rank,omitempty"` Tool string `json:"tool"` - BalancedIndex float64 `json:"balanced_index"` - BalancedDeltaPercent float64 `json:"balanced_delta_percent"` + BalancedIndex *float64 `json:"balanced_index,omitempty"` + BalancedDeltaPercent *float64 `json:"balanced_delta_percent,omitempty"` PrimaryValue float64 `json:"primary_value"` PrimaryRank int `json:"primary_rank"` PrimaryDeltaPercent *float64 `json:"primary_delta_percent,omitempty"` @@ -33,28 +40,51 @@ type jsonRankingRow struct { } type jsonRanking struct { - Method string `json:"method"` - PrimaryMetric string `json:"primary_metric"` - Winners []jsonWinner `json:"winners"` - Rows []jsonRankingRow `json:"rows"` + Available bool `json:"available"` + UnavailableReason string `json:"unavailable_reason,omitempty"` + Method string `json:"method"` + PrimaryMetric string `json:"primary_metric"` + IncludedDimensions []string `json:"included_dimensions"` + Winners []jsonWinner `json:"winners"` + Rows []jsonRankingRow `json:"rows"` } -func writeJSON(writer io.Writer, report model.Report) error { +func writeJSON(writer io.Writer, renderer *Renderer) error { + report := renderer.rawReport payload := struct { model.Report - Ranking jsonRanking `json:"ranking"` - }{Report: report, Ranking: makeJSONRanking(report)} + Ranking jsonRanking `json:"ranking"` + Failures []jsonFailure `json:"failures,omitempty"` + Reliability []string `json:"reliability_caveats,omitempty"` + }{ + Report: report, Ranking: makeJSONRanking(report, renderer.ranking), + Failures: makeJSONFailures(report), Reliability: renderer.rawCaveats, + } encoder := json.NewEncoder(writer) encoder.SetIndent("", " ") return encoder.Encode(payload) } -func makeJSONRanking(report model.Report) jsonRanking { - ranking := calculateRanking(report) +func makeJSONFailures(report model.Report) []jsonFailure { + failures := benchmarkFailures(report) + result := make([]jsonFailure, 0, len(failures)) + for _, failure := range failures { + result = append(result, jsonFailure{ + Tool: report.Benchmarks[failure.benchmark].Tool.Name, + Failed: failure.failed, Total: failure.total, + ExitCodes: append([]int(nil), failure.exitCodes...), + }) + } + return result +} + +func makeJSONRanking(report model.Report, ranking rankingData) jsonRanking { result := jsonRanking{ + Available: ranking.Available, UnavailableReason: ranking.UnavailableReason, Method: "equal-weight geometric mean of normalized category costs; included: " + balancedIndexCategories(ranking), - PrimaryMetric: ranking.primaryLabel, + PrimaryMetric: ranking.primaryLabel, + IncludedDimensions: append([]string(nil), includedDimensionsFromRanking(ranking)...), } for _, winner := range ranking.winners { result.Winners = append(result.Winners, jsonWinner{ @@ -62,27 +92,32 @@ func makeJSONRanking(report model.Report) jsonRanking { Value: winner.value, }) } - for _, row := range ranking.rows { + for _, row := range ranking.Rows { item := jsonRankingRow{ - Rank: row.overallRank, Tool: report.Benchmarks[row.benchmark].Tool.Name, - BalancedIndex: row.overallScore, - BalancedDeltaPercent: (row.overallScore/ranking.bestOverall - 1) * 100, - PrimaryValue: row.primaryValue, PrimaryRank: row.primaryRank, - CPUValue: row.cpuValue, CPURank: row.cpuRank, - FootprintBytes: row.footprintValue, FootprintRank: row.footprintRank, + Tool: report.Benchmarks[row.Benchmark].Tool.Name, + PrimaryValue: row.PrimaryValue, PrimaryRank: row.PrimaryRank, + CPUValue: row.CPUValue, CPURank: row.CPURank, + FootprintBytes: row.FootprintValue, FootprintRank: row.FootprintRank, + } + if ranking.Available { + item.Rank = row.OverallRank + item.BalancedIndex = floatPointer(row.OverallScore) + item.BalancedDeltaPercent = floatPointer( + (row.OverallScore/ranking.BestOverall - 1) * 100, + ) } - if ranking.primaryRatio { - item.PrimaryDeltaPercent = floatPointer((row.primaryScore - 1) * 100) + if ranking.PrimaryRatio { + item.PrimaryDeltaPercent = floatPointer((row.PrimaryScore - 1) * 100) } - if ranking.cpuRatio { - item.CPUDeltaPercent = floatPointer((row.cpuScore - 1) * 100) + if ranking.CPURatio { + item.CPUDeltaPercent = floatPointer((row.CPUScore - 1) * 100) } - if ranking.footprintRatio { - item.FootprintDeltaPercent = floatPointer((row.footprintScore - 1) * 100) + if ranking.FootprintRatio { + item.FootprintDeltaPercent = floatPointer((row.FootprintScore - 1) * 100) } - if ranking.ramAvailable { - item.RAMValueBytes, item.RAMRank = row.ramValue, row.ramRank - item.RAMDeltaPercent = floatPointer((row.ramScore - 1) * 100) + if ranking.RAMAvailable { + item.RAMValueBytes, item.RAMRank = row.RAMValue, row.RAMRank + item.RAMDeltaPercent = floatPointer((row.RAMScore - 1) * 100) } result.Rows = append(result.Rows, item) } diff --git a/internal/report/ranking_markdown.go b/internal/report/ranking_markdown.go index b078d1f..6d13157 100644 --- a/internal/report/ranking_markdown.go +++ b/internal/report/ranking_markdown.go @@ -7,8 +7,14 @@ import ( "github.com/shellcell/snailrace/internal/model" ) -func writeMarkdownRanking(writer io.Writer, report model.Report) { - ranking := calculateRanking(report) +func writeMarkdownRankingWith(writer io.Writer, report model.Report, ranking rankingData) { + if len(ranking.Rows) == 0 { + fmt.Fprintf( + writer, "## Ranking Unavailable\n\n%s.\n\n", + escapeMarkdown(ranking.UnavailableReason), + ) + return + } fmt.Fprintln(writer, "## Category Leaders (Point Estimates)") fmt.Fprintln(writer, "\n| Category | Tool | Value |") fmt.Fprintln(writer, "|---|---|---:|") @@ -22,6 +28,13 @@ func writeMarkdownRanking(writer io.Writer, report model.Report) { writer, "\nComparison baseline: **%s**. Lower balanced index is better.\n", escapeMarkdown(report.Benchmarks[baselineIndex(report)].Tool.Name), ) + if !ranking.Available { + fmt.Fprintf( + writer, "\n## Overall Ranking Unavailable\n\n%s.\n\n", + escapeMarkdown(ranking.UnavailableReason), + ) + return + } fmt.Fprintln(writer, "\n## Overall Ranking (Point Estimates)") fmt.Fprintf( writer, @@ -29,32 +42,32 @@ func writeMarkdownRanking(writer io.Writer, report model.Report) { ranking.primaryLabel, ) fmt.Fprintln(writer, "|---:|---|---:|---:|---:|---:|---:|") - for _, row := range ranking.rows { + for _, row := range ranking.Rows { ramValue := "N/A" - if ranking.ramAvailable { + if ranking.RAMAvailable { ramValue = rankingCell( - formatBytes(row.ramValue), row.ramScore, row.ramRank, true, + formatBytes(row.RAMValue), row.RAMScore, row.RAMRank, true, ) } fmt.Fprintf( writer, "| #%d | %s | %s | %s | %s | %s | %s |\n", - row.overallRank, - escapeMarkdown(reportToolLabel(report, row.benchmark)), + row.OverallRank, + escapeMarkdown(reportToolLabel(report, row.Benchmark)), rankingCell( - formatScore(row.overallScore), row.overallScore/ranking.bestOverall, - row.overallRank, true, + formatScore(row.OverallScore), row.OverallScore/ranking.BestOverall, + row.OverallRank, true, ), rankingCell( - ranking.primaryUnit(row.primaryValue), row.primaryScore, - row.primaryRank, ranking.primaryRatio, + ranking.primaryUnit(row.PrimaryValue), row.PrimaryScore, + row.PrimaryRank, ranking.PrimaryRatio, ), rankingCell( - cpuRankingValue(report, row), row.cpuScore, row.cpuRank, ranking.cpuRatio, + cpuRankingValue(report, row), row.CPUScore, row.CPURank, ranking.CPURatio, ), ramValue, rankingCell( - formatBytes(row.footprintValue), row.footprintScore, - row.footprintRank, ranking.footprintRatio, + formatBytes(row.FootprintValue), row.FootprintScore, + row.FootprintRank, ranking.FootprintRatio, ), ) } diff --git a/internal/report/ranking_test.go b/internal/report/ranking_test.go index 6f37ab8..248ca30 100644 --- a/internal/report/ranking_test.go +++ b/internal/report/ranking_test.go @@ -7,21 +7,22 @@ import ( "github.com/shellcell/snailrace/internal/model" ) -func TestAutomaticBaselineUsesBalancedWinner(t *testing.T) { +func TestBalancedWinnerSelectsBaseline(t *testing.T) { benchmarks := []model.Benchmark{ rankingBenchmark("fast", 1, 3, 300, 300, 300), rankingBenchmark("middle", 2, 2, 200, 200, 200), rankingBenchmark("efficient", 3, 1, 100, 100, 100), } config := model.Config{Mode: "command"} - if got := analysis.AutomaticBaseline(config, benchmarks); got != 3 { - t.Fatalf("baseline = %d, want efficient tool at index 3", got) + numeric := analysis.Calculate(config, benchmarks) + if !numeric.Available || numeric.Rows[0].Benchmark+1 != 3 { + t.Fatalf("baseline = %d, want efficient tool at index 3", numeric.Rows[0].Benchmark+1) } ranking := calculateRanking(model.Report{Config: config, Benchmarks: benchmarks}) if ranking.winners[1].benchmarks[0] != 0 || ranking.winners[2].benchmarks[0] != 2 { t.Fatalf("unexpected category winners: %+v", ranking.winners) } - if ranking.rows[0].overallScore/ranking.bestOverall != 1 { + if ranking.Rows[0].OverallScore/ranking.BestOverall != 1 { t.Fatal("best overall row should have zero difference from best") } } @@ -35,9 +36,9 @@ func TestCompositeRAMWinnerIsNormalizedToOne(t *testing.T) { ranking := calculateRanking(model.Report{ Config: model.Config{Mode: "command"}, Benchmarks: benchmarks, }) - for _, row := range ranking.rows { - if row.ramRank == 1 && row.ramScore != 1 { - t.Fatalf("RAM winner score = %g, want 1", row.ramScore) + for _, row := range ranking.Rows { + if row.RAMRank == 1 && row.RAMScore != 1 { + t.Fatalf("RAM winner score = %g, want 1", row.RAMScore) } } } @@ -63,10 +64,10 @@ func TestZeroBestCategoryHasNoRatioDelta(t *testing.T) { ranking := calculateRanking(model.Report{ Config: model.Config{Mode: "command"}, Benchmarks: benchmarks, }) - if ranking.cpuRatio { + if ranking.CPURatio { t.Fatal("CPU ratio should be unavailable when the best value is zero") } - if got := rankingCell("0 ns", ranking.rows[0].cpuScore, 1, false); got != "0 ns Β· Ξ” n/a (#1)" { + if got := rankingCell("0 ns", ranking.Rows[0].CPUScore, 1, false); got != "0 ns Β· Ξ” n/a (#1)" { t.Fatalf("zero-best ranking cell = %q", got) } } diff --git a/internal/report/ranking_text.go b/internal/report/ranking_text.go index b65d4f0..71d09cd 100644 --- a/internal/report/ranking_text.go +++ b/internal/report/ranking_text.go @@ -8,8 +8,11 @@ import ( "github.com/shellcell/snailrace/internal/model" ) -func writeTextRanking(writer io.Writer, report model.Report) { - ranking := calculateRanking(report) +func writeTextRankingWith(writer io.Writer, report model.Report, ranking rankingData) { + if len(ranking.Rows) == 0 { + fmt.Fprintf(writer, "Ranking unavailable\t%s\n\n", ranking.UnavailableReason) + return + } fmt.Fprintln(writer, "Category leaders (point estimates)\tTool\tValue") for _, winner := range ranking.winners { fmt.Fprintf( @@ -22,36 +25,40 @@ func writeTextRanking(writer io.Writer, report model.Report) { writer, "Comparison baseline\t%s\n\n", report.Benchmarks[baselineIndex(report)].Tool.Name, ) + if !ranking.Available { + fmt.Fprintf(writer, "Overall ranking unavailable\t%s\n\n", ranking.UnavailableReason) + return + } fmt.Fprintf( writer, "Overall ranking (point estimates)\tTool\tBalanced\t%s\tCPU cost\tRAM aggregate\tLinked size\n", ranking.primaryLabel, ) - for _, row := range ranking.rows { + for _, row := range ranking.Rows { ramValue := "N/A" - if ranking.ramAvailable { + if ranking.RAMAvailable { ramValue = rankingCell( - formatBytes(row.ramValue), row.ramScore, row.ramRank, true, + formatBytes(row.RAMValue), row.RAMScore, row.RAMRank, true, ) } fmt.Fprintf( writer, "#%d\t%s\t%s\t%s\t%s\t%s\t%s\n", - row.overallRank, reportToolLabel(report, row.benchmark), + row.OverallRank, reportToolLabel(report, row.Benchmark), rankingCell( - formatScore(row.overallScore), row.overallScore/ranking.bestOverall, - row.overallRank, true, + formatScore(row.OverallScore), row.OverallScore/ranking.BestOverall, + row.OverallRank, true, ), rankingCell( - ranking.primaryUnit(row.primaryValue), row.primaryScore, - row.primaryRank, ranking.primaryRatio, + ranking.primaryUnit(row.PrimaryValue), row.PrimaryScore, + row.PrimaryRank, ranking.PrimaryRatio, ), rankingCell( - cpuRankingValue(report, row), row.cpuScore, row.cpuRank, ranking.cpuRatio, + cpuRankingValue(report, row), row.CPUScore, row.CPURank, ranking.CPURatio, ), ramValue, rankingCell( - formatBytes(row.footprintValue), row.footprintScore, - row.footprintRank, ranking.footprintRatio, + formatBytes(row.FootprintValue), row.FootprintScore, + row.FootprintRank, ranking.FootprintRatio, ), ) } @@ -71,10 +78,10 @@ func rankingCell(value string, score float64, rank int, ratioAvailable bool) str } func cpuRankingValue(report model.Report, row rankingRow) string { - if report.Config.Mode == "tui" && report.Config.DurationSeconds > 0 { - return formatPercent(row.cpuValue) + if report.Config.FixedDurationTUI() { + return formatPercent(row.CPUValue) } - return formatDuration(row.cpuValue) + return formatDuration(row.CPUValue) } func formatScore(value float64) string { diff --git a/internal/report/ranking_types.go b/internal/report/ranking_types.go index 944fab7..a598dcf 100644 --- a/internal/report/ranking_types.go +++ b/internal/report/ranking_types.go @@ -1,22 +1,8 @@ package report -type rankingRow struct { - benchmark int - overallRank int - primaryRank int - cpuRank int - ramRank int - footprintRank int - overallScore float64 - primaryScore float64 - cpuScore float64 - ramScore float64 - footprintScore float64 - primaryValue float64 - cpuValue float64 - ramValue float64 - footprintValue float64 -} +import "github.com/shellcell/snailrace/internal/analysis" + +type rankingRow = analysis.RankingRow type categoryWinner struct { category string @@ -25,19 +11,9 @@ type categoryWinner struct { } type rankingData struct { - rows []rankingRow + analysis.Ranking winners []categoryWinner primaryLabel string primaryUnit func(float64) string - ramAvailable bool - ramPresent bool samplingInterval string - primaryRatio bool - cpuRatio bool - footprintRatio bool - indexPrimary bool - indexCPU bool - indexRAM bool - indexFootprint bool - bestOverall float64 } diff --git a/internal/report/ranking_winners.go b/internal/report/ranking_winners.go index 8547bd0..60f5022 100644 --- a/internal/report/ranking_winners.go +++ b/internal/report/ranking_winners.go @@ -8,46 +8,51 @@ import ( ) func rankingWinners(report model.Report, ranking rankingData) []categoryWinner { - if len(ranking.rows) == 0 { + if len(ranking.Rows) == 0 { return nil } - winners := []categoryWinner{ - winnerForRank(report, ranking.rows, "OVERALL", func(row rankingRow) int { - return row.overallRank - }, func(row rankingRow) string { - return fmt.Sprintf("%.3fx balanced index", row.overallScore) - }), - winnerForRank(report, ranking.rows, ranking.primaryLabel, func(row rankingRow) int { - return row.primaryRank + var winners []categoryWinner + appendWinner := func(winner categoryWinner) { + if len(winner.benchmarks) > 0 { + winners = append(winners, winner) + } + } + if ranking.Available { + appendWinner(winnerForRank(ranking.Rows, "OVERALL", func(row rankingRow) int { + return row.OverallRank }, func(row rankingRow) string { - return ranking.primaryUnit(row.primaryValue) - }), + return fmt.Sprintf("%.3fx balanced index", row.OverallScore) + })) } - if !(report.Config.Mode == "tui" && report.Config.DurationSeconds > 0) { - winners = append(winners, winnerForRank( - report, ranking.rows, "CPU COST", func(row rankingRow) int { - return row.cpuRank + appendWinner(winnerForRank(ranking.Rows, ranking.primaryLabel, func(row rankingRow) int { + return row.PrimaryRank + }, func(row rankingRow) string { + return ranking.primaryUnit(row.PrimaryValue) + })) + if !report.Config.FixedDurationTUI() { + appendWinner(winnerForRank( + ranking.Rows, "CPU COST", func(row rankingRow) int { + return row.CPURank }, func(row rankingRow) string { - return formatDuration(row.cpuValue) + return formatDuration(row.CPUValue) })) } - if ranking.ramAvailable { - winners = append(winners, winnerForRank( - report, ranking.rows, "RAM COST", - func(row rankingRow) int { return row.ramRank }, - func(row rankingRow) string { return formatBytes(row.ramValue) + " aggregate" }, + if ranking.RAMAvailable { + appendWinner(winnerForRank( + ranking.Rows, "RAM COST", + func(row rankingRow) int { return row.RAMRank }, + func(row rankingRow) string { return formatBytes(row.RAMValue) + " aggregate" }, )) } - winners = append(winners, winnerForRank( - report, ranking.rows, "LINKED SIZE", - func(row rankingRow) int { return row.footprintRank }, - func(row rankingRow) string { return formatBytes(row.footprintValue) }, + appendWinner(winnerForRank( + ranking.Rows, "LINKED SIZE", + func(row rankingRow) int { return row.FootprintRank }, + func(row rankingRow) string { return formatBytes(row.FootprintValue) }, )) return winners } func winnerForRank( - report model.Report, rows []rankingRow, category string, rank func(rankingRow) int, @@ -56,13 +61,13 @@ func winnerForRank( var benchmarks []int for _, row := range rows { if rank(row) == 1 { - benchmarks = append(benchmarks, row.benchmark) + benchmarks = append(benchmarks, row.Benchmark) } } result := categoryWinner{category: category, benchmarks: benchmarks} if len(benchmarks) > 0 { for _, row := range rows { - if row.benchmark == benchmarks[0] { + if row.Benchmark == benchmarks[0] { result.value = value(row) break } diff --git a/internal/report/render.go b/internal/report/render.go new file mode 100644 index 0000000..d095a00 --- /dev/null +++ b/internal/report/render.go @@ -0,0 +1,86 @@ +package report + +import ( + "time" + + "github.com/shellcell/snailrace/internal/model" +) + +// Renderer caches report analysis and is intended for sequential rendering. +// Callers must not mutate the source report while the renderer is in use. +type Renderer struct { + rawReport model.Report + displayReport model.Report + metadata ReportMetadata + ranking rankingData + dimensionText string + caveats []string + rawCaveats []string + chartGroups []chartGroup + charts []svgChart + chartsReady bool + comparisons map[comparisonKey]deltaResult + baselineRuns map[metricID]map[int]float64 +} + +type ReportMetadata struct { + MeasuredAt time.Time + ToolNames []string +} + +type comparisonKey struct { + candidate int + metric metricID +} + +func NewRenderer(input model.Report) *Renderer { + display := safeDisplayReport(input) + ranking := calculateRanking(input) + dimensions := includedDimensionsFromRanking(ranking) + toolNames := make([]string, len(input.Benchmarks)) + for index, benchmark := range input.Benchmarks { + toolNames[index] = benchmark.Tool.Name + } + return &Renderer{ + rawReport: input, displayReport: display, ranking: ranking, + metadata: ReportMetadata{MeasuredAt: input.MeasuredAt, ToolNames: toolNames}, + dimensionText: dimensionsText(dimensions), + caveats: reliabilityCaveats(display), rawCaveats: reliabilityCaveats(input), + chartGroups: chartGroups(display), comparisons: make(map[comparisonKey]deltaResult), + baselineRuns: make(map[metricID]map[int]float64), + } +} + +func (renderer *Renderer) Metadata() ReportMetadata { + metadata := renderer.metadata + metadata.ToolNames = append([]string(nil), metadata.ToolNames...) + return metadata +} + +func (renderer *Renderer) comparison(candidate int, metric metricID) deltaResult { + key := comparisonKey{candidate: candidate, metric: metric} + if result, ok := renderer.comparisons[key]; ok { + return result + } + baseline := renderer.displayReport.Benchmarks[baselineIndex(renderer.displayReport)] + row := metricCatalog[metric] + baselineRuns, ok := renderer.baselineRuns[metric] + if !ok { + baselineRuns = indexMetricValues(baseline.Runs, row) + renderer.baselineRuns[metric] = baselineRuns + } + result := compareMetricWithBaselineIndex( + baseline, baselineRuns, renderer.displayReport.Benchmarks[candidate], row, + renderer.displayReport.Config.IntervalMS/1000, + ) + renderer.comparisons[key] = result + return result +} + +func (renderer *Renderer) reportCharts() []svgChart { + if !renderer.chartsReady { + renderer.charts = buildReportCharts(renderer) + renderer.chartsReady = true + } + return renderer.charts +} diff --git a/internal/report/report.go b/internal/report/report.go index 6000c01..3da6de8 100644 --- a/internal/report/report.go +++ b/internal/report/report.go @@ -3,33 +3,38 @@ package report import ( "fmt" "io" - "strings" "github.com/shellcell/snailrace/internal/model" + "github.com/shellcell/snailrace/internal/style" ) -func Write(writer io.Writer, format string, report model.Report) error { - switch strings.ToLower(format) { - case "text", "txt": - return writeText(writer, report) - case "json": - return writeJSON(writer, report) - case "markdown", "md": - return writeMarkdown(writer, report) - case "html": - return writeHTML(writer, report) - case "svg": - return writeSVG(writer, report) +func (renderer *Renderer) Write(writer io.Writer, format string) error { + info, ok := LookupFormat(format) + if !ok { + return fmt.Errorf("unknown report format %q", format) + } + switch info.Name { + case FormatText: + return writeText(writer, renderer) + case FormatJSON: + return writeJSON(writer, renderer) + case FormatMarkdown: + return renderer.WriteMarkdownWithCharts(writer, nil, "") + case FormatHTML: + return writeHTML(writer, renderer) + case FormatSVG: + return writeSVG(writer, renderer) default: return fmt.Errorf("unknown report format %q", format) } } -func ValidFormat(format string) bool { - switch strings.ToLower(format) { - case "text", "txt", "json", "markdown", "md", "html", "svg": - return true - default: - return false +func safeDisplayReport(report model.Report) model.Report { + report.Benchmarks = append([]model.Benchmark(nil), report.Benchmarks...) + for index := range report.Benchmarks { + report.Benchmarks[index].Tool.Name = style.SafeText( + report.Benchmarks[index].Tool.Name, + ) } + return report } diff --git a/internal/report/report_bench_test.go b/internal/report/report_bench_test.go new file mode 100644 index 0000000..c339648 --- /dev/null +++ b/internal/report/report_bench_test.go @@ -0,0 +1,154 @@ +package report + +import ( + "bytes" + "io" + "testing" + + "github.com/shellcell/snailrace/internal/model" +) + +func TestCachedRendererMatchesIndependentFormats(t *testing.T) { + report := benchmarkReport(3, 4) + renderer := NewRenderer(report) + for _, format := range []string{"html", "svg", "markdown", "json", "text"} { + var cached, independent bytes.Buffer + if err := renderer.Write(&cached, format); err != nil { + t.Fatal(err) + } + if err := NewRenderer(report).Write(&independent, format); err != nil { + t.Fatal(err) + } + if cached.String() != independent.String() { + t.Fatalf("cached %s output differs", format) + } + } +} + +func TestRendererOutputIsOrderIndependentAndRepeatable(t *testing.T) { + report := benchmarkReport(3, 4) + formats := []string{"html", "svg", "markdown", "json", "text"} + want := make(map[string]string, len(formats)) + for _, format := range formats { + var output bytes.Buffer + if err := NewRenderer(report).Write(&output, format); err != nil { + t.Fatal(err) + } + want[format] = output.String() + } + for _, order := range [][]string{ + formats, + {"text", "json", "markdown", "svg", "html", "text"}, + } { + renderer := NewRenderer(report) + for _, format := range order { + var output bytes.Buffer + if err := renderer.Write(&output, format); err != nil { + t.Fatal(err) + } + if output.String() != want[format] { + t.Fatalf("%s output depends on rendering order", format) + } + } + } +} + +func TestRendererMetadataIsIndependent(t *testing.T) { + report := benchmarkReport(2, 2) + originalName := report.Benchmarks[0].Tool.Name + renderer := NewRenderer(report) + report.Benchmarks[0].Tool.Name = "changed" + metadata := renderer.Metadata() + if metadata.ToolNames[0] != originalName { + t.Fatalf("renderer metadata changed with source: %+v", metadata) + } + metadata.ToolNames[0] = "also changed" + if renderer.Metadata().ToolNames[0] != originalName { + t.Fatal("renderer returned mutable metadata") + } +} + +func BenchmarkWriteHTML10Tools100Runs(b *testing.B) { + report := benchmarkReport(10, 100) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + if err := NewRenderer(report).Write(io.Discard, "html"); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkReportCharts10Tools100Runs(b *testing.B) { + report := benchmarkReport(10, 100) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + if charts := NewRenderer(report).reportCharts(); len(charts) == 0 { + b.Fatal("no charts") + } + } +} + +func BenchmarkWriteAllFormatsCached10Tools100Runs(b *testing.B) { + report := benchmarkReport(10, 100) + formats := []string{"html", "svg", "markdown", "json", "text"} + b.ReportAllocs() + b.ResetTimer() + for range b.N { + renderer := NewRenderer(report) + for _, format := range formats { + if err := renderer.Write(io.Discard, format); err != nil { + b.Fatal(err) + } + } + } +} + +func BenchmarkWriteAllFormatsIndependent10Tools100Runs(b *testing.B) { + report := benchmarkReport(10, 100) + formats := []string{"html", "svg", "markdown", "json", "text"} + b.ReportAllocs() + b.ResetTimer() + for range b.N { + for _, format := range formats { + if err := NewRenderer(report).Write(io.Discard, format); err != nil { + b.Fatal(err) + } + } + } +} + +func benchmarkReport(toolCount, runCount int) model.Report { + benchmarks := make([]model.Benchmark, toolCount) + for tool := range toolCount { + runs := make([]model.Run, runCount) + for run := range runCount { + scale := float64(tool+1) * (1 + float64(run%7)/100) + runs[run] = model.Run{ + Index: run + 1, StopReason: "exited", WallSeconds: scale / 100, + CPUUserSeconds: scale / 200, CPUSystemSeconds: scale / 500, + AverageCPUPercent: 70 + scale, MeanResidentBytes: scale * 1024 * 1024, + PeakResidentBytes: scale * 2 * 1024 * 1024, + OSMaxRSSBytes: scale * 2 * 1024 * 1024, + PeakVirtualBytes: scale * 4 * 1024 * 1024, + PeakProcesses: 2, PeakThreads: 4, PeakFileDescriptors: 8, + SampleCount: 10, SampleCoverageSeconds: scale / 100, + } + } + benchmarks[tool] = model.Benchmark{ + Tool: model.ToolInfo{ + Name: "tool", Command: []string{"tool"}, + DiskFootprintBytes: int64(tool+1) * 1024 * 1024, + }, + Runs: runs, Summary: model.Summarize(runs), + } + } + return model.Report{ + Config: model.Config{ + Runs: runCount, Baseline: 1, Mode: "command", IntervalMS: 10, + IndexDimensions: []string{"time", "cpu", "ram", "disk"}, + }, + Host: model.HostInfo{OS: "linux", LogicalCPUs: 8}, Benchmarks: benchmarks, + } +} diff --git a/internal/report/semantics.go b/internal/report/semantics.go new file mode 100644 index 0000000..0645806 --- /dev/null +++ b/internal/report/semantics.go @@ -0,0 +1,75 @@ +package report + +import ( + "fmt" + "strings" + + "github.com/shellcell/snailrace/internal/model" +) + +func includedDimensionsFromRanking(ranking rankingData) []string { + var result []string + if ranking.IndexPrimary { + result = append(result, "time") + } + if ranking.IndexCPU { + result = append(result, "cpu") + } + if ranking.IndexRAM { + result = append(result, "ram") + } + if ranking.IndexFootprint { + result = append(result, "disk") + } + return result +} + +func dimensionsText(dimensions []string) string { + if len(dimensions) == 0 { + return "none" + } + return strings.Join(dimensions, ", ") +} + +func reliabilityCaveats(report model.Report) []string { + interval := report.Config.IntervalMS / 1000 + var result []string + for _, benchmark := range report.Benchmarks { + if benchmark.Tool.SHA256 != "" && !benchmark.Tool.ProvenanceVerified { + result = append(result, benchmark.Tool.Name+ + ": post-run executable verification was not completed") + } + if !model.SamplingReliable(benchmark, interval) { + result = append(result, fmt.Sprintf( + "%s: sampled process metrics are limited by sparse observations", + benchmark.Tool.Name, + )) + } + if report.Host.OS != "darwin" { + continue + } + physical := benchmark.Summary.PeakPhysicalFootprintBytes + switch { + case physical == nil: + result = append(result, benchmark.Tool.Name+": physical footprint unavailable") + case physical.N < len(benchmark.Runs): + result = append(result, fmt.Sprintf( + "%s: physical footprint available for %d of %d runs", + benchmark.Tool.Name, physical.N, len(benchmark.Runs), + )) + } + } + return result +} + +func physicalFootprintAvailable(report model.Report) bool { + if report.Host.OS != "darwin" || len(report.Benchmarks) == 0 { + return false + } + for _, benchmark := range report.Benchmarks { + if benchmark.Summary.PeakPhysicalFootprintBytes == nil { + return false + } + } + return true +} diff --git a/internal/report/svg.go b/internal/report/svg.go index 2e044a0..6aa15ce 100644 --- a/internal/report/svg.go +++ b/internal/report/svg.go @@ -5,8 +5,6 @@ import ( "html" "io" "strings" - - "github.com/shellcell/snailrace/internal/model" ) const svgReportWidth = 1500 @@ -21,11 +19,18 @@ const svgReportStyle = `` -func writeSVG(writer io.Writer, report model.Report) error { +func writeSVG(writer io.Writer, renderer *Renderer) error { + report := renderer.displayReport + checked := newErrorWriter(writer) + writer = checked var content strings.Builder - y := svgHeader(&content, report) + y := svgHeader(&content, renderer) y = svgCommandLegend(&content, report, y) - y = svgCharts(&content, reportCharts(report), y) + charts := renderer.reportCharts() + if failures := failureChart(report); failures.height > 0 { + charts = append([]svgChart{failures}, charts...) + } + y = svgCharts(&content, charts, y) height := y + 36 fmt.Fprintf( writer, @@ -36,13 +41,17 @@ func writeSVG(writer io.Writer, report model.Report) error { `%s%s`, svgReportWidth, height, svgReportStyle, content.String(), ) - return nil + return checked.Err() } -func svgHeader(output *strings.Builder, report model.Report) int { - panelHeight, nextSection := 72, 230 - if report.Config.Mode == "tui" { - panelHeight, nextSection = 96, 254 +func svgHeader(output *strings.Builder, renderer *Renderer) int { + report := renderer.displayReport + caveats := renderer.caveats + panelHeight := 96 + len(caveats)*22 + nextSection := 254 + len(caveats)*22 + if report.Config.IsTUI() { + panelHeight += 24 + nextSection += 24 } fmt.Fprint(output, ``) fmt.Fprint(output, `🐌 SNAILRACE CHARTS`) @@ -71,14 +80,25 @@ func svgHeader(output *strings.Builder, report model.Report) int { html.EscapeString(report.Host.LoadBefore), report.Host.ProcessesBefore, html.EscapeString(clip(report.Host.Kernel, 32)), ) - if report.Config.Mode == "tui" { + fmt.Fprintf( + output, `balanced dimensions %s`, + html.EscapeString(renderer.dimensionText), + ) + for index, caveat := range caveats { + fmt.Fprintf( + output, `reliability %s`, + 227+index*22, html.EscapeString(caveat), + ) + } + if report.Config.IsTUI() { duration := "until exit" if report.Config.DurationSeconds > 0 { duration = formatDuration(report.Config.DurationSeconds) } fmt.Fprintf( - output, `TUI %s Β· %dx%d`, - duration, report.Config.TerminalWidth, report.Config.TerminalHeight, + output, `TUI %s Β· %dx%d`, + 227+len(caveats)*22, duration, + report.Config.TerminalWidth, report.Config.TerminalHeight, ) } return nextSection @@ -93,11 +113,11 @@ func svgCharts(output *strings.Builder, charts []svgChart, y int) int { left, right := balanceCharts(charts) leftY, rightY := y, y for _, chart := range left { - output.WriteString(chart.embedded(24, leftY)) + chart.writeEmbedded(output, 24, leftY) leftY += chart.height + 16 } for _, chart := range right { - output.WriteString(chart.embedded(756, rightY)) + chart.writeEmbedded(output, 756, rightY) rightY += chart.height + 16 } if rightY > leftY { diff --git a/internal/report/svg_test.go b/internal/report/svg_test.go index 2c061ec..b07f769 100644 --- a/internal/report/svg_test.go +++ b/internal/report/svg_test.go @@ -24,7 +24,7 @@ func TestSVGIsStandaloneCompleteReport(t *testing.T) { Notes: []string{"No outliers removed."}, } var output bytes.Buffer - if err := writeSVG(&output, report); err != nil { + if err := writeSVG(&output, NewRenderer(report)); err != nil { t.Fatal(err) } decoder := xml.NewDecoder(bytes.NewReader(output.Bytes())) @@ -51,12 +51,13 @@ func TestMarkdownReferencesSeparateSVGCharts(t *testing.T) { benchmarkWithWallTimes("candidate", 9, 9), }, } - charts, err := WriteChartFiles(t.TempDir(), report, false) + renderer := NewRenderer(report) + charts, err := renderer.WriteChartFiles(t.TempDir(), false) if err != nil { t.Fatal(err) } var output bytes.Buffer - if err := WriteMarkdownWithCharts(&output, report, charts, "charts"); err != nil { + if err := renderer.WriteMarkdownWithCharts(&output, charts, "charts"); err != nil { t.Fatal(err) } if strings.Contains(output.String(), " 0 { duration = formatDuration(report.Config.DurationSeconds) @@ -58,12 +62,13 @@ func writeVerboseText(writer io.Writer, report model.Report) error { } fmt.Fprintln(w) writeTextCommandLegend(w, report) + writeTextFailures(w, report) if len(report.Benchmarks) > 1 { - writeTextRanking(w, report) + writeTextRankingWith(w, report, renderer.ranking) } if len(report.Benchmarks) > 1 { fmt.Fprintln(w, "Detailed baseline deltas\t") - writeTextComparison(w, report) + writeTextComparison(w, renderer) } fmt.Fprintln(w, "Statistical detail\t") for index, benchmark := range report.Benchmarks { @@ -98,7 +103,7 @@ func writeTextBenchmark( label += " [BASELINE]" } fmt.Fprintf(w, "%s\t\n", label) - commandLines := wrapText(strings.Join(benchmark.Tool.Command, " "), 100) + commandLines := wrapText(fullCommand(benchmark), 100) for index, line := range commandLines { field := "" if index == 0 { @@ -119,12 +124,12 @@ func writeTextBenchmark( formatCount(benchmark.Summary.ValidSampleCount.Mean), formatDuration(benchmark.Summary.SampleCoverageSeconds.Mean), ) - if !benchmarkSamplesReliable(benchmark, intervalSeconds) { + if !model.SamplingReliable(benchmark, intervalSeconds) { fmt.Fprintln(w, "Sampling quality\tLIMITED: fewer than two valid samples or intervals") } fmt.Fprintln(w, "Metric\tMean Β± Οƒ\t95% CI mean\tMedian\tP95\tRange") - for _, row := range metricRows { - if !available(row, operatingSystem) { + for _, row := range metricCatalog { + if !availableFor(row, operatingSystem, benchmark.Summary) { fmt.Fprintf(w, "%s\tN/A\tN/A\tN/A\tN/A\tN/A\n", row.name) continue } diff --git a/internal/report/text_bars.go b/internal/report/text_bars.go index 46d44ec..e3192f2 100644 --- a/internal/report/text_bars.go +++ b/internal/report/text_bars.go @@ -14,14 +14,21 @@ import ( const compactBarWidth = 20 // writeCompactText renders the short, bar-chart stdout report used by default. -func writeCompactText(writer io.Writer, report model.Report) error { +func writeCompactText(writer io.Writer, renderer *Renderer) error { + report := renderer.displayReport terminal := writerIsTerminal(writer) var body strings.Builder body.WriteString("\n") + fmt.Fprintf(&body, "BALANCED DIMENSIONS %s\n", renderer.dimensionText) + for _, caveat := range renderer.caveats { + fmt.Fprintf(&body, "RELIABILITY %s\n", caveat) + } + body.WriteString("\n") + writeTextFailures(&body, report) if len(report.Benchmarks) == 1 { - writeCompactSingle(&body, report, terminal) + writeCompactSingle(&body, report, renderer.ranking, terminal) } else if len(report.Benchmarks) > 1 { - writeCompactBars(&body, report, terminal) + writeCompactBars(&body, report, renderer.ranking, terminal) } _, err := io.WriteString(writer, body.String()) return err @@ -38,7 +45,7 @@ type compactMetric struct { } func compactMetrics(report model.Report, ranking rankingData) []compactMetric { - fixedTUI := report.Config.Mode == "tui" && report.Config.DurationSeconds > 0 + fixedTUI := report.Config.FixedDurationTUI() primaryStdDev := func(s model.Summary) float64 { return s.WallSeconds.StdDev } cpuStdDev := func(s model.Summary) float64 { return s.CPUTotalSeconds.StdDev } if fixedTUI { @@ -46,7 +53,7 @@ func compactMetrics(report model.Report, ranking rankingData) []compactMetric { cpuStdDev = primaryStdDev } ramNote := "" - if ranking.ramPresent && !ranking.ramAvailable { + if ranking.RAMPresent && !ranking.RAMAvailable { ramNote = " (sampling-limited Β· excluded from balanced index" if ranking.samplingInterval != "" { ramNote += "; lower -interval, now " + ranking.samplingInterval @@ -63,31 +70,31 @@ func compactMetrics(report model.Report, ranking rankingData) []compactMetric { return []compactMetric{ { name: ranking.primaryLabel, unit: ranking.primaryUnit, - available: ranking.primaryRatio, - value: func(r rankingRow) float64 { return r.primaryValue }, + available: ranking.PrimaryRatio, + value: func(r rankingRow) float64 { return r.PrimaryValue }, stdDev: primaryStdDev, }, { name: "CPU", unit: func(v float64) string { return cpuCompactUnit(fixedTUI, v) }, - available: ranking.cpuRatio, - value: func(r rankingRow) float64 { return r.cpuValue }, + available: ranking.CPURatio, + value: func(r rankingRow) float64 { return r.CPUValue }, stdDev: cpuStdDev, }, { - name: "RAM", note: ramNote, unit: formatBytes, available: ranking.ramPresent, - value: func(r rankingRow) float64 { return r.ramValue }, + name: "RAM", note: ramNote, unit: formatBytes, available: ranking.RAMPresent, + value: func(r rankingRow) float64 { return r.RAMValue }, stdDev: func(s model.Summary) float64 { return s.MeanResidentBytes.StdDev }, }, { - name: "PHYS", unit: formatBytes, available: report.Host.OS == "darwin", + name: "PHYS", unit: formatBytes, available: physicalFootprintAvailable(report), value: func(r rankingRow) float64 { - return report.Benchmarks[r.benchmark].Summary.PhysicalFootprintStats().Mean + return report.Benchmarks[r.Benchmark].Summary.PhysicalFootprintStats().Mean }, stdDev: func(s model.Summary) float64 { return s.PhysicalFootprintStats().StdDev }, }, { - name: "DISK", note: diskNote, unit: formatBytes, available: ranking.footprintRatio, - value: func(r rankingRow) float64 { return r.footprintValue }, + name: "DISK", note: diskNote, unit: formatBytes, available: ranking.FootprintRatio, + value: func(r rankingRow) float64 { return r.FootprintValue }, }, } } @@ -99,19 +106,28 @@ func cpuCompactUnit(fixedTUI bool, value float64) string { return formatDuration(value) } -func writeCompactBars(body *strings.Builder, report model.Report, terminal bool) { - ranking := calculateRanking(report) - rows := rowsByBenchmark(ranking.rows) +func writeCompactBars( + body *strings.Builder, report model.Report, ranking rankingData, terminal bool, +) { + if len(ranking.Rows) == 0 { + fmt.Fprintf(body, "BALANCED unavailable: %s\n\n", ranking.UnavailableReason) + return + } + rows := rowsByBenchmark(ranking.Rows) labelWidth := compactLabelWidth(report) - winner := report.Benchmarks[ranking.rows[0].benchmark].Tool.Name - fmt.Fprintf( - body, "BALANCED %s winner %s\n", - strings.Join(indexDimensionLabels(report.Config), ","), winner, - ) - writeBarGroup(body, report, terminal, labelWidth, rows, func(r rankingRow) float64 { - return r.overallScore - }, func(r rankingRow) string { return formatScore(r.overallScore) }) + if ranking.Available { + winner := report.Benchmarks[ranking.Rows[0].Benchmark].Tool.Name + fmt.Fprintf( + body, "BALANCED %s winner %s\n", + balancedIndexCategories(ranking), winner, + ) + writeBarGroup(body, report, terminal, labelWidth, rows, func(r rankingRow) float64 { + return r.OverallScore + }, func(r rankingRow) string { return formatScore(r.OverallScore) }) + } else { + fmt.Fprintf(body, "BALANCED unavailable: %s\n\n", ranking.UnavailableReason) + } for _, metric := range compactMetrics(report, ranking) { if !metric.available { @@ -121,7 +137,7 @@ func writeCompactBars(body *strings.Builder, report model.Report, terminal bool) writeBarGroup(body, report, terminal, labelWidth, rows, metric.value, func(r rankingRow) string { text := metric.unit(metric.value(r)) if metric.stdDev != nil { - sigma := metric.stdDev(report.Benchmarks[r.benchmark].Summary) + sigma := metric.stdDev(report.Benchmarks[r.Benchmark].Summary) text += " Β± " + metric.unit(sigma) } return text @@ -168,19 +184,20 @@ func writeBarGroup( body.WriteString("\n") } -func writeCompactSingle(body *strings.Builder, report model.Report, terminal bool) { - ranking := calculateRanking(report) - if len(ranking.rows) == 0 { +func writeCompactSingle( + body *strings.Builder, report model.Report, ranking rankingData, terminal bool, +) { + if len(ranking.Rows) == 0 { return } - row := ranking.rows[0] + row := ranking.Rows[0] for _, metric := range compactMetrics(report, ranking) { if !metric.available { continue } text := metric.unit(metric.value(row)) if metric.stdDev != nil { - sigma := metric.stdDev(report.Benchmarks[row.benchmark].Summary) + sigma := metric.stdDev(report.Benchmarks[row.Benchmark].Summary) text += " Β± " + metric.unit(sigma) } fmt.Fprintf(body, " %-5s %s%s\n", metric.name, text, redNote(metric.note, terminal)) @@ -210,7 +227,7 @@ func textBar(fraction float64, width int) string { func rowsByBenchmark(rows []rankingRow) map[int]rankingRow { result := make(map[int]rankingRow, len(rows)) for _, row := range rows { - result[row.benchmark] = row + result[row.Benchmark] = row } return result } @@ -231,11 +248,3 @@ func padLabel(label string, width int) string { } return label } - -func indexDimensionLabels(config model.Config) []string { - dimensions := config.IndexDimensions - if len(dimensions) == 0 { - dimensions = []string{"time", "cpu", "ram"} - } - return dimensions -} diff --git a/internal/report/text_test.go b/internal/report/text_test.go index a7cf889..b447854 100644 --- a/internal/report/text_test.go +++ b/internal/report/text_test.go @@ -2,6 +2,7 @@ package report import ( "bytes" + "encoding/json" "strings" "testing" @@ -18,7 +19,7 @@ func TestFixedTUIComparisonEmphasizesResources(t *testing.T) { }, } var output bytes.Buffer - if err := writeText(&output, input); err != nil { + if err := writeText(&output, NewRenderer(input)); err != nil { t.Fatal(err) } if !strings.Contains(output.String(), "Average CPU") { @@ -39,7 +40,7 @@ func TestCompactReportShowsBarsAndOmitsDetail(t *testing.T) { Notes: []string{"a methodology note"}, } var output bytes.Buffer - if err := writeText(&output, report); err != nil { + if err := writeText(&output, NewRenderer(report)); err != nil { t.Fatal(err) } text := output.String() @@ -66,7 +67,7 @@ func TestVerboseReportKeepsStatisticalDetail(t *testing.T) { Notes: []string{"a methodology note"}, } var output bytes.Buffer - if err := writeText(&output, report); err != nil { + if err := writeText(&output, NewRenderer(report)); err != nil { t.Fatal(err) } if !strings.Contains(output.String(), "Statistical detail") { @@ -80,7 +81,7 @@ func TestCompactSingleToolListsKeyMetrics(t *testing.T) { Benchmarks: []model.Benchmark{rankingBenchmark("solo", 1, 1, 100, 200, 1000)}, } var output bytes.Buffer - if err := writeText(&output, report); err != nil { + if err := writeText(&output, NewRenderer(report)); err != nil { t.Fatal(err) } text := output.String() @@ -104,16 +105,16 @@ func samplingLimitedRAMReport() model.Report { func TestSamplingLimitedRAMIsShownButExcludedFromIndex(t *testing.T) { report := samplingLimitedRAMReport() ranking := calculateRanking(report) - if ranking.ramAvailable || !ranking.ramPresent { + if ranking.RAMAvailable || !ranking.RAMPresent { t.Fatalf("want RAM present but sampling-limited, got present=%v available=%v", - ranking.ramPresent, ranking.ramAvailable) + ranking.RAMPresent, ranking.RAMAvailable) } if strings.Contains(balancedIndexCategories(ranking), "RAM") { t.Fatal("sampling-limited RAM must stay out of the balanced index") } // Compact stdout shows the RAM bars with a remark. var compact bytes.Buffer - if err := writeText(&compact, report); err != nil { + if err := writeText(&compact, NewRenderer(report)); err != nil { t.Fatal(err) } if !strings.Contains(compact.String(), "RAM") || @@ -122,12 +123,12 @@ func TestSamplingLimitedRAMIsShownButExcludedFromIndex(t *testing.T) { } // HTML ranking table and the RAM chart show it with a remark, not N/A. var htmlRanking bytes.Buffer - writeHTMLRanking(&htmlRanking, report) + writeHTMLRanking(&htmlRanking, report, calculateRanking(report)) if !strings.Contains(htmlRanking.String(), "sampling-limited") { t.Fatal("HTML ranking should mark RAM sampling-limited instead of N/A") } foundRAMChart := false - for _, chart := range rankingCharts(report) { + for _, chart := range rankingCharts(report, calculateRanking(report)) { if chart.title == "RAM AGGREGATE" { foundRAMChart = true if !strings.Contains(chart.description, "sampling-limited") { @@ -140,6 +141,43 @@ func TestSamplingLimitedRAMIsShownButExcludedFromIndex(t *testing.T) { } } +func TestAllFormatsExposeActualDimensionsAndReliability(t *testing.T) { + report := samplingLimitedRAMReport() + for _, format := range []string{"text", "markdown", "html", "svg"} { + var output bytes.Buffer + if err := NewRenderer(report).Write(&output, format); err != nil { + t.Fatalf("%s: %v", format, err) + } + text := strings.ToLower(output.String()) + if !strings.Contains(text, "time, cpu") { + t.Fatalf("%s omits actual included dimensions", format) + } + if !strings.Contains(text, "reliability") { + t.Fatalf("%s omits reliability caveat", format) + } + if !strings.Contains(text, "first:") || !strings.Contains(text, "second:") { + t.Fatalf("%s omits a per-tool reliability caveat", format) + } + } + var output bytes.Buffer + if err := NewRenderer(report).Write(&output, "json"); err != nil { + t.Fatal(err) + } + var decoded struct { + Ranking struct { + Included []string `json:"included_dimensions"` + } `json:"ranking"` + Reliability []string `json:"reliability_caveats"` + } + if err := json.Unmarshal(output.Bytes(), &decoded); err != nil { + t.Fatal(err) + } + if strings.Join(decoded.Ranking.Included, ",") != "time,cpu" || + len(decoded.Reliability) == 0 { + t.Fatalf("JSON semantics = %+v", decoded) + } +} + func TestColumnGapsFindsRunsOfAtLeastTwoSpaces(t *testing.T) { gaps := columnGaps("a bb c d") want := [][2]int{{1, 3}, {5, 8}} diff --git a/internal/report/ui_test.go b/internal/report/ui_test.go index ed3d843..3bb487e 100644 --- a/internal/report/ui_test.go +++ b/internal/report/ui_test.go @@ -31,7 +31,7 @@ func TestJSONIncludesComputedRanking(t *testing.T) { }, } var output bytes.Buffer - if err := writeJSON(&output, report); err != nil { + if err := writeJSON(&output, NewRenderer(report)); err != nil { t.Fatal(err) } for _, expected := range []string{"\"ranking\"", "\"winners\"", "balanced_index"} { @@ -41,6 +41,48 @@ func TestJSONIncludesComputedRanking(t *testing.T) { } } +func TestUnavailableRankingSerializesWithoutNonFiniteValues(t *testing.T) { + report := model.Report{ + Config: model.Config{ + Mode: "command", Baseline: 1, IntervalMS: 10, + IndexDimensions: []string{"ram"}, + }, + Benchmarks: []model.Benchmark{benchmarkWithWallTimes("short", 0.001)}, + } + var output bytes.Buffer + if err := writeJSON(&output, NewRenderer(report)); err != nil { + t.Fatal(err) + } + if !strings.Contains(output.String(), `"available": false`) || + strings.Contains(output.String(), "+Inf") || strings.Contains(output.String(), "NaN") { + t.Fatalf("unexpected unavailable ranking JSON: %s", output.String()) + } +} + +func TestFailedCommandsAreProminentInEveryFormat(t *testing.T) { + failed := benchmarkWithWallTimes("failed", 0.001) + failed.Runs[0].ExitCode = 7 + failed.Summary = model.Summarize(failed.Runs) + report := model.Report{ + Config: model.Config{ + Mode: "command", Baseline: 2, IndexDimensions: []string{"time"}, + }, + Benchmarks: []model.Benchmark{failed, benchmarkWithWallTimes("success", 1)}, + } + for _, format := range []string{"text", "html", "markdown", "svg", "json"} { + t.Run(format, func(t *testing.T) { + var output bytes.Buffer + if err := NewRenderer(report).Write(&output, format); err != nil { + t.Fatal(err) + } + value := strings.ToLower(output.String()) + if !strings.Contains(value, "failed") || !strings.Contains(value, "7") { + t.Fatalf("%s does not prominently report failure: %s", format, output.String()) + } + }) + } +} + func TestHTMLPreservesLongCommandInWrappingBlock(t *testing.T) { command := "tool --value=" + strings.Repeat("x", 300) report := model.Report{ @@ -50,7 +92,7 @@ func TestHTMLPreservesLongCommandInWrappingBlock(t *testing.T) { }}, } var output bytes.Buffer - if err := writeHTML(&output, report); err != nil { + if err := writeHTML(&output, NewRenderer(report)); err != nil { t.Fatal(err) } if !strings.Contains(output.String(), command) || diff --git a/internal/runner/benchmark.go b/internal/runner/benchmark.go index e76163d..90093b5 100644 --- a/internal/runner/benchmark.go +++ b/internal/runner/benchmark.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "os" "github.com/shellcell/snailrace/internal/model" ) @@ -19,16 +20,59 @@ func Benchmark( config Config, options Options, ) ([]model.Benchmark, error) { + if err := validateBenchmarkInputs(ctx, specs, config); err != nil { + return nil, err + } + specs = append([]Spec(nil), specs...) + for index := range specs { + specs[index].Args = append([]string(nil), specs[index].Args...) + } + prepared := make([]preparedSpec, len(specs)) + for index, spec := range specs { + prepared[index] = preparedSpec{Spec: spec} + } benchmarks := make([]model.Benchmark, len(specs)) progress := newProgressTracker(options, specs, config) defer progress.finish() inspector := newToolInspector() + pinned := make([]*os.File, len(specs)) + defer func() { + for _, file := range pinned { + if file != nil { + _ = file.Close() + } + } + }() for index, spec := range specs { benchmarks[index].Runs = make([]model.Run, 0, config.Runs) - tool, err := inspector.inspect(spec) + tool, err := inspector.inspect(ctx, spec) if err != nil { return nil, err } + file, err := inspector.pin(ctx, &tool) + if err != nil { + return nil, fmt.Errorf("hash %q: %w", tool.Name, err) + } + pinned[index] = file + if pinnable := !executableIsScript(file); pinnable { + if tool.ShellTarget { + target := tool.Executable + if pinExecutionSupported { + target = pinnedExecutablePath(file) + } + if command, ok := pinShellExecutable(spec.Shell, target); ok { + prepared[index].Shell = command + prepared[index].shellTarget = true + if pinExecutionSupported { + prepared[index].executable = file + } + } + } else if pinExecutionSupported { + prepared[index].executable = file + } else { + prepared[index].executablePath = tool.Executable + } + } benchmarks[index].Tool = tool progress.setTool(index, tool) } @@ -41,7 +85,7 @@ func Benchmark( interrupted := false for warmup, order := range warmupOrder { for _, index := range order { - spec := specs[index] + spec := prepared[index] progress.update( index, spec.Name, warmup+1, config.Warmups, true, false, nil, ) @@ -77,7 +121,7 @@ func Benchmark( index, specs[index].Name, runIndex+1, config.Runs, false, false, benchmarks[index].Runs, ) - run, err := runOnce(ctx, specs[index], config.Interval, options) + run, err := runOnce(ctx, prepared[index], config.Interval, options) if err != nil { if ctx.Err() != nil { interrupted = true @@ -106,8 +150,13 @@ func Benchmark( return nil, ctx.Err() } for index := range benchmarks { - if err := inspector.addHash(&benchmarks[index].Tool); err != nil { - return nil, fmt.Errorf("hash %q: %w", benchmarks[index].Tool.Name, err) + if !interrupted { + if err := inspector.verify( + ctx, benchmarks[index].Tool, pinned[index], + ); err != nil { + return nil, fmt.Errorf("verify %q: %w", benchmarks[index].Tool.Name, err) + } + benchmarks[index].Tool.ProvenanceVerified = true } benchmarks[index].Summary = model.Summarize(benchmarks[index].Runs) } @@ -116,3 +165,9 @@ func Benchmark( } return benchmarks, nil } + +func executableIsScript(file *os.File) bool { + var magic [2]byte + count, _ := file.ReadAt(magic[:], 0) + return count == len(magic) && magic == [2]byte{'#', '!'} +} diff --git a/internal/runner/benchmark_linux_test.go b/internal/runner/benchmark_linux_test.go new file mode 100644 index 0000000..c2bac8e --- /dev/null +++ b/internal/runner/benchmark_linux_test.go @@ -0,0 +1,29 @@ +package runner + +import ( + "context" + "testing" + "time" +) + +func TestPinnedExecutionDoesNotLeakDescriptorsToChild(t *testing.T) { + // The child inspects its own descriptor table with shell builtins: a pin + // descriptor leaked through exec would appear at 3 or above. Sampling + // PeakFileDescriptors instead is unreliable β€” libc startup transiently + // opens files (locale archive, gconv), which a sample may legitimately + // catch. + benchmarks, err := Benchmark( + context.Background(), + []Spec{{Name: "fds", Args: []string{ + "/bin/sh", "-c", + "[ ! -e /proc/self/fd/3 ] && [ ! -e /proc/self/fd/9 ]", + }}}, + Config{Runs: 1, Interval: time.Millisecond}, Options{}, + ) + if err != nil { + t.Fatal(err) + } + if code := benchmarks[0].Runs[0].ExitCode; code != 0 { + t.Fatalf("child saw inherited descriptors (exit %d)", code) + } +} diff --git a/internal/runner/benchmark_progress.go b/internal/runner/benchmark_progress.go index 7f7d560..c0f5b9d 100644 --- a/internal/runner/benchmark_progress.go +++ b/internal/runner/benchmark_progress.go @@ -14,7 +14,10 @@ type progressTracker struct { tools int fixed time.Duration interval float64 + runs int estimates []ProgressEstimate + summaries []model.SummaryAccumulator + processed []int } func newProgressTracker(options Options, specs []Spec, config Config) *progressTracker { @@ -22,14 +25,18 @@ func newProgressTracker(options Options, specs []Spec, config Config) *progressT callback: options.Progress, started: time.Now(), total: len(specs) * (config.Runs + config.Warmups), tools: len(specs), fixed: options.Duration, + runs: config.Runs, interval: float64(config.Interval) / float64(time.Millisecond), } if tracker.callback == nil { return tracker } tracker.estimates = make([]ProgressEstimate, len(specs)) + tracker.summaries = make([]model.SummaryAccumulator, len(specs)) + tracker.processed = make([]int, len(specs)) for index, spec := range specs { tracker.estimates[index] = ProgressEstimate{ToolName: spec.Name, Total: config.Runs} + tracker.summaries[index] = model.NewSummaryAccumulator(config.Runs) } return tracker } @@ -57,8 +64,17 @@ func (tracker *progressTracker) update( if !warmup { estimate := &tracker.estimates[tool] if len(runs) > 0 && (!estimate.HasEstimate || estimate.Completed != len(runs)) { + if len(runs) < tracker.processed[tool] { + tracker.summaries[tool] = model.NewSummaryAccumulator(tracker.runs) + tracker.processed[tool] = 0 + } + for _, run := range runs[tracker.processed[tool]:] { + tracker.summaries[tool].Add(run) + } + tracker.processed[tool] = len(runs) estimate.HasEstimate = true - estimate.Estimate = model.Summarize(runs) + snapshot := tracker.summaries[tool].Snapshot() + estimate.Estimate = &snapshot } estimate.Completed = len(runs) estimate.Runs = runs @@ -74,12 +90,12 @@ func (tracker *progressTracker) update( Iteration: iteration, Iterations: iterations, Completed: tracker.completed, Total: tracker.total, Warmup: warmup, FixedDuration: tracker.fixed, IntervalMS: tracker.interval, - Elapsed: elapsed, ETA: eta, + ETA: eta, Estimates: append([]ProgressEstimate(nil), tracker.estimates...), } if len(runs) > 0 { event.HasEstimate = true - event.Estimate = tracker.estimates[tool].Estimate + event.Estimate = *tracker.estimates[tool].Estimate } tracker.callback(event) } diff --git a/internal/runner/benchmark_progress_test.go b/internal/runner/benchmark_progress_test.go index 76921fe..4cd40c5 100644 --- a/internal/runner/benchmark_progress_test.go +++ b/internal/runner/benchmark_progress_test.go @@ -2,8 +2,11 @@ package runner import ( "context" + "reflect" "testing" "time" + + "github.com/shellcell/snailrace/internal/model" ) func TestBenchmarkProgressIncludesRunningEstimate(t *testing.T) { @@ -31,3 +34,69 @@ func TestBenchmarkProgressIncludesRunningEstimate(t *testing.T) { ) } } + +func BenchmarkProgressUpdates1000Runs(b *testing.B) { + runs := make([]model.Run, 1000) + for index := range runs { + runs[index] = model.Run{ + Index: index + 1, WallSeconds: float64(index+1) / 1000, + CPUUserSeconds: 0.01, PeakResidentBytes: float64(index+1) * 1024, + MeanResidentBytes: float64(index+1) * 512, + SampleCount: 10, SampleCoverageSeconds: 0.1, + } + } + b.ReportAllocs() + b.ResetTimer() + for range b.N { + tracker := newProgressTracker( + Options{Progress: func(ProgressEvent) {}}, []Spec{{Name: "tool"}}, + Config{Runs: len(runs), Interval: time.Millisecond}, + ) + for index := range runs { + tracker.update(0, "tool", index+1, len(runs), false, false, runs[:index]) + tracker.update(0, "tool", index+1, len(runs), false, true, runs[:index+1]) + } + } +} + +func TestProgressSummariesMatchFinalStatistics(t *testing.T) { + runs := []model.Run{ + {Index: 1, WallSeconds: 3, CPUUserSeconds: 1, MeanResidentBytes: 30, + PeakPhysicalFootprintBytes: 300, PhysicalFootprintValid: true}, + {Index: 2, WallSeconds: 1, CPUUserSeconds: 2, MeanResidentBytes: 10, + PeakPhysicalFootprintBytes: 100, PhysicalFootprintValid: true}, + {Index: 3, WallSeconds: 2, CPUUserSeconds: 3, MeanResidentBytes: 20, + PeakPhysicalFootprintBytes: 200, PhysicalFootprintValid: true}, + } + var estimates []model.Summary + var raceEstimates []*model.Summary + tracker := newProgressTracker( + Options{Progress: func(event ProgressEvent) { + if event.HasEstimate { + estimates = append(estimates, event.Estimate) + } + if len(event.Estimates) > 0 && event.Estimates[0].Estimate != nil { + raceEstimates = append(raceEstimates, event.Estimates[0].Estimate) + } + }}, + []Spec{{Name: "tool"}}, Config{Runs: len(runs), Interval: time.Millisecond}, + ) + for index := range runs { + tracker.update(0, "tool", index+1, len(runs), false, true, runs[:index+1]) + } + if len(estimates) != len(runs) { + t.Fatalf("estimates = %d, want %d", len(estimates), len(runs)) + } + for index, estimate := range estimates { + want := model.Summarize(runs[:index+1]) + if !reflect.DeepEqual(estimate, want) { + t.Fatalf("estimate %d = %+v, want %+v", index, estimate, want) + } + } + if estimates[0].PeakPhysicalFootprintBytes.Mean != 300 { + t.Fatal("earlier progress event was mutated by a later update") + } + if raceEstimates[0].WallSeconds.Mean != 3 { + t.Fatal("earlier race estimate was mutated by a later update") + } +} diff --git a/internal/runner/benchmark_test.go b/internal/runner/benchmark_test.go index 867125c..987dab5 100644 --- a/internal/runner/benchmark_test.go +++ b/internal/runner/benchmark_test.go @@ -3,6 +3,8 @@ package runner import ( "context" "errors" + "os" + "path/filepath" "testing" "time" ) @@ -33,6 +35,9 @@ func TestBenchmarkInterruptionKeepsCompletedRounds(t *testing.T) { if len(benchmark.Runs) != 1 { t.Fatalf("tool %d recorded %d runs, want 1 completed round", index, len(benchmark.Runs)) } + if benchmark.Tool.SHA256 == "" || benchmark.Tool.ProvenanceVerified { + t.Fatalf("interrupted tool provenance = %+v", benchmark.Tool) + } } } @@ -50,6 +55,148 @@ func TestBenchmarkInterruptionBeforeAnyRoundFails(t *testing.T) { } } +func TestBenchmarkContinuesAfterNonZeroExit(t *testing.T) { + benchmarks, err := Benchmark( + context.Background(), + []Spec{ + {Name: "exit", Shell: "exit 7"}, + {Name: "true", Args: []string{"true"}}, + }, + Config{Runs: 3, Warmups: 1, Interval: time.Millisecond}, + Options{}, + ) + if err != nil { + t.Fatal(err) + } + for index, benchmark := range benchmarks { + if len(benchmark.Runs) != 3 { + t.Fatalf("tool %d recorded %d runs, want 3", index, len(benchmark.Runs)) + } + } + for _, run := range benchmarks[0].Runs { + if run.ExitCode != 7 { + t.Fatalf("exit code = %d, want 7", run.ExitCode) + } + } +} + +func TestNativeExecutableRunsExitZero(t *testing.T) { + benchmarks, err := Benchmark( + context.Background(), + []Spec{{Name: "direct", Args: []string{"true"}}, {Name: "shell", Shell: "true"}}, + Config{Runs: 1, Interval: time.Millisecond}, Options{}, + ) + if err != nil { + t.Fatal(err) + } + for _, benchmark := range benchmarks { + if benchmark.Runs[0].ExitCode != 0 { + t.Fatalf( + "native %s exit code = %d, want 0", + benchmark.Tool.Name, benchmark.Runs[0].ExitCode, + ) + } + if !benchmark.Tool.ProvenanceVerified { + t.Fatalf("native %s provenance not verified", benchmark.Tool.Name) + } + } +} + +func TestShellBenchmarksInspectDistinctTargetExecutables(t *testing.T) { + // Generated targets with different sizes keep the assertions hermetic: + // real system binaries can collide byte-for-byte (uname and sleep are the + // same size on Ubuntu 24.04). + directory := t.TempDir() + first := filepath.Join(directory, "first-tool") + second := filepath.Join(directory, "second-tool") + if err := os.WriteFile(first, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile( + second, []byte("#!/bin/sh\n# deliberately larger target\nexit 0\n"), 0o755, + ); err != nil { + t.Fatal(err) + } + benchmarks, err := Benchmark( + context.Background(), + []Spec{{Name: "first", Shell: first}, {Name: "second", Shell: second}}, + Config{Runs: 1, Interval: time.Millisecond}, Options{}, + ) + if err != nil { + t.Fatal(err) + } + if benchmarks[0].Tool.Executable == benchmarks[1].Tool.Executable { + t.Fatalf("shell targets share executable %q", benchmarks[0].Tool.Executable) + } + if !benchmarks[0].Tool.ProvenanceVerified || !benchmarks[1].Tool.ProvenanceVerified { + t.Fatal("completed benchmark should verify executable provenance") + } + if benchmarks[0].Tool.DiskFootprintBytes == benchmarks[1].Tool.DiskFootprintBytes { + t.Fatalf( + "shell targets share disk footprint %d", + benchmarks[0].Tool.DiskFootprintBytes, + ) + } +} + +func TestScriptExecutionPreservesOriginalPath(t *testing.T) { + directory := t.TempDir() + script := filepath.Join(directory, "tool") + if err := os.WriteFile(filepath.Join(directory, "value"), []byte("ok\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile( + script, []byte("#!/bin/sh\ncat \"$(dirname \"$0\")/value\"\n"), 0o755, + ); err != nil { + t.Fatal(err) + } + benchmarks, err := Benchmark( + context.Background(), + []Spec{{Name: "direct", Args: []string{script}}, {Name: "shell", Shell: script}}, + Config{Runs: 1, Interval: time.Millisecond}, Options{}, + ) + if err != nil { + t.Fatal(err) + } + for _, benchmark := range benchmarks { + if benchmark.Runs[0].ExitCode != 0 { + t.Fatalf("script %s exit code = %d", benchmark.Tool.Name, benchmark.Runs[0].ExitCode) + } + } +} + +func TestBenchmarkRejectsInvalidInputs(t *testing.T) { + validSpec := Spec{Name: "true", Args: []string{"true"}} + validConfig := Config{Runs: 1, Interval: time.Millisecond} + tests := []struct { + name string + specs []Spec + config Config + }{ + {"no specs", nil, validConfig}, + {"empty spec", []Spec{{Name: "empty"}}, validConfig}, + {"empty name", []Spec{{Args: []string{"true"}}}, validConfig}, + {"both command forms", []Spec{{Name: "both", Shell: "true", Args: []string{"true"}}}, validConfig}, + {"zero runs", []Spec{validSpec}, Config{Interval: time.Millisecond}}, + {"zero interval", []Spec{validSpec}, Config{Runs: 1}}, + {"bad schedule index", []Spec{validSpec}, Config{ + Runs: 1, Interval: time.Millisecond, MeasurementOrder: [][]int{{1}}, + }}, + {"wrong schedule rounds", []Spec{validSpec}, Config{ + Runs: 2, Interval: time.Millisecond, MeasurementOrder: [][]int{{0}}, + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := Benchmark( + context.Background(), test.specs, test.config, Options{}, + ); err == nil { + t.Fatal("expected validation error") + } + }) + } +} + func completedRoundCount(estimates []ProgressEstimate) int { if len(estimates) == 0 { return 0 diff --git a/internal/runner/executable_darwin.go b/internal/runner/executable_darwin.go new file mode 100644 index 0000000..c90f578 --- /dev/null +++ b/internal/runner/executable_darwin.go @@ -0,0 +1,12 @@ +package runner + +import "os" + +// macOS cannot execve /dev/fd nodes and has no fexecve, so execution cannot +// be pinned to the inspected descriptor; provenance relies on the before and +// after hash verification instead. +const pinExecutionSupported = false + +func pinnedExecutablePath(*os.File) string { return "" } + +func pinnedExtraFiles(*os.File) []*os.File { return nil } diff --git a/internal/runner/executable_linux.go b/internal/runner/executable_linux.go new file mode 100644 index 0000000..8d1c5dc --- /dev/null +++ b/internal/runner/executable_linux.go @@ -0,0 +1,17 @@ +package runner + +import ( + "fmt" + "os" +) + +const pinExecutionSupported = true + +// The child resolves the parent's /proc entry, which stays alive as long as +// the parent keeps the descriptor open β€” no fd inheritance is needed, so +// pinnedExtraFiles has nothing to pass down. +func pinnedExecutablePath(file *os.File) string { + return fmt.Sprintf("/proc/%d/fd/%d", os.Getpid(), file.Fd()) +} + +func pinnedExtraFiles(*os.File) []*os.File { return nil } diff --git a/internal/runner/monitor.go b/internal/runner/monitor.go new file mode 100644 index 0000000..31496c3 --- /dev/null +++ b/internal/runner/monitor.go @@ -0,0 +1,130 @@ +package runner + +import ( + "time" + + "github.com/shellcell/snailrace/internal/platform" +) + +type processGroupSampler func(int) (platform.Metrics, bool) + +type sampleAggregate struct { + peak platform.Metrics + footprintMissing bool + footprintInvalidated bool + residentByteSamples uint64 + sampleCount uint64 + residentByteSeconds float64 + sampleCoverageSeconds float64 + previousRSS uint64 + previousAt time.Time +} + +func (aggregate *sampleAggregate) observe(current platform.Metrics, at time.Time) { + aggregate.addCoverage(at) + aggregate.residentByteSamples += current.ResidentBytes + aggregate.sampleCount++ + aggregate.peak.ResidentBytes = max(aggregate.peak.ResidentBytes, current.ResidentBytes) + aggregate.peak.VirtualBytes = max(aggregate.peak.VirtualBytes, current.VirtualBytes) + aggregate.peak.Processes = max(aggregate.peak.Processes, current.Processes) + aggregate.peak.Threads = max(aggregate.peak.Threads, current.Threads) + aggregate.peak.FileDescriptors = max( + aggregate.peak.FileDescriptors, current.FileDescriptors, + ) + aggregate.observeFootprint(current) + aggregate.previousRSS, aggregate.previousAt = current.ResidentBytes, at +} + +// The aggregate footprint is valid only when every sample carried a valid +// footprint and nothing invalidated it; a sample that merely lacks a footprint +// leaves the aggregate unusable without marking the run invalid. +func (aggregate *sampleAggregate) observeFootprint(current platform.Metrics) { + if current.PhysicalFootprintValid { + aggregate.peak.PhysicalFootprintBytes = max( + aggregate.peak.PhysicalFootprintBytes, current.PhysicalFootprintBytes, + ) + } else { + aggregate.footprintMissing = true + } + if current.PhysicalFootprintInvalid { + aggregate.footprintInvalidated = true + } + aggregate.peak.PhysicalFootprintValid = !aggregate.footprintMissing && + !aggregate.footprintInvalidated + aggregate.peak.PhysicalFootprintInvalid = aggregate.footprintInvalidated +} + +func (aggregate *sampleAggregate) invalidatePhysicalFootprint() { + aggregate.footprintInvalidated = true + aggregate.peak.PhysicalFootprintInvalid = true + aggregate.peak.PhysicalFootprintValid = false +} + +func (aggregate *sampleAggregate) addCoverage(at time.Time) { + if aggregate.previousAt.IsZero() { + return + } + seconds := at.Sub(aggregate.previousAt).Seconds() + aggregate.residentByteSeconds += float64(aggregate.previousRSS) * seconds + aggregate.sampleCoverageSeconds += seconds +} + +func (aggregate sampleAggregate) meanResident() float64 { + if aggregate.sampleCoverageSeconds > 0 { + return aggregate.residentByteSeconds / aggregate.sampleCoverageSeconds + } + if aggregate.sampleCount == 0 { + return 0 + } + return float64(aggregate.residentByteSamples) / float64(aggregate.sampleCount) +} + +type monitorHandle struct { + stop chan struct{} + done <-chan sampleAggregate +} + +func startMonitor(groupID int, interval time.Duration, sample processGroupSampler) monitorHandle { + stop := make(chan struct{}) + done := make(chan sampleAggregate, 1) + go monitorProcessGroup(groupID, interval, sample, stop, done) + return monitorHandle{stop: stop, done: done} +} + +func (monitor monitorHandle) finish() sampleAggregate { + close(monitor.stop) + return <-monitor.done +} + +func monitorProcessGroup( + groupID int, + interval time.Duration, + sample processGroupSampler, + stop <-chan struct{}, + done chan<- sampleAggregate, +) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + var aggregate sampleAggregate + observe := func() { + current, valid := sample(groupID) + if !valid { + if current.PhysicalFootprintInvalid { + aggregate.invalidatePhysicalFootprint() + } + return + } + aggregate.observe(current, time.Now()) + } + observe() + for { + select { + case <-ticker.C: + observe() + case <-stop: + aggregate.addCoverage(time.Now()) + done <- aggregate + return + } + } +} diff --git a/internal/runner/options.go b/internal/runner/options.go index 2ca0306..b4e90bc 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -18,6 +18,8 @@ type Config struct { type Options struct { ShowOutput bool Output io.Writer + Input io.Reader + TerminalOut io.Writer TUI bool Duration time.Duration Width uint16 @@ -42,7 +44,6 @@ type ProgressEvent struct { Estimates []ProgressEstimate FixedDuration time.Duration IntervalMS float64 - Elapsed time.Duration ETA time.Duration } @@ -51,7 +52,7 @@ type ProgressEstimate struct { Completed int Total int HasEstimate bool - Estimate model.Summary + Estimate *model.Summary Runs []model.Run DiskFootprintBytes int64 } diff --git a/internal/runner/prepare.go b/internal/runner/prepare.go new file mode 100644 index 0000000..0773191 --- /dev/null +++ b/internal/runner/prepare.go @@ -0,0 +1,31 @@ +package runner + +import ( + "context" + "io" + "os/exec" + "syscall" +) + +func RunPreparation(ctx context.Context, shell string, output io.Writer) error { + if err := ctx.Err(); err != nil { + return err + } + cmd := exec.Command("/bin/sh", "-c", shell) + configureProcessGroup(cmd) + cmd.Stdout, cmd.Stderr = output, output + if err := cmd.Start(); err != nil { + return err + } + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + select { + case err := <-done: + signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) + return err + case <-ctx.Done(): + signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) + <-done + return ctx.Err() + } +} diff --git a/internal/runner/prepare_test.go b/internal/runner/prepare_test.go new file mode 100644 index 0000000..c320527 --- /dev/null +++ b/internal/runner/prepare_test.go @@ -0,0 +1,70 @@ +package runner + +import ( + "context" + "errors" + "fmt" + "os" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +func TestPreparationCancellationKillsDescendants(t *testing.T) { + pidPath := t.TempDir() + "/child.pid" + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- RunPreparation( + ctx, fmt.Sprintf("sleep 30 & echo $! > %q; wait", pidPath), os.Stderr, + ) + }() + var pid int + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + data, err := os.ReadFile(pidPath) + if err == nil { + pid, err = strconv.Atoi(strings.TrimSpace(string(data))) + if err == nil { + break + } + } + time.Sleep(5 * time.Millisecond) + } + if pid == 0 { + cancel() + <-done + t.Fatal("preparation did not start its child") + } + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("preparation error = %v, want cancellation", err) + } + deadline = time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if err := syscall.Kill(pid, 0); errors.Is(err, syscall.ESRCH) { + return + } + if state, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)); err == nil && + strings.Contains(string(state), ") Z ") { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("preparation child %d survived cancellation", pid) +} + +func TestCancelledPreparationDoesNotStart(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + marker := t.TempDir() + "/started" + err := RunPreparation(ctx, fmt.Sprintf("touch %q", marker), os.Stderr) + if !errors.Is(err, context.Canceled) { + t.Fatalf("preparation error = %v, want cancellation", err) + } + if _, err := os.Stat(marker); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("cancelled preparation started: %v", err) + } +} diff --git a/internal/runner/run.go b/internal/runner/run.go index 78fe32c..edd0d44 100644 --- a/internal/runner/run.go +++ b/internal/runner/run.go @@ -2,7 +2,9 @@ package runner import ( "context" + "errors" "os" + "os/exec" "syscall" "time" @@ -12,7 +14,7 @@ import ( func runOnce( ctx context.Context, - spec Spec, + spec commandSpec, interval time.Duration, options Options, ) (model.Run, error) { @@ -32,114 +34,28 @@ func runOnce( if err := cmd.Start(); err != nil { return model.Run{}, err } - stop := make(chan struct{}) - done := make(chan platform.Metrics, 1) - go monitor(cmd.Process.Pid, interval, stop, done) + groupID := cmd.Process.Pid + monitor := startMonitor(groupID, interval, platform.NewGroupSampler().Sample) waitErr := cmd.Wait() elapsed := time.Since(started) - signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) - close(stop) - peak := <-done + signalProcessGroup(groupID, syscall.SIGKILL) + samples := monitor.finish() user, system, rusageRSS := platform.ResourceUsage(cmd.ProcessState) - peak.Processes = max(peak.Processes, 1) - peak.Threads = max(peak.Threads, 1) if ctx.Err() != nil { return model.Run{}, ctx.Err() } - if waitErr != nil { + if waitErr != nil && !isNonZeroExit(cmd, waitErr) { return model.Run{}, waitErr } - meanResident := sampledMeanResident(peak) - return model.Run{ - ExitCode: cmd.ProcessState.ExitCode(), WallSeconds: elapsed.Seconds(), - CPUUserSeconds: user, CPUSystemSeconds: system, - AverageCPUPercent: averageCPUPercent(user, system, elapsed), - PeakResidentBytes: float64(peak.ResidentBytes), - PeakPhysicalFootprintBytes: float64(peak.PhysicalFootprintBytes), - OSMaxRSSBytes: float64(rusageRSS), - MeanResidentBytes: meanResident, - PeakVirtualBytes: float64(peak.VirtualBytes), - PeakProcesses: float64(peak.Processes), - PeakThreads: float64(peak.Threads), - PeakFileDescriptors: float64(peak.FileDescriptors), - StopReason: "exited", - SampleCount: int(peak.SampleCount), - SampleCoverageSeconds: peak.SampleCoverageSeconds, - }, nil + return makeRun( + cmd.ProcessState, elapsed, user, system, rusageRSS, samples, "exited", + ), nil } -func monitor( - pid int, - interval time.Duration, - stop <-chan struct{}, - done chan<- platform.Metrics, -) { - ticker := time.NewTicker(interval) - defer ticker.Stop() - var peak platform.Metrics - var previousRSS uint64 - var previousAt time.Time - addCoverage := func(now time.Time) { - if previousAt.IsZero() { - return - } - seconds := now.Sub(previousAt).Seconds() - peak.ResidentByteSeconds += float64(previousRSS) * seconds - peak.SampleCoverageSeconds += seconds - } - sample := func() { - if current, valid := platform.SampleTree(pid); valid { - now := time.Now() - addCoverage(now) - updatePeaks(&peak, current) - previousRSS, previousAt = current.ResidentBytes, now - } - } - if platform.SampleImmediately() { - sample() - } - for { - select { - case <-ticker.C: - sample() - case <-stop: - addCoverage(time.Now()) - done <- peak - return - } - } -} - -func updatePeaks(peak *platform.Metrics, current platform.Metrics) { - peak.ResidentByteSamples += current.ResidentBytes - peak.SampleCount++ - peak.ResidentBytes = max(peak.ResidentBytes, current.ResidentBytes) - peak.PhysicalFootprintBytes = max( - peak.PhysicalFootprintBytes, current.PhysicalFootprintBytes, - ) - peak.VirtualBytes = max(peak.VirtualBytes, current.VirtualBytes) - peak.Processes = max(peak.Processes, current.Processes) - peak.Threads = max(peak.Threads, current.Threads) - peak.FileDescriptors = max( - peak.FileDescriptors, current.FileDescriptors, - ) -} - -func sampledMeanResident(metrics platform.Metrics) float64 { - if metrics.SampleCoverageSeconds > 0 { - return metrics.ResidentByteSeconds / metrics.SampleCoverageSeconds - } - if metrics.SampleCount == 0 { - return 0 - } - return float64(metrics.ResidentByteSamples) / float64(metrics.SampleCount) -} - -func averageCPUPercent(user, system float64, elapsed time.Duration) float64 { - if elapsed <= 0 { - return 0 - } - return (user + system) / elapsed.Seconds() * 100 +func isNonZeroExit(cmd *exec.Cmd, err error) bool { + var exitError *exec.ExitError + return errors.As(err, &exitError) && cmd.ProcessState != nil && + cmd.ProcessState.ExitCode() > 0 } diff --git a/internal/runner/run_test.go b/internal/runner/run_test.go index f63e9c2..fd05c0a 100644 --- a/internal/runner/run_test.go +++ b/internal/runner/run_test.go @@ -28,31 +28,79 @@ func TestRunOnceUsesInjectedOutputWriter(t *testing.T) { } } -func TestRunOnceRejectsFailedCommand(t *testing.T) { - _, err := runOnce( - context.Background(), Spec{Name: "false", Args: []string{"/bin/false"}}, +func TestRunOnceRecordsNonZeroExit(t *testing.T) { + run, err := runOnce( + context.Background(), Spec{Name: "exit", Shell: "exit 7"}, time.Millisecond, Options{}, ) - if err == nil { - t.Fatal("failed command should not become a benchmark observation") + if err != nil { + t.Fatal(err) + } + if run.ExitCode != 7 { + t.Fatalf("exit code = %d, want 7", run.ExitCode) } } func TestMeanResidentUsesObservedTimeWeights(t *testing.T) { - metrics := platform.Metrics{ - ResidentByteSeconds: 300, SampleCoverageSeconds: 2, - ResidentByteSamples: 999, SampleCount: 3, - } - if got := sampledMeanResident(metrics); got != 150 { + var samples sampleAggregate + started := time.Unix(0, 0) + samples.observe(platform.Metrics{ResidentBytes: 100}, started) + samples.observe(platform.Metrics{ResidentBytes: 200}, started.Add(time.Second)) + samples.addCoverage(started.Add(2 * time.Second)) + if got := samples.meanResident(); got != 150 { t.Fatalf("weighted mean RSS = %g, want 150", got) } } func TestUpdatePeaksTracksPhysicalFootprint(t *testing.T) { - peak := platform.Metrics{PhysicalFootprintBytes: 100} - updatePeaks(&peak, platform.Metrics{PhysicalFootprintBytes: 250}) - updatePeaks(&peak, platform.Metrics{PhysicalFootprintBytes: 200}) - if peak.PhysicalFootprintBytes != 250 { - t.Fatalf("peak physical footprint = %d, want 250", peak.PhysicalFootprintBytes) + var samples sampleAggregate + samples.observe(platform.Metrics{ + PhysicalFootprintBytes: 100, PhysicalFootprintValid: true, + }, time.Unix(0, 0)) + samples.observe(platform.Metrics{ + PhysicalFootprintBytes: 250, PhysicalFootprintValid: true, + }, time.Unix(1, 0)) + samples.observe(platform.Metrics{ + PhysicalFootprintBytes: 200, PhysicalFootprintValid: true, + }, time.Unix(2, 0)) + if samples.peak.PhysicalFootprintBytes != 250 { + t.Fatalf("peak physical footprint = %d, want 250", samples.peak.PhysicalFootprintBytes) + } +} + +func TestUpdatePeaksInvalidatesPartialPhysicalFootprint(t *testing.T) { + var samples sampleAggregate + samples.observe(platform.Metrics{ + PhysicalFootprintBytes: 100, PhysicalFootprintValid: true, + }, time.Unix(0, 0)) + samples.observe( + platform.Metrics{PhysicalFootprintBytes: 200}, time.Unix(1, 0), + ) + if samples.peak.PhysicalFootprintValid { + t.Fatal("a run with an invalid physical-footprint sample should be unavailable") + } +} + +func TestMonitorUsesInjectedSampler(t *testing.T) { + monitor := startMonitor(42, time.Hour, func(pid int) (platform.Metrics, bool) { + if pid != 42 { + t.Fatalf("sampled PID = %d, want 42", pid) + } + return platform.Metrics{ResidentBytes: 123, Processes: 1}, true + }) + samples := monitor.finish() + if samples.sampleCount != 1 || samples.peak.ResidentBytes != 123 { + t.Fatalf("samples = %+v", samples) + } +} + +func TestMonitorPreservesInvalidPhysicalFootprintSample(t *testing.T) { + monitor := startMonitor(42, time.Hour, func(int) (platform.Metrics, bool) { + return platform.Metrics{PhysicalFootprintInvalid: true}, false + }) + samples := monitor.finish() + if samples.sampleCount != 0 || !samples.peak.PhysicalFootprintInvalid || + samples.peak.PhysicalFootprintValid { + t.Fatalf("invalid sample = %+v", samples) } } diff --git a/internal/runner/spec.go b/internal/runner/spec.go index f11895d..6b71236 100644 --- a/internal/runner/spec.go +++ b/internal/runner/spec.go @@ -2,6 +2,7 @@ package runner import ( "context" + "os" "os/exec" ) @@ -11,9 +12,56 @@ type Spec struct { Shell string } +// preparedSpec executes through the inspected descriptor when executable is +// set, so the bytes that were hashed are provably the bytes that run even if +// the path is swapped underneath us. executablePath is the weaker fallback +// for platforms without descriptor execution (macOS): runs are pinned to the +// inspected absolute path, preventing PATH re-resolution between runs. +// shellTarget means Shell was already rewritten to invoke the pinned target +// as its leading command word. +type preparedSpec struct { + Spec + executable *os.File + executablePath string + shellTarget bool +} + func (spec Spec) command(ctx context.Context) *exec.Cmd { if spec.Shell != "" { return exec.CommandContext(ctx, "/bin/sh", "-c", spec.Shell) } return exec.CommandContext(ctx, spec.Args[0], spec.Args[1:]...) } + +func (spec preparedSpec) command(ctx context.Context) *exec.Cmd { + path := spec.executablePath + if spec.executable != nil { + path = pinnedExecutablePath(spec.executable) + } + var command *exec.Cmd + if spec.Shell != "" { + shell := "/bin/sh" + if path != "" && !spec.shellTarget { + shell = path + } + command = exec.CommandContext(ctx, shell, "-c", spec.Shell) + // argv[0] keeps the conventional name so the measured process and + // anything it reports (ps, error messages) see the usual identity, + // not a descriptor path. + command.Args[0] = "/bin/sh" + } else { + if path == "" { + path = spec.Args[0] + } + command = exec.CommandContext(ctx, path, spec.Args[1:]...) + command.Args[0] = spec.Args[0] + } + if spec.executable != nil { + command.ExtraFiles = pinnedExtraFiles(spec.executable) + } + return command +} + +type commandSpec interface { + command(context.Context) *exec.Cmd +} diff --git a/internal/runner/spec_test.go b/internal/runner/spec_test.go new file mode 100644 index 0000000..60cbc30 --- /dev/null +++ b/internal/runner/spec_test.go @@ -0,0 +1,52 @@ +package runner + +import ( + "context" + "os" + "testing" +) + +func TestPreparedSpecPreservesInvocationIdentity(t *testing.T) { + executable, err := os.Open("/bin/sh") + if err != nil { + t.Fatal(err) + } + defer executable.Close() + + direct := preparedSpec{ + Spec: Spec{Args: []string{"display-name", "argument"}}, executable: executable, + }.command(context.Background()) + if direct.Args[0] != "display-name" || direct.Args[1] != "argument" { + t.Fatalf("direct argv = %v", direct.Args) + } + + shell := preparedSpec{ + Spec: Spec{Shell: "exit 0"}, executable: executable, + }.command(context.Background()) + if shell.Args[0] != "/bin/sh" || shell.Args[1] != "-c" { + t.Fatalf("shell argv = %v", shell.Args) + } + + target := preparedSpec{ + Spec: Spec{Shell: "exec tool"}, executable: executable, shellTarget: true, + }.command(context.Background()) + if target.Path != "/bin/sh" || target.Args[0] != "/bin/sh" { + t.Fatalf("shell target path/argv = %q/%v", target.Path, target.Args) + } +} + +func TestPreparedSpecPinsResolvedPathWithoutDescriptor(t *testing.T) { + direct := preparedSpec{ + Spec: Spec{Args: []string{"tool", "arg"}}, executablePath: "/resolved/tool", + }.command(context.Background()) + if direct.Path != "/resolved/tool" || direct.Args[0] != "tool" { + t.Fatalf("direct path/argv = %q/%v", direct.Path, direct.Args) + } + + shell := preparedSpec{ + Spec: Spec{Shell: "exit 0"}, executablePath: "/resolved/sh", + }.command(context.Background()) + if shell.Path != "/resolved/sh" || shell.Args[0] != "/bin/sh" { + t.Fatalf("shell path/argv = %q/%v", shell.Path, shell.Args) + } +} diff --git a/internal/runner/terminal.go b/internal/runner/terminal.go index 5665c6b..2d4e1d0 100644 --- a/internal/runner/terminal.go +++ b/internal/runner/terminal.go @@ -6,8 +6,10 @@ import ( "os" "os/signal" "syscall" + "time" "github.com/creack/pty" + "golang.org/x/sys/unix" ) func ResolveTerminalSize(width, height uint16) (uint16, uint16) { @@ -29,28 +31,140 @@ func ResolveTerminalSize(width, height uint16) (uint16, uint16) { return size.Cols, size.Rows } -func terminalSize(width, height uint16) *pty.Winsize { - width, height = ResolveTerminalSize(width, height) - return &pty.Winsize{Cols: width, Rows: height} +func terminalSize(input *os.File, width, height uint16) *pty.Winsize { + if input == nil { + input = os.Stdin + } + size := &pty.Winsize{Cols: 80, Rows: 24} + if current, err := pty.GetsizeFull(input); err == nil { + size = current + } + if width > 0 { + size.Cols = width + } + if height > 0 { + size.Rows = height + } + return size } -func followTerminalResize(ctx context.Context, terminal *os.File) { +func followTerminalResize( + ctx context.Context, input *os.File, terminal *os.File, +) <-chan struct{} { resizes := make(chan os.Signal, 1) + done := make(chan struct{}) signal.Notify(resizes, syscall.SIGWINCH) go func() { + defer close(done) defer signal.Stop(resizes) for { select { case <-resizes: - pty.InheritSize(os.Stdin, terminal) + _ = pty.InheritSize(input, terminal) case <-ctx.Done(): return } } }() + return done } func copyTerminalOutput(writer io.Writer, terminal *os.File, done chan<- struct{}) { defer close(done) - io.Copy(writer, terminal) + _, _ = io.Copy(writer, terminal) +} + +func copyInteractiveTerminalOutput( + ctx context.Context, terminalFD, outputFD int, done chan<- struct{}, +) { + defer close(done) + buffer := make([]byte, 4096) + scratch := make([]unix.PollFd, 1) + for { + revents, ok := pollDescriptor(ctx, scratch, terminalFD, unix.POLLIN) + if !ok { + return + } + if revents&unix.POLLIN == 0 { + if revents&(unix.POLLHUP|unix.POLLERR|unix.POLLNVAL) != 0 { + return + } + continue + } + count, err := unix.Read(terminalFD, buffer) + if count > 0 && !writeDescriptor(ctx, scratch, outputFD, buffer[:count]) { + return + } + if err != nil || count == 0 { + return + } + } +} + +func copyTerminalInput( + ctx context.Context, terminal *os.File, inputFD int, done chan<- struct{}, +) { + defer close(done) + buffer := make([]byte, 4096) + scratch := make([]unix.PollFd, 1) + for { + revents, ok := pollDescriptor(ctx, scratch, inputFD, unix.POLLIN) + if !ok { + return + } + if revents&unix.POLLIN == 0 { + if revents&(unix.POLLHUP|unix.POLLERR|unix.POLLNVAL) != 0 { + return + } + continue + } + count, err := unix.Read(inputFD, buffer) + if count > 0 && !writeDescriptor(ctx, scratch, int(terminal.Fd()), buffer[:count]) { + return + } + if err != nil || count == 0 { + return + } + } +} + +func pollDescriptor( + ctx context.Context, scratch []unix.PollFd, fd int, events int16, +) (int16, bool) { + for { + if ctx.Err() != nil { + return 0, false + } + scratch[0] = unix.PollFd{Fd: int32(fd), Events: events} + ready, err := unix.Poll(scratch[:1], int((100 * time.Millisecond).Milliseconds())) + if err == unix.EINTR { + continue + } + if err != nil { + return 0, false + } + if ready > 0 { + return scratch[0].Revents, true + } + } +} + +func writeDescriptor( + ctx context.Context, scratch []unix.PollFd, fd int, data []byte, +) bool { + for len(data) > 0 { + revents, ok := pollDescriptor(ctx, scratch, fd, unix.POLLOUT) + if !ok || revents&(unix.POLLHUP|unix.POLLERR|unix.POLLNVAL) != 0 { + return false + } + count, err := unix.Write(fd, data) + if err == unix.EINTR || err == unix.EAGAIN { + continue + } + if err != nil || count == 0 { + return false + } + data = data[count:] + } + return true } diff --git a/internal/runner/tool.go b/internal/runner/tool.go index 58a185f..e32285c 100644 --- a/internal/runner/tool.go +++ b/internal/runner/tool.go @@ -1,8 +1,10 @@ package runner import ( + "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" "io" "os" @@ -15,27 +17,43 @@ import ( ) type toolInspector struct { - cache map[string]model.ToolInfo - hashCache map[string]string + cache map[string]model.ToolInfo + hashCache map[string]string + fileCache map[string]os.FileInfo + linkedFiles func(context.Context, string) ([]platform.LinkedDependency, error) } func newToolInspector() *toolInspector { + return newToolInspectorWith(platform.LinkedFiles) +} + +func newToolInspectorWith( + linkedFiles func(context.Context, string) ([]platform.LinkedDependency, error), +) *toolInspector { return &toolInspector{ cache: make(map[string]model.ToolInfo), hashCache: make(map[string]string), + fileCache: make(map[string]os.FileInfo), linkedFiles: linkedFiles, } } func inspectTool(spec Spec) (model.ToolInfo, error) { - return newToolInspector().inspect(spec) + return newToolInspector().inspect(context.Background(), spec) } -func (inspector *toolInspector) inspect(spec Spec) (model.ToolInfo, error) { +func (inspector *toolInspector) inspect( + ctx context.Context, spec Spec, +) (model.ToolInfo, error) { + if err := ctx.Err(); err != nil { + return model.ToolInfo{}, err + } tool := model.ToolInfo{Name: spec.Name} executable := "/bin/sh" if spec.Shell != "" { tool.Command = []string{spec.Shell} + tool.ShellCommand = true if candidate := shellExecutable(spec.Shell); candidate != "" { executable = candidate + tool.ShellTarget = true } } else { tool.Command = append([]string(nil), spec.Args...) @@ -45,11 +63,19 @@ func (inspector *toolInspector) inspect(spec Spec) (model.ToolInfo, error) { "find executable %q: %w", spec.Args[0], err, ) } - executable, _ = filepath.Abs(path) + executable = path + if absolute, err := filepath.Abs(path); err == nil { + executable = absolute + } + } + if resolved, err := filepath.EvalSymlinks(executable); err == nil { + executable = resolved } tool.Executable = executable if cached, ok := inspector.cache[executable]; ok { cached.Name, cached.Command = tool.Name, tool.Command + cached.ShellCommand = tool.ShellCommand + cached.ShellTarget = tool.ShellTarget return cached, nil } info, err := os.Stat(executable) @@ -59,7 +85,12 @@ func (inspector *toolInspector) inspect(spec Spec) (model.ToolInfo, error) { ) } tool.SizeBytes = info.Size() - for _, dependency := range platform.LinkedFiles(executable) { + inspector.fileCache[executable] = info + dependencies, err := inspector.linkedFiles(ctx, executable) + if err != nil { + return model.ToolInfo{}, err + } + for _, dependency := range dependencies { if dependency.SharedCache { tool.SharedCacheFiles = append(tool.SharedCacheFiles, dependency.Path) continue @@ -78,28 +109,102 @@ func (inspector *toolInspector) inspect(spec Spec) (model.ToolInfo, error) { return tool, nil } -func (inspector *toolInspector) addHash(tool *model.ToolInfo) error { +func (inspector *toolInspector) pin( + ctx context.Context, tool *model.ToolInfo, +) (*os.File, error) { + file, err := os.Open(tool.Executable) + if err != nil { + return nil, err + } + info, err := file.Stat() + if err != nil { + file.Close() + return nil, err + } + inspected := inspector.fileCache[tool.Executable] + if inspected == nil || !os.SameFile(inspected, info) { + file.Close() + return nil, errors.New("executable changed while being inspected") + } + if inspected.Size() != info.Size() || !inspected.ModTime().Equal(info.ModTime()) { + file.Close() + return nil, errors.New("executable contents changed while being inspected") + } if hash, ok := inspector.hashCache[tool.Executable]; ok { tool.SHA256 = hash - return nil + } else { + hash, hashErr := hashOpenFile(ctx, file) + if hashErr != nil { + file.Close() + return nil, hashErr + } + tool.SHA256 = hash + inspector.hashCache[tool.Executable] = hash } - file, err := os.Open(tool.Executable) + if _, err := file.Seek(0, io.SeekStart); err != nil { + file.Close() + return nil, err + } + return file, nil +} + +func (inspector *toolInspector) verify( + ctx context.Context, tool model.ToolInfo, file *os.File, +) error { + before, ok := inspector.fileCache[tool.Executable] + if !ok { + return errors.New("missing inspected executable identity") + } + after, err := os.Stat(tool.Executable) if err != nil { return err } - defer file.Close() - hash := sha256.New() - if _, err := io.Copy(hash, file); err != nil { + if !os.SameFile(before, after) { + return errors.New("executable was replaced during measurement") + } + if _, err := file.Seek(0, io.SeekStart); err != nil { return err } - tool.SHA256 = hex.EncodeToString(hash.Sum(nil)) - inspector.hashCache[tool.Executable] = tool.SHA256 + hash, err := hashOpenFile(ctx, file) + if err != nil { + return err + } + if hash != tool.SHA256 { + return errors.New("executable changed during measurement") + } return nil } +func hashOpenFile(ctx context.Context, file *os.File) (string, error) { + hash := sha256.New() + buffer := make([]byte, 128*1024) + for { + if err := ctx.Err(); err != nil { + return "", err + } + count, readErr := file.Read(buffer) + if count > 0 { + _, _ = hash.Write(buffer[:count]) + } + if errors.Is(readErr, io.EOF) { + return hex.EncodeToString(hash.Sum(nil)), nil + } + if readErr != nil { + return "", readErr + } + } +} + +// shellSpecial lists characters that give the leading word of a shell command +// meaning beyond a plain executable name β€” quoting, operators, expansions, +// globs, assignments, comments. Such words are never treated as pinnable +// targets: substituting them could change what the shell would have run. +const shellSpecial = "'\"|&;<>()$`*?[~=#" + func shellExecutable(command string) string { fields := strings.Fields(command) - if len(fields) == 0 || strings.ContainsAny(fields[0], "'\"|&;<>()$`") { + if len(fields) == 0 || shellBuiltin(fields[0]) || + strings.ContainsAny(fields[0], shellSpecial) { return "" } path, err := exec.LookPath(fields[0]) @@ -107,8 +212,48 @@ func shellExecutable(command string) string { return "" } absolute, err := filepath.Abs(path) - if err != nil { - return path + if err == nil { + path = absolute + } + if resolved, err := filepath.EvalSymlinks(path); err == nil { + path = resolved + } + return path +} + +var shellBuiltins = map[string]bool{ + "!": true, ".": true, ":": true, "[": true, "alias": true, + "bg": true, "break": true, "cd": true, "command": true, + "continue": true, "echo": true, "eval": true, "exec": true, + "exit": true, "export": true, "false": true, "fc": true, + "fg": true, "getopts": true, "hash": true, "jobs": true, + "kill": true, "printf": true, "pwd": true, "read": true, + "readonly": true, "return": true, "set": true, "shift": true, + "test": true, "time": true, "times": true, "trap": true, + "true": true, "type": true, "ulimit": true, "umask": true, + "unalias": true, "unset": true, "wait": true, "{": true, + "builtin": true, "declare": true, "local": true, "source": true, + "typeset": true, +} + +func shellBuiltin(name string) bool { return shellBuiltins[name] } + +// pinShellExecutable swaps the leading command word for the pinned descriptor +// path so /bin/sh execs the inspected bytes; it declines when the word carries +// shell syntax that substitution could change the meaning of. +func pinShellExecutable(command, executable string) (string, bool) { + trimmed := strings.TrimLeft(command, " \t\r\n") + if trimmed == "" { + return command, false + } + end := strings.IndexAny(trimmed, " \t\r\n") + if end < 0 { + end = len(trimmed) + } + if strings.ContainsAny(trimmed[:end], shellSpecial) { + return command, false } - return absolute + prefix := command[:len(command)-len(trimmed)] + quoted := "'" + strings.ReplaceAll(executable, "'", `'"'"'`) + "'" + return prefix + quoted + trimmed[end:], true } diff --git a/internal/runner/tool_test.go b/internal/runner/tool_test.go index e2f076f..e33ea3e 100644 --- a/internal/runner/tool_test.go +++ b/internal/runner/tool_test.go @@ -1,19 +1,137 @@ package runner import ( + "context" + "errors" + "os" + "os/exec" "path/filepath" "testing" + + "github.com/shellcell/snailrace/internal/platform" ) -func TestShellExecutableFindsSimpleCommand(t *testing.T) { - got := shellExecutable("sh -c true") - if filepath.Base(got) != "sh" { - t.Fatalf("executable = %q, want sh", got) +func TestInspectionHonorsCancelledContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := newToolInspector().inspect( + ctx, Spec{Name: "true", Args: []string{"true"}}, + ); err == nil { + t.Fatal("cancelled inspection should fail") + } +} + +func TestToolInspectorUsesInjectedDependencyDiscovery(t *testing.T) { + calls := 0 + inspector := newToolInspectorWith(func( + context.Context, string, + ) ([]platform.LinkedDependency, error) { + calls++ + return nil, nil + }) + for _, name := range []string{"first", "second"} { + if _, err := inspector.inspect( + context.Background(), Spec{Name: name, Args: []string{"true"}}, + ); err != nil { + t.Fatal(err) + } + } + if calls != 1 { + t.Fatalf("dependency discovery calls = %d, want 1 cached call", calls) + } + expected := errors.New("dependency failure") + inspector = newToolInspectorWith(func( + context.Context, string, + ) ([]platform.LinkedDependency, error) { + return nil, expected + }) + if _, err := inspector.inspect( + context.Background(), Spec{Name: "tool", Args: []string{"true"}}, + ); !errors.Is(err, expected) { + t.Fatalf("dependency error = %v, want %v", err, expected) + } +} + +func TestShellInspectionReportsTargetExecutable(t *testing.T) { + tool, err := inspectTool(Spec{Name: "shell", Shell: "sleep 0"}) + if err != nil { + t.Fatal(err) + } + want := shellExecutable("sleep 0") + if tool.Executable != want { + t.Fatalf("shell target executable = %q, want %q", tool.Executable, want) + } +} + +func TestShellInspectionPreservesBuiltins(t *testing.T) { + tool, err := inspectTool(Spec{Name: "shell", Shell: "printf test"}) + if err != nil { + t.Fatal(err) + } + want, err := filepath.EvalSymlinks("/bin/sh") + if err != nil { + t.Fatal(err) + } + if tool.Executable != want || tool.ShellTarget { + t.Fatalf("builtin inspection = %+v, want shell executable", tool) } } -func TestShellExecutableRejectsCompoundCommand(t *testing.T) { - if got := shellExecutable("'quoted tool' --flag"); got != "" { - t.Fatalf("executable = %q, want empty", got) +func TestPinShellExecutablePreservesArguments(t *testing.T) { + got, ok := pinShellExecutable(" curl --url 'https://example.com/a b'", "/proc/self/fd/3") + want := " '/proc/self/fd/3' --url 'https://example.com/a b'" + if !ok || got != want { + t.Fatalf("pinned command = %q, %v; want %q, true", got, ok, want) + } +} + +func TestShellTargetRejectsExpansionSyntax(t *testing.T) { + commands := []string{ + "FOO=bar curl url", "curl* url", "cur?l url", "[tool] x", + "~/bin/tool run", "#comment", "$TOOL run", + } + for _, command := range commands { + if got := shellExecutable(command); got != "" { + t.Errorf("shellExecutable(%q) = %q, want rejection", command, got) + } + if _, ok := pinShellExecutable(command, "/proc/self/fd/3"); ok { + t.Errorf("pinShellExecutable(%q) accepted expansion syntax", command) + } + } +} + +func TestToolVerificationDetectsExecutableReplacement(t *testing.T) { + path := filepath.Join(t.TempDir(), "tool") + source, err := exec.LookPath("true") + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(source) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o755); err != nil { + t.Fatal(err) + } + inspector := newToolInspector() + tool, err := inspector.inspect( + context.Background(), Spec{Name: "tool", Args: []string{path}}, + ) + if err != nil { + t.Fatal(err) + } + file, err := inspector.pin(context.Background(), &tool) + if err != nil { + t.Fatal(err) + } + defer file.Close() + if err := os.Rename(path, path+".old"); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o755); err != nil { + t.Fatal(err) + } + if err := inspector.verify(context.Background(), tool, file); err == nil { + t.Fatal("replacement executable should fail provenance verification") } } diff --git a/internal/runner/tui.go b/internal/runner/tui.go index 2bf217f..f7914e4 100644 --- a/internal/runner/tui.go +++ b/internal/runner/tui.go @@ -5,7 +5,6 @@ import ( "errors" "io" "os" - "os/exec" "syscall" "time" @@ -18,25 +17,37 @@ import ( func runTUIOnce( ctx context.Context, - spec Spec, + spec commandSpec, interval time.Duration, options Options, ) (model.Run, error) { - if options.Interactive && (!term.IsTerminal(int(os.Stdin.Fd())) || - !term.IsTerminal(int(os.Stdout.Fd()))) { + input := options.Input + if input == nil { + input = os.Stdin + } + output := options.TerminalOut + if output == nil { + output = os.Stdout + } + inputTerminal, inputIsTerminal := input.(interface{ Fd() uintptr }) + outputTerminal, outputIsTerminal := output.(interface{ Fd() uintptr }) + inputFile, inputIsFile := input.(*os.File) + if options.Interactive && (!inputIsTerminal || !outputIsTerminal || !inputIsFile || + !term.IsTerminal(int(inputTerminal.Fd())) || + !term.IsTerminal(int(outputTerminal.Fd()))) { return model.Run{}, errors.New( "interactive TUI mode requires terminal stdin and stdout", ) } - size := terminalSize(options.Width, options.Height) + size := terminalSize(inputFile, options.Width, options.Height) var state *term.State var err error if options.Interactive { - state, err = term.MakeRaw(int(os.Stdin.Fd())) + state, err = term.MakeRaw(int(inputTerminal.Fd())) if err != nil { return model.Run{}, err } - defer term.Restore(int(os.Stdin.Fd()), state) + defer term.Restore(int(inputTerminal.Fd()), state) } cmd := spec.command(ctx) @@ -46,64 +57,91 @@ func runTUIOnce( return model.Run{}, err } defer terminal.Close() - resizeContext, stopResize := context.WithCancel(ctx) - defer stopResize() + inputContext, stopInput := context.WithCancel(ctx) + outputContext, stopOutput := context.WithCancel(ctx) + defer stopInput() + defer stopOutput() outputDone := make(chan struct{}) + var inputDone, resizeDone <-chan struct{} if options.Interactive { - go io.Copy(terminal, os.Stdin) - go copyTerminalOutput(os.Stdout, terminal, outputDone) + inputFinished := make(chan struct{}) + inputDone = inputFinished + go copyTerminalInput( + inputContext, terminal, int(inputTerminal.Fd()), inputFinished, + ) + go copyInteractiveTerminalOutput( + outputContext, int(terminal.Fd()), int(outputTerminal.Fd()), outputDone, + ) if options.FollowResize { - followTerminalResize(resizeContext, terminal) + resizeDone = followTerminalResize(inputContext, inputFile, terminal) } } else { go copyTerminalOutput(io.Discard, terminal, outputDone) } - stopMonitor := make(chan struct{}) - monitorDone := make(chan platform.Metrics, 1) - go monitor(cmd.Process.Pid, interval, stopMonitor, monitorDone) + groupID := cmd.Process.Pid + monitor := startMonitor(groupID, interval, platform.NewGroupSampler().Sample) waitDone := make(chan error, 1) go func() { waitDone <- cmd.Wait() }() - waitErr, elapsed, reason := waitForTUI( - ctx, cmd, options.Duration, started, waitDone, + waitResult := waitForTUI( + ctx, groupID, options.Duration, started, waitDone, ) - signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) - close(stopMonitor) - peak := <-monitorDone + signalProcessGroup(groupID, syscall.SIGKILL) + samples := monitor.finish() + stopInput() + if inputDone != nil { + <-inputDone + } + if resizeDone != nil { + <-resizeDone + } + select { + case <-outputDone: + case <-time.After(100 * time.Millisecond): + stopOutput() + _ = terminal.Close() + <-outputDone + } user, system, rusageRSS := platform.ResourceUsage(cmd.ProcessState) - peak.Processes = max(peak.Processes, 1) - peak.Threads = max(peak.Threads, 1) if ctx.Err() != nil { return model.Run{}, ctx.Err() } - if options.Duration > 0 && reason == "exited" { + if options.Duration > 0 && waitResult.reason == "exited" { return model.Run{}, errors.New("TUI exited before the fixed duration") } - if waitErr != nil && reason == "exited" { - return model.Run{}, waitErr + if waitResult.err != nil && waitResult.reason == "exited" && + !isNonZeroExit(cmd, waitResult.err) { + return model.Run{}, waitResult.err } - select { - case <-outputDone: - case <-time.After(100 * time.Millisecond): - } - return makeTUIRun(cmd, elapsed, user, system, rusageRSS, peak, reason), nil + return makeRun( + cmd.ProcessState, waitResult.elapsed, user, system, rusageRSS, + samples, waitResult.reason, + ), nil +} + +type tuiWaitResult struct { + err error + elapsed time.Duration + reason string } func waitForTUI( ctx context.Context, - cmd *exec.Cmd, + groupID int, duration time.Duration, started time.Time, waitDone <-chan error, -) (error, time.Duration, string) { +) tuiWaitResult { if duration == 0 { select { case err := <-waitDone: - return err, time.Since(started), "exited" + return tuiWaitResult{err: err, elapsed: time.Since(started), reason: "exited"} case <-ctx.Done(): - signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) - return <-waitDone, time.Since(started), "interrupted" + signalProcessGroup(groupID, syscall.SIGKILL) + return tuiWaitResult{ + err: <-waitDone, elapsed: time.Since(started), reason: "interrupted", + } } } remaining := time.Until(started.Add(duration)) @@ -114,18 +152,15 @@ func waitForTUI( defer timer.Stop() select { case err := <-waitDone: - return err, time.Since(started), "exited" + return tuiWaitResult{err: err, elapsed: time.Since(started), reason: "exited"} case <-ctx.Done(): - signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) - return <-waitDone, time.Since(started), "interrupted" - case <-timer.C: - signalProcessGroup(cmd.Process.Pid, syscall.SIGTERM) - select { - case err := <-waitDone: - return err, time.Since(started), "duration" - case <-time.After(time.Second): - signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) - return <-waitDone, time.Since(started), "duration" + signalProcessGroup(groupID, syscall.SIGKILL) + return tuiWaitResult{ + err: <-waitDone, elapsed: time.Since(started), reason: "interrupted", } + case <-timer.C: + elapsed := time.Since(started) + signalProcessGroup(groupID, syscall.SIGKILL) + return tuiWaitResult{err: <-waitDone, elapsed: elapsed, reason: "duration"} } } diff --git a/internal/runner/tui_result.go b/internal/runner/tui_result.go index e964f5e..10c27d4 100644 --- a/internal/runner/tui_result.go +++ b/internal/runner/tui_result.go @@ -1,35 +1,45 @@ package runner import ( - "os/exec" + "os" "time" "github.com/shellcell/snailrace/internal/model" - "github.com/shellcell/snailrace/internal/platform" ) -func makeTUIRun( - cmd *exec.Cmd, +func makeRun( + state *os.ProcessState, elapsed time.Duration, user, system float64, rusageRSS uint64, - peak platform.Metrics, + samples sampleAggregate, reason string, ) model.Run { + peak := samples.peak + peak.Processes = max(peak.Processes, 1) + peak.Threads = max(peak.Threads, 1) return model.Run{ - ExitCode: cmd.ProcessState.ExitCode(), WallSeconds: elapsed.Seconds(), + ExitCode: state.ExitCode(), WallSeconds: elapsed.Seconds(), CPUUserSeconds: user, CPUSystemSeconds: system, AverageCPUPercent: averageCPUPercent(user, system, elapsed), PeakResidentBytes: float64(peak.ResidentBytes), PeakPhysicalFootprintBytes: float64(peak.PhysicalFootprintBytes), + PhysicalFootprintValid: peak.PhysicalFootprintValid, OSMaxRSSBytes: float64(rusageRSS), - MeanResidentBytes: sampledMeanResident(peak), + MeanResidentBytes: samples.meanResident(), PeakVirtualBytes: float64(peak.VirtualBytes), PeakProcesses: float64(peak.Processes), PeakThreads: float64(peak.Threads), PeakFileDescriptors: float64(peak.FileDescriptors), StopReason: reason, - SampleCount: int(peak.SampleCount), - SampleCoverageSeconds: peak.SampleCoverageSeconds, + SampleCount: int(samples.sampleCount), + SampleCoverageSeconds: samples.sampleCoverageSeconds, } } + +func averageCPUPercent(user, system float64, elapsed time.Duration) float64 { + if elapsed <= 0 { + return 0 + } + return (user + system) / elapsed.Seconds() * 100 +} diff --git a/internal/runner/tui_test.go b/internal/runner/tui_test.go index 3763e19..6c0fd08 100644 --- a/internal/runner/tui_test.go +++ b/internal/runner/tui_test.go @@ -2,8 +2,12 @@ package runner import ( "context" + "os" "testing" "time" + + "github.com/creack/pty" + "golang.org/x/sys/unix" ) func TestFixedDurationTUIStopsProcessGroup(t *testing.T) { @@ -27,7 +31,7 @@ func TestFixedDurationTUIStopsProcessGroup(t *testing.T) { func TestFixedDurationTUIRejectsEarlyExit(t *testing.T) { _, err := runTUIOnce( - context.Background(), Spec{Name: "true", Args: []string{"/bin/true"}}, + context.Background(), Spec{Name: "true", Args: []string{"true"}}, time.Millisecond, Options{TUI: true, Duration: 50 * time.Millisecond}, ) if err == nil { @@ -35,9 +39,163 @@ func TestFixedDurationTUIRejectsEarlyExit(t *testing.T) { } } +func TestFixedDurationTUIExcludesTerminationDelay(t *testing.T) { + duration := 50 * time.Millisecond + started := time.Now() + run, err := runTUIOnce( + context.Background(), + Spec{Name: "ignore-term", Shell: "trap '' TERM; while :; do :; done"}, + 5*time.Millisecond, + Options{TUI: true, Duration: duration}, + ) + if err != nil { + t.Fatal(err) + } + if runtime := time.Since(started); runtime > 2*time.Second { + t.Fatalf("fixed-duration run took %s, teardown should be outside measurement", runtime) + } + if run.WallSeconds < duration.Seconds() || run.WallSeconds > 0.5 { + t.Fatalf("wall time = %.3fs, want approximately %.3fs", run.WallSeconds, duration.Seconds()) + } +} + +func TestTUIRecordsNonZeroExit(t *testing.T) { + run, err := runTUIOnce( + context.Background(), Spec{Name: "exit", Shell: "exit 7"}, + time.Millisecond, Options{TUI: true}, + ) + if err != nil { + t.Fatal(err) + } + if run.ExitCode != 7 { + t.Fatalf("exit code = %d, want 7", run.ExitCode) + } +} + func TestTerminalSizeOverridesDimensions(t *testing.T) { width, height := ResolveTerminalSize(132, 43) if width != 132 || height != 43 { t.Fatalf("terminal size = %dx%d, want 132x43", width, height) } } + +func TestTerminalInputPumpStopsWithoutInput(t *testing.T) { + input, inputWriter, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer input.Close() + defer inputWriter.Close() + terminal, terminalReader, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer terminal.Close() + defer terminalReader.Close() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go copyTerminalInput(ctx, terminal, int(input.Fd()), done) + cancel() + select { + case <-done: + case <-time.After(500 * time.Millisecond): + t.Fatal("terminal input pump survived cancellation") + } +} + +func TestInteractiveTUIUsesInjectedTerminalStreams(t *testing.T) { + input, output, err := pty.Open() + if err != nil { + t.Fatal(err) + } + defer input.Close() + defer output.Close() + _, err = runTUIOnce( + context.Background(), Spec{Name: "sleep", Shell: "sleep 30"}, + 5*time.Millisecond, Options{ + TUI: true, Interactive: true, Duration: 30 * time.Millisecond, + Input: input, TerminalOut: output, + }, + ) + if err != nil { + t.Fatal(err) + } +} + +func TestTerminalInputPumpCancelsWhileDestinationIsFull(t *testing.T) { + input, inputWriter, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer input.Close() + defer inputWriter.Close() + terminalReader, terminal, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer terminalReader.Close() + defer terminal.Close() + fillPipe(t, terminal) + if _, err := inputWriter.Write([]byte("input")); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go copyTerminalInput(ctx, terminal, int(input.Fd()), done) + time.Sleep(20 * time.Millisecond) + cancel() + select { + case <-done: + case <-time.After(500 * time.Millisecond): + t.Fatal("terminal input pump blocked on a full destination") + } +} + +func TestTerminalOutputPumpCancelsWhileDestinationIsFull(t *testing.T) { + source, sourceWriter, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer source.Close() + defer sourceWriter.Close() + outputReader, output, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer outputReader.Close() + defer output.Close() + fillPipe(t, output) + if _, err := sourceWriter.Write([]byte("output")); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go copyInteractiveTerminalOutput( + ctx, int(source.Fd()), int(output.Fd()), done, + ) + time.Sleep(20 * time.Millisecond) + cancel() + select { + case <-done: + case <-time.After(500 * time.Millisecond): + t.Fatal("terminal output pump blocked on a full destination") + } +} + +func fillPipe(t *testing.T, pipe *os.File) { + t.Helper() + buffer := []byte{0} + for { + poll := []unix.PollFd{{Fd: int32(pipe.Fd()), Events: unix.POLLOUT}} + ready, err := unix.Poll(poll, 0) + if err != nil { + t.Fatal(err) + } + if ready == 0 { + return + } + if _, err := unix.Write(int(pipe.Fd()), buffer); err != nil { + t.Fatal(err) + } + } +} diff --git a/internal/runner/validation.go b/internal/runner/validation.go new file mode 100644 index 0000000..81537dc --- /dev/null +++ b/internal/runner/validation.go @@ -0,0 +1,81 @@ +package runner + +import ( + "context" + "errors" + "fmt" + "strings" +) + +func validateBenchmarkInputs(ctx context.Context, specs []Spec, config Config) error { + if ctx == nil { + return errors.New("context cannot be nil") + } + if len(specs) == 0 { + return errors.New("at least one command is required") + } + if config.Runs < 1 { + return errors.New("runs must be at least 1") + } + if config.Warmups < 0 { + return errors.New("warmups cannot be negative") + } + if config.Interval <= 0 { + return errors.New("sampling interval must be positive") + } + for index, spec := range specs { + if strings.TrimSpace(spec.Name) == "" { + return fmt.Errorf("command %d name cannot be blank", index+1) + } + if spec.Shell != "" && strings.TrimSpace(spec.Shell) == "" { + return fmt.Errorf("command %d shell source cannot be blank", index+1) + } + hasShell := spec.Shell != "" + hasArgs := len(spec.Args) > 0 && strings.TrimSpace(spec.Args[0]) != "" + if hasShell == hasArgs { + return fmt.Errorf("command %d must define either shell source or argv", index+1) + } + } + if len(config.MeasurementOrder) > 0 && len(config.MeasurementOrder) != config.Runs { + return fmt.Errorf( + "measurement order has %d rounds, want %d", + len(config.MeasurementOrder), config.Runs, + ) + } + if len(config.MeasurementOrder) == config.Runs { + if err := validateSchedule(config.MeasurementOrder, len(specs)); err != nil { + return fmt.Errorf("measurement order: %w", err) + } + } + if len(config.WarmupOrder) > 0 && len(config.WarmupOrder) != config.Warmups { + return fmt.Errorf( + "warmup order has %d rounds, want %d", + len(config.WarmupOrder), config.Warmups, + ) + } + if len(config.WarmupOrder) == config.Warmups { + if err := validateSchedule(config.WarmupOrder, len(specs)); err != nil { + return fmt.Errorf("warmup order: %w", err) + } + } + return nil +} + +func validateSchedule(schedule [][]int, tools int) error { + for round, order := range schedule { + if len(order) != tools { + return fmt.Errorf("round %d has %d tools, want %d", round+1, len(order), tools) + } + seen := make([]bool, tools) + for _, index := range order { + if index < 0 || index >= tools { + return fmt.Errorf("round %d contains tool index %d", round+1, index) + } + if seen[index] { + return fmt.Errorf("round %d repeats tool index %d", round+1, index) + } + seen[index] = true + } + } + return nil +} diff --git a/internal/style/command.go b/internal/style/command.go new file mode 100644 index 0000000..e52500e --- /dev/null +++ b/internal/style/command.go @@ -0,0 +1,41 @@ +package style + +import ( + "strconv" + "strings" + "unicode" +) + +func CommandText(command []string, shell bool) string { + if len(command) == 0 { + return "" + } + if shell { + return safeCommandText(command[0]) + } + arguments := make([]string, len(command)) + for index, argument := range command { + arguments[index] = quoteArgument(argument) + } + return strings.Join(arguments, " ") +} + +func quoteArgument(argument string) string { + if strings.IndexFunc(argument, unicode.IsControl) >= 0 { + return strconv.QuoteToGraphic(argument) + } + if argument != "" && strings.IndexFunc(argument, func(character rune) bool { + return !(unicode.IsLetter(character) || unicode.IsDigit(character) || + strings.ContainsRune("_@%+=:,./-", character)) + }) < 0 { + return argument + } + return "'" + strings.ReplaceAll(argument, "'", `'"'"'`) + "'" +} + +func safeCommandText(command string) string { + if strings.IndexFunc(command, unicode.IsControl) >= 0 { + return strconv.QuoteToGraphic(command) + } + return command +} diff --git a/internal/style/command_test.go b/internal/style/command_test.go new file mode 100644 index 0000000..3bb1ddc --- /dev/null +++ b/internal/style/command_test.go @@ -0,0 +1,14 @@ +package style + +import "testing" + +func TestCommandTextQuotesArgumentsWithoutChangingShellCommands(t *testing.T) { + if got, want := CommandText( + []string{"tool", "two words", "", "it's", "line\n"}, false, + ), `tool 'two words' '' 'it'"'"'s' "line\n"`; got != want { + t.Fatalf("command = %q, want %q", got, want) + } + if got := CommandText([]string{"printf '%s' value"}, true); got != "printf '%s' value" { + t.Fatalf("shell command = %q", got) + } +} diff --git a/internal/style/palette.go b/internal/style/palette.go index fbfca47..8f8d115 100644 --- a/internal/style/palette.go +++ b/internal/style/palette.go @@ -15,5 +15,9 @@ var toolPalette = [...]Color{ } func Tool(index int) Color { - return toolPalette[index%len(toolPalette)] + index %= len(toolPalette) + if index < 0 { + index += len(toolPalette) + } + return toolPalette[index] } diff --git a/internal/style/palette_test.go b/internal/style/palette_test.go new file mode 100644 index 0000000..b3ceaec --- /dev/null +++ b/internal/style/palette_test.go @@ -0,0 +1,9 @@ +package style + +import "testing" + +func TestToolColorHandlesNegativeIndex(t *testing.T) { + if got, want := Tool(-1), Tool(len(toolPalette)-1); got != want { + t.Fatalf("negative color = %+v, want %+v", got, want) + } +} diff --git a/internal/style/text.go b/internal/style/text.go new file mode 100644 index 0000000..5886958 --- /dev/null +++ b/internal/style/text.go @@ -0,0 +1,20 @@ +package style + +import ( + "strconv" + "strings" + "unicode" +) + +func SafeText(value string) string { + var result strings.Builder + for _, character := range value { + if !unicode.IsControl(character) { + result.WriteRune(character) + continue + } + quoted := strconv.QuoteRuneToGraphic(character) + result.WriteString(quoted[1 : len(quoted)-1]) + } + return result.String() +} diff --git a/internal/style/width.go b/internal/style/width.go new file mode 100644 index 0000000..f33f239 --- /dev/null +++ b/internal/style/width.go @@ -0,0 +1,33 @@ +package style + +import "unicode" + +// wideRanges lists Unicode blocks that monospace terminals render as two +// cells: East Asian Wide and Fullwidth blocks plus the common emoji planes. +var wideRanges = [][2]rune{ + {0x1100, 0x115F}, {0x2329, 0x232A}, {0x2E80, 0x303E}, {0x3041, 0x33FF}, + {0x3400, 0x4DBF}, {0x4E00, 0x9FFF}, {0xA000, 0xA4CF}, {0xA960, 0xA97F}, + {0xAC00, 0xD7A3}, {0xF900, 0xFAFF}, {0xFE10, 0xFE19}, {0xFE30, 0xFE6F}, + {0xFF00, 0xFF60}, {0xFFE0, 0xFFE6}, {0x1F300, 0x1F64F}, {0x1F680, 0x1F6FF}, + {0x1F900, 0x1FAFF}, {0x20000, 0x2FFFD}, {0x30000, 0x3FFFD}, +} + +func RuneWidth(character rune) int { + if unicode.In(character, unicode.Mn, unicode.Me, unicode.Cf) { + return 0 + } + for _, span := range wideRanges { + if character >= span[0] && character <= span[1] { + return 2 + } + } + return 1 +} + +func DisplayWidth(value string) int { + width := 0 + for _, character := range value { + width += RuneWidth(character) + } + return width +} diff --git a/internal/style/width_test.go b/internal/style/width_test.go new file mode 100644 index 0000000..1755c36 --- /dev/null +++ b/internal/style/width_test.go @@ -0,0 +1,24 @@ +package style + +import "testing" + +func TestDisplayWidth(t *testing.T) { + tests := []struct { + value string + width int + }{ + {"", 0}, + {"curl #1", 7}, + {"🐌", 2}, + {"🏁", 2}, + {"ζ€§θƒ½γƒ†γ‚Ήγƒˆ", 10}, + {"ν•œκΈ€", 4}, + {"café", 4}, + {"a‍b", 2}, + } + for _, test := range tests { + if got := DisplayWidth(test.value); got != test.width { + t.Errorf("DisplayWidth(%q) = %d, want %d", test.value, got, test.width) + } + } +}