From 00ce0b7000d3ead77179741c7f54dca64ef86b60 Mon Sep 17 00:00:00 2001 From: CodSpeed Bot Date: Mon, 17 Aug 2026 12:02:10 +0000 Subject: [PATCH] add continuous benchmarking with CodSpeed Add a CodSpeed workflow that runs the Go benchmarks on every pull request and push to master, using the walltime instrument. Next to the benchmarks that already existed, add benchmarks for some of the hot paths of the CLI: - constructing the command-tree (done on every invocation of the CLI) - loading the CLI configuration file - parsing "docker run" flags - parsing "--mount" flags - formatting "docker ps" and "docker image ls" output - parsing and executing "--format" templates --- .github/workflows/codspeed.yml | 66 +++++++++++++++++++++++++ README.md | 1 + TESTING.md | 30 +++++++++++ cli/command/commands/commands_test.go | 20 ++++++++ cli/command/container/opts_test.go | 55 +++++++++++++++++++++ cli/command/formatter/container_test.go | 63 +++++++++++++++++++++++ cli/command/formatter/image_test.go | 57 +++++++++++++++++++++ cli/config/config_test.go | 36 ++++++++++++++ opts/mount_test.go | 22 +++++++++ templates/templates_test.go | 41 +++++++++++++++ 10 files changed, 391 insertions(+) create mode 100644 .github/workflows/codspeed.yml create mode 100644 cli/command/commands/commands_test.go diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 000000000000..f44158d39728 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,66 @@ +name: codspeed + +# Default to 'contents: read', which grants actions to read commits. +# +# If any permission is set, any permission not included in the list is +# implicitly set to "none". +# +# see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + # "workflow_dispatch" also allows CodSpeed to trigger backtest performance + # analysis to generate initial data. + workflow_dispatch: + push: + branches: + - 'master' + - '[0-9]+.[0-9]+' + - '[0-9]+.x' + tags: + - 'v*' + pull_request: + +jobs: + benchmark: + runs-on: ubuntu-24.04 + permissions: + contents: read # required for actions/checkout + id-token: write # required for OIDC authentication with CodSpeed + steps: + - + name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - + name: Set up Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: "1.26.4" + cache: false + - + name: Prepare + run: | + # run in go modules mode to prevent traversing to nested modules + ln -s vendor.mod go.mod + ln -s vendor.sum go.sum + - + name: Run benchmarks + uses: CodSpeedHQ/action@4296e51e7041e24dadb86d1d6e8b9320d223dbe8 # v5.0.3 + with: + mode: walltime + run: | + go test -bench=. \ + ./cli/command/commands/ \ + ./cli/command/container/ \ + ./cli/command/formatter/ \ + ./cli/command/system/ \ + ./cli/config/ \ + ./opts/ \ + ./templates/ diff --git a/README.md b/README.md index 1183834102b4..c1b7f0c92c3c 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/docker/cli)](https://goreportcard.com/report/github.com/docker/cli) [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/docker/cli/badge)](https://scorecard.dev/viewer/?uri=github.com/docker/cli) [![Codecov](https://img.shields.io/codecov/c/github/docker/cli?logo=codecov)](https://codecov.io/gh/docker/cli) +[![CodSpeed](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://app.codspeed.io/maksimtech/cli?utm_source=badge) ## About diff --git a/TESTING.md b/TESTING.md index ab1a09e32db7..9918c695f14c 100644 --- a/TESTING.md +++ b/TESTING.md @@ -32,6 +32,36 @@ Fakes, and testing utilities can be found in [internal/test](https://godoc.org/github.com/docker/cli/internal/test) and [gotest.tools](https://godoc.org/gotest.tools). +## Benchmarks + +Performance sensitive code should be covered by benchmarks. Benchmarks use the +standard Go [testing](https://pkg.go.dev/testing#hdr-Benchmarks) conventions and +live next to the unit tests in `_test.go` files, named using the convention: + +``` +Benchmark[] +``` + +Prefer [`b.Loop()`](https://pkg.go.dev/testing#B.Loop) over `for i := 0; i < b.N; i++`, +and call `b.ReportAllocs()` to keep track of allocations. + +Benchmarks can be run locally with: + +```shell +go test -bench=. ./templates/ +``` + +Benchmarks are also run continuously in CI through +[CodSpeed](https://app.codspeed.io/maksimtech/cli), which reports the +performance impact of a pull request. The packages that are benchmarked in CI +are listed in [.github/workflows/codspeed.yml](.github/workflows/codspeed.yml). +To run them the same way CodSpeed does, install the +[CodSpeed CLI](https://codspeed.io/docs/cli) and run: + +```shell +codspeed run --skip-upload --mode walltime -- go test -bench=. ./templates/ +``` + ## End-to-End Test Suite The end-to-end test suite tests a cli binary against a real API backend. diff --git a/cli/command/commands/commands_test.go b/cli/command/commands/commands_test.go new file mode 100644 index 000000000000..bb11b56bd885 --- /dev/null +++ b/cli/command/commands/commands_test.go @@ -0,0 +1,20 @@ +package commands + +import ( + "testing" + + "github.com/docker/cli/internal/test" + "github.com/spf13/cobra" +) + +// BenchmarkAddCommands measures the cost of constructing the command-tree. +// The command-tree is constructed on every invocation of the CLI, and as +// part of generating shell-completion scripts and documentation. +func BenchmarkAddCommands(b *testing.B) { + dockerCLI := test.NewFakeCli(nil) + + b.ReportAllocs() + for b.Loop() { + AddCommands(&cobra.Command{Use: "docker"}, dockerCLI) + } +} diff --git a/cli/command/container/opts_test.go b/cli/command/container/opts_test.go index 0781e99687c0..5cacb06277ca 100644 --- a/cli/command/container/opts_test.go +++ b/cli/command/container/opts_test.go @@ -1157,3 +1157,58 @@ func TestConvertToStandardNotation(t *testing.T) { } } } + +// BenchmarkParseRun measures parsing the flags of a "docker run" invocation, +// which includes constructing the flag-set, and converting the options to +// the container-, host-, and networking-config. +func BenchmarkParseRun(b *testing.B) { + for _, tc := range []struct { + doc string + args []string + }{ + { + doc: "minimal", + args: []string{"ubuntu", "bash"}, + }, + { + doc: "many flags", + args: []string{ + "--hostname", "my-hostname", + "--user", "1000:1000", + "--workdir", "/some/workdir", + "--env", "FOO=bar", + "--env", "SOME_OTHER_VAR=some-other-value", + "--label", "com.example.label=some-value", + "--publish", "8080:80/tcp", + "--publish", "127.0.0.1:8443:443", + "--expose", "9000-9010", + "--volume", "/tmp/source:/mnt/source:ro", + "--mount", "type=volume,source=my-volume,target=/data,readonly", + "--tmpfs", "/run:size=1m", + "--network", "my-network", + "--dns", "1.1.1.1", + "--add-host", "example.com:127.0.0.1", + "--cap-add", "NET_ADMIN", + "--cap-drop", "MKNOD", + "--memory", "512m", + "--cpus", "1.5", + "--restart", "on-failure:5", + "--ulimit", "nofile=1024:2048", + "--health-cmd", "curl -f http://localhost/ || exit 1", + "--health-interval", "30s", + "--log-driver", "json-file", + "--log-opt", "max-size=10m", + "ubuntu", "bash", + }, + }, + } { + b.Run(tc.doc, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + if _, _, _, err := parseRun(tc.args); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/cli/command/formatter/container_test.go b/cli/command/formatter/container_test.go index 1c68aeaea059..c6610fd6ec58 100644 --- a/cli/command/formatter/container_test.go +++ b/cli/command/formatter/container_test.go @@ -953,3 +953,66 @@ func TestDisplayablePorts(t *testing.T) { assert.Check(t, is.Equal(port.expected, actual)) } } + +// genContainers generates a list of containers to be used as a realistic +// input for benchmarks; the containers have all fields set that are used +// by the various formats. +func genContainers(count int) []container.Summary { + created := time.Now().Add(-72 * time.Hour).Unix() + containers := make([]container.Summary, 0, count) + for i := range count { + containers = append(containers, container.Summary{ + ID: fmt.Sprintf("%064x", i), + Names: []string{fmt.Sprintf("/container_%d", i), fmt.Sprintf("/other_name_%d", i)}, + Image: "docker.io/library/ubuntu:24.04", + ImageID: fmt.Sprintf("sha256:%064x", i), + Command: `/bin/sh -c "while true; do echo hello world; sleep 1; done"`, + Created: created, + Ports: []container.PortSummary{ + {IP: netip.MustParseAddr("0.0.0.0"), PrivatePort: 80, PublicPort: uint16(30000 + i), Type: "tcp"}, + {IP: netip.MustParseAddr("::"), PrivatePort: 443, PublicPort: uint16(40000 + i), Type: "tcp"}, + {PrivatePort: 8080, Type: "tcp"}, + }, + SizeRw: 123456789, + SizeRootFs: 987654321, + Labels: map[string]string{ + "com.docker.compose.project": "some-project", + "com.docker.compose.service": fmt.Sprintf("service-%d", i), + "org.opencontainers.image.source": "https://github.com/docker/cli", + }, + State: "running", + Status: "Up 3 days (healthy)", + Mounts: []container.MountPoint{ + {Type: "volume", Name: fmt.Sprintf("volume-%d", i), Destination: "/data"}, + {Type: "bind", Source: "/tmp/source", Destination: "/mnt/source"}, + }, + }) + } + return containers +} + +func BenchmarkContainerWrite(b *testing.B) { + containers := genContainers(100) + for _, tc := range []struct { + doc string + format Format + }{ + {doc: "table", format: NewContainerFormat("table", false, false)}, + {doc: "table-with-size", format: NewContainerFormat("table", false, true)}, + {doc: "quiet", format: NewContainerFormat("table", true, false)}, + {doc: "raw", format: NewContainerFormat("raw", false, false)}, + {doc: "json", format: NewContainerFormat("json", false, false)}, + {doc: "custom", format: NewContainerFormat(`{{.ID}}: {{.Names}} {{.Ports}} {{.Labels}} {{.Mounts}}`, false, false)}, + } { + b.Run(tc.doc, func(b *testing.B) { + b.ReportAllocs() + out := bytes.NewBuffer(nil) + for b.Loop() { + out.Reset() + if err := ContainerWrite(Context{Format: tc.format, Output: out, Trunc: true}, containers); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/cli/command/formatter/image_test.go b/cli/command/formatter/image_test.go index c3906687c4fa..c212eeb3e74c 100644 --- a/cli/command/formatter/image_test.go +++ b/cli/command/formatter/image_test.go @@ -367,3 +367,60 @@ func TestImageContextWriteWithNoImage(t *testing.T) { }) } } + +// genImages generates a list of images to be used as a realistic input for +// benchmarks; the images have all fields set that are used by the various +// formats. +func genImages(count int) []image.Summary { + created := time.Now().AddDate(0, 0, -1).Unix() + images := make([]image.Summary, 0, count) + for i := range count { + images = append(images, image.Summary{ + ID: fmt.Sprintf("sha256:%064x", i), + Created: created, + RepoTags: []string{ + fmt.Sprintf("docker.io/library/image-%d:latest", i), + fmt.Sprintf("example.com/some/longer/name/image-%d:v1.2.3", i), + }, + RepoDigests: []string{ + fmt.Sprintf("docker.io/library/image-%d@sha256:%064x", i, i), + }, + Size: 123456789, + SharedSize: 12345678, + Containers: 3, + Labels: map[string]string{"org.opencontainers.image.source": "https://github.com/docker/cli"}, + }) + } + return images +} + +func BenchmarkImageWrite(b *testing.B) { + images := genImages(100) + for _, tc := range []struct { + doc string + format Format + digest bool + }{ + {doc: "table", format: NewImageFormat("table", false, false)}, + {doc: "table-with-digest", format: NewImageFormat("table", false, true), digest: true}, + {doc: "quiet", format: NewImageFormat("table", true, false)}, + {doc: "raw", format: NewImageFormat("raw", false, false)}, + {doc: "json", format: NewImageFormat("json", false, false)}, + {doc: "custom", format: NewImageFormat(`{{.Repository}}:{{.Tag}} {{.ID}} {{.Size}}`, false, false)}, + } { + b.Run(tc.doc, func(b *testing.B) { + b.ReportAllocs() + out := bytes.NewBuffer(nil) + for b.Loop() { + out.Reset() + ctx := ImageContext{ + Context: Context{Format: tc.format, Output: out, Trunc: true}, + Digest: tc.digest, + } + if err := ImageWrite(ctx, images); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/cli/config/config_test.go b/cli/config/config_test.go index 922641a726ad..9d266bb7c456 100644 --- a/cli/config/config_test.go +++ b/cli/config/config_test.go @@ -482,3 +482,39 @@ func TestSetDir(t *testing.T) { SetDir(expected) assert.Check(t, is.Equal(Dir(), expected)) } + +// benchConfig is a realistic configuration file, as loaded on every +// invocation of the CLI. +const benchConfig = `{ + "auths": { + "https://index.docker.io/v1/": {"auth": "am9lam9lOmhlbGxv", "email": "user@example.com"}, + "registry.example.com": {"auth": "am9lam9lOmhlbGxv"}, + "registry.example.com:5000": {"auth": "am9lam9lOmhlbGxv"}, + "some-other-registry.example.com": {"auth": "am9lam9lOmhlbGxv", "identitytoken": "super-secret-token"} + }, + "credsStore": "desktop", + "credHelpers": { + "registry.example.com": "secretservice", + "other.example.com": "pass" + }, + "psFormat": "table {{.ID}}\\t{{.Image}}\\t{{.Command}}\\t{{.Status}}", + "imagesFormat": "table {{.Repository}}\\t{{.Tag}}\\t{{.ID}}\\t{{.Size}}", + "detachKeys": "ctrl-e,e", + "currentContext": "default", + "plugins": { + "buildx": {"defaultBuilder": "default"}, + "compose": {"someOption": "someValue"} + }, + "aliases": {"builder": "buildx"}, + "features": {"containerd-snapshotter": "true"}, + "experimental": "enabled" +}` + +func BenchmarkLoadFromReader(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + if _, err := LoadFromReader(strings.NewReader(benchConfig)); err != nil { + b.Fatal(err) + } + } +} diff --git a/opts/mount_test.go b/opts/mount_test.go index 2014c98d55c4..545caa1a98a9 100644 --- a/opts/mount_test.go +++ b/opts/mount_test.go @@ -586,3 +586,25 @@ func TestMountOptSetBindRecursive(t *testing.T) { }, m.Value())) }) } + +func BenchmarkMountOptSet(b *testing.B) { + for _, tc := range []struct { + doc string + value string + }{ + {doc: "volume", value: "type=volume,source=my-volume,target=/data"}, + {doc: "volume-with-opts", value: "type=volume,source=my-volume,target=/data,readonly,volume-nocopy,volume-driver=local,volume-label=foo=bar,volume-opt=type=nfs,volume-opt=device=:/some/path"}, + {doc: "bind", value: "type=bind,source=/home/path,target=/target,readonly,bind-propagation=rprivate"}, + {doc: "tmpfs", value: "type=tmpfs,target=/target,tmpfs-size=1m,tmpfs-mode=0700"}, + } { + b.Run(tc.doc, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + var m MountOpt + if err := m.Set(tc.value); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/templates/templates_test.go b/templates/templates_test.go index ed1ee5b95d13..b69d1ad3cb67 100644 --- a/templates/templates_test.go +++ b/templates/templates_test.go @@ -229,3 +229,44 @@ func TestJoinElements(t *testing.T) { }) } } + +func BenchmarkParse(b *testing.B) { + for _, tc := range []struct { + doc string + format string + }{ + {doc: "simple", format: `{{.ID}}`}, + {doc: "json", format: `{{json .}}`}, + {doc: "table", format: "{{.ID}}\t{{.Image}}\t{{.Command}}\t{{.RunningFor}}\t{{.Status}}\t{{.Ports}}\t{{.Names}}"}, + {doc: "functions", format: `{{pad (upper (truncate .Name 12)) 1 1}}{{join (split .Names ",") " "}}`}, + } { + b.Run(tc.doc, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + if _, err := Parse(tc.format); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func BenchmarkExecute(b *testing.B) { + tmpl, err := Parse(`{{pad (upper (truncate .Name 12)) 1 1}}{{json .Labels}}{{join (split .Names ",") " "}}`) + assert.NilError(b, err) + + data := map[string]any{ + "Name": "some-container-name", + "Names": "first,second,third", + "Labels": map[string]string{"label1": "value1", "label2": "value2"}, + } + + b.ReportAllocs() + var buf bytes.Buffer + for b.Loop() { + buf.Reset() + if err := tmpl.Execute(&buf, data); err != nil { + b.Fatal(err) + } + } +}