Add loopback pprof listener

README.md

@@ -123,6 +123,7 @@ Bazel's two keyspaces map onto what the index already stores. A CAS blob is addr
 
 Bazel traffic shows up in `plaid-cache status` and `plaid-cache stats` beside the toolchain's, since both go through the same tiers.
 
+
 #### Choosing a protocol
 
 The two protocols reach the same store, so the choice is about what a client can say over each.
@@ -207,6 +208,30 @@ Bazel treats the resulting dangling reference as a failed download rather than a
 
 If you set a lifecycle rule on the bucket, expiring action records earlier than output bodies keeps the dangling reference on the harmless side: an action record with no body is a miss, where a body with no action record is merely unreferenced.
 
+### Runtime profiling
+
+Go runtime profiles are available only from a separate, opt-in loopback
+listener:
+
+```sh
+plaid-cache serve -pprof-addr 127.0.0.1:6060
+```
+
+The listener serves the standard `/debug/pprof/` endpoints. For example:
+
+```sh
+go tool pprof http://127.0.0.1:6060/debug/pprof/heap
+go tool pprof 'http://127.0.0.1:6060/debug/pprof/profile?seconds=30'
+curl http://127.0.0.1:6060/debug/pprof/goroutine
+curl http://127.0.0.1:6060/debug/pprof/mutex
+curl http://127.0.0.1:6060/debug/pprof/block
+curl -o trace.out 'http://127.0.0.1:6060/debug/pprof/trace?seconds=5'
+```
+
+`PLAID_GOCACHE_PPROF_ADDR` is the equivalent environment or configuration-file
+setting. It defaults to empty, accepts only loopback IP addresses, has no
+authentication, and never adds profiling routes to the Bazel HTTP listener.
+
 ### Monitoring a shared daemon
 
 `plaid-cache status` reads the local daemon over a unix socket, which is the right answer for a cache on the machine you are sitting at and no answer at all for one serving a room full of builders. `-bazel-monitoring` adds two read-only routes to the Bazel HTTP address for that case:
@@ -539,6 +564,7 @@ be a surprising amount of reach for this one to have.
 | `PLAID_GOCACHE_COMPACT_AFTER` | Pruned entries that must accumulate before the index is compacted. Deletes in an LSM are writes, so pruning grows the index until a compaction reclaims it. | `1000` |
 | `PLAID_GOCACHE_BAZEL_ADDR` | Address for the Bazel HTTP remote cache, e.g. `localhost:9095`. Empty serves it not at all. | empty |
 | `PLAID_GOCACHE_BAZEL_GRPC_ADDR` | Address for the Bazel gRPC remote cache, e.g. `localhost:9096`. Empty serves it not at all. | empty |
+| `PLAID_GOCACHE_PPROF_ADDR` | Loopback address for the separate Go runtime-profiling listener, e.g. `127.0.0.1:6060`. Empty serves it not at all. | empty |
 | `PLAID_GOCACHE_BAZEL_MONITORING` | `1` also serves `/status` and `/metrics` on the Bazel HTTP address, for `plaid-cache status -from` and for a Prometheus scrape. Off by default: they describe the host rather than the cache's contents. | unset |
 | `PLAID_GOCACHE_DISABLE_BAZEL_VERIFY` | `1` stops both Bazel listeners from checking that an uploaded CAS body hashes to the digest naming it, and lets a gRPC client name a digest function this cache cannot compute. For clients whose digest function is not SHA-256. | unset |
 | `PLAID_GOCACHE_DISABLE_EVICTION` | `1` disables eviction entirely. | unset |

cmd/plaid-cache/commands.go

@@ -103,7 +103,7 @@ func openStores(ctx context.Context, cfg *config.Config) (*stores, error) {
 // runServe runs the daemon in the foreground.
 func (a *app) runServe(ctx context.Context) int {
 	var limits limitFlags
-	var bazelAddr, bazelGRPCAddr string
+	var bazelAddr, bazelGRPCAddr, pprofAddr string
 	var bazelMonitoring bool
 	register := func(fs *flag.FlagSet) {
 		limits.register(fs)
@@ -113,6 +113,8 @@ func (a *app) runServe(ctx context.Context) int {
 			"also serve Bazel's gRPC remote-cache protocol on this address, e.g. localhost:9096 (default: PLAID_GOCACHE_BAZEL_GRPC_ADDR)")
 		fs.BoolVar(&bazelMonitoring, "bazel-monitoring", false,
 			"also serve "+bazel.StatusPath+" and "+bazel.MetricsPath+" on the Bazel HTTP address (default: PLAID_GOCACHE_BAZEL_MONITORING)")
+		fs.StringVar(&pprofAddr, "pprof-addr", "",
+			"serve Go runtime profiles on this address, e.g. 127.0.0.1:6060 (default: PLAID_GOCACHE_PPROF_ADDR)")
 	}
 	if _, err := a.parseFlags("serve", register, a.args[1:]); err != nil {
 		a.errf("plaid-cache: %v\n", err)
@@ -132,6 +134,9 @@ func (a *app) runServe(ctx context.Context) int {
 	if bazelGRPCAddr != "" {
 		cfg.BazelGRPCAddr = bazelGRPCAddr
 	}
+	if pprofAddr != "" {
+		cfg.PprofAddr = pprofAddr
+	}
 	// The flag can turn monitoring on and not off, matching the addresses above:
 	// a flag given is a decision, a flag omitted is silence, and silence must
 	// leave the environment's answer standing.
@@ -170,12 +175,12 @@ func (a *app) runServe(ctx context.Context) int {
 	})
 	logf("serving on %s (pid %d)", cfg.SocketPath(), os.Getpid())
 
-	// The Bazel listener runs beside the socket rather than instead of it, and
-	// it holds the index the deferred closes above release. Stopping the daemon
-	// and waiting for it therefore has to happen before those run, which is
-	// what the ordering of these two defers buys.
-	var bazelWG sync.WaitGroup
-	defer bazelWG.Wait()
+	// Optional TCP listeners run beside the socket rather than instead of it,
+	// and they hold the index the deferred closes above release. Stopping the
+	// daemon and waiting for them therefore has to happen before those run,
+	// which is what the ordering of these two defers buys.
+	var listenerWG sync.WaitGroup
+	defer listenerWG.Wait()
 	defer srv.Stop()
 
 	if cfg.BazelAddr != "" {
@@ -196,9 +201,9 @@ func (a *app) runServe(ctx context.Context) int {
 			logf("serving monitoring on http://%s%s and http://%s%s",
 				bazelLn.Addr(), bazel.StatusPath, bazelLn.Addr(), bazel.MetricsPath)
 		}
-		bazelWG.Add(1)
+		listenerWG.Add(1)
 		go func() {
-			defer bazelWG.Done()
+			defer listenerWG.Done()
 			if err := srv.ServeBazel(ctx, bazelLn); err != nil && !errors.Is(err, context.Canceled) {
 				logf("bazel listener: %v", err)
 			}
@@ -212,15 +217,31 @@ func (a *app) runServe(ctx context.Context) int {
 			return exitError
 		}
 		logf("serving the Bazel gRPC cache on grpc://%s", grpcLn.Addr())
-		bazelWG.Add(1)
+		listenerWG.Add(1)
 		go func() {
-			defer bazelWG.Done()
+			defer listenerWG.Done()
 			if err := srv.ServeBazelGRPC(ctx, grpcLn); err != nil && !errors.Is(err, context.Canceled) {
 				logf("bazel grpc listener: %v", err)
 			}
 		}()
 	}
 
+	if cfg.PprofAddr != "" {
+		pprofLn, lerr := daemon.ListenPprof(cfg.PprofAddr)
+		if lerr != nil {
+			a.errf("plaid-cache: pprof listener: %v\n", lerr)
+			return exitError
+		}
+		logf("serving Go runtime profiles on http://%s/debug/pprof/", pprofLn.Addr())
+		listenerWG.Add(1)
+		go func() {
+			defer listenerWG.Done()
+			if err := srv.ServePprof(ctx, pprofLn); err != nil && !errors.Is(err, context.Canceled) {
+				logf("pprof listener: %v", err)
+			}
+		}()
+	}
+
 	if err := srv.Serve(ctx, ln); err != nil && !errors.Is(err, context.Canceled) {
 		a.errf("plaid-cache: %v\n", err)
 		return exitError

cmd/plaid-cache/flags_test.go

@@ -5,6 +5,7 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"strings"
 	"testing"
 	"time"
@@ -143,6 +144,35 @@ func TestGCFlagsAreUsageErrorsNotFailures(t *testing.T) {
 	}
 }
 
+// TestServePprofAddressFlagOverridesEnvironment pins that an explicit profile
+// address wins over the environment, matching the other optional listeners.
+func TestServePprofAddressFlagOverridesEnvironment(t *testing.T) {
+	a, _, errb := newApp(t, "serve", "-pprof-addr", "not-an-address")
+	t.Setenv("PLAID_GOCACHE_PPROF_ADDR", "127.0.0.1:0")
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	done := make(chan int, 1)
+	go func() { done <- a.runServe(ctx) }()
+
+	select {
+	case code := <-done:
+		if code != exitError {
+			t.Fatalf("runServe(-pprof-addr) = %d, want %d (stderr: %s)", code, exitError, errb)
+		}
+		if !strings.Contains(errb.String(), "pprof listener") {
+			t.Fatalf("stderr = %q, want the pprof listener error", errb)
+		}
+	case <-time.After(5 * time.Second):
+		cancel()
+		select {
+		case <-done:
+		case <-time.After(5 * time.Second):
+			t.Fatal("runServe did not return after cancelling its context")
+		}
+		t.Fatal("runServe did not use the explicit pprof address")
+	}
+}
+
 // TestGCAppliesFlagsWithoutADaemon pins that the flags work on the standalone
 // path too, where this process owns the index.
 func TestGCAppliesFlagsWithoutADaemon(t *testing.T) {

cmd/plaid-cache/main.go

@@ -162,6 +162,12 @@ cache already holds, so an action that re-runs stops re-uploading outputs the
 cache has. Both may be served at once, and a build that uses either reads what
 the other stored.
 
+-pprof-addr serves Go runtime profiles on its own loopback address rather than
+adding them to the Bazel listener. It is off unless asked for and has no
+authentication:
+  plaid-cache serve -pprof-addr 127.0.0.1:6060
+  go tool pprof http://127.0.0.1:6060/debug/pprof/heap
+
 -bazel-monitoring adds two routes to the HTTP address — /status and /metrics —
 so a daemon serving a room full of builders can be read without a shell on its
 host. They are off unless asked for, because they describe the host rather than

cmd/plaid-cache/main_test.go

@@ -42,7 +42,8 @@ func clearCacheEnv(t *testing.T) {
 		"PLAID_GOCACHE_S3_ENDPOINT_URL", "PLAID_GOCACHE_MIN_UPLOAD_SIZE",
 		"PLAID_GOCACHE_UPLOAD_CONCURRENCY", "PLAID_GOCACHE_IDLE_TIMEOUT",
 		"PLAID_GOCACHE_EVICT_INTERVAL", "PLAID_GOCACHE_DISABLE_EVICTION",
-		"PLAID_GOCACHE_DISABLE_DAEMON", "PLAID_GOCACHE_LOG", "PLAID_GOCACHE_COMPACT_AFTER", "XDG_CACHE_HOME",
+		"PLAID_GOCACHE_DISABLE_DAEMON", "PLAID_GOCACHE_LOG", "PLAID_GOCACHE_COMPACT_AFTER",
+		"PLAID_GOCACHE_PPROF_ADDR", "XDG_CACHE_HOME",
 	} {
 		t.Setenv(n, "")
 	}

internal/config/config.go

@@ -171,6 +171,14 @@ type Config struct {
 	// bind applies, and for the same reason.
 	BazelGRPCAddr string
 
+	// PprofAddr is the loopback TCP address that serves Go runtime profiles,
+	// e.g. "127.0.0.1:6060". Empty, the default, serves no profiles.
+	//
+	// Profiles expose process runtime data and have no authentication, so the
+	// listener accepts only loopback IP addresses. It is separate from
+	// BazelAddr so enabling diagnostics cannot add routes to the cache listener.
+	PprofAddr string
+
 	// BazelMonitoring serves the two monitoring routes — /status and /metrics —
 	// on the Bazel HTTP listener, so that `plaid-cache status -from` and a
 	// Prometheus scrape can read a daemon an operator has no shell on. False,
@@ -252,6 +260,7 @@ func Load() (*Config, error) {
 		S3EndpointURL:     src("PLAID_GOCACHE_S3_ENDPOINT_URL"),
 		BazelAddr:         src("PLAID_GOCACHE_BAZEL_ADDR"),
 		BazelGRPCAddr:     src("PLAID_GOCACHE_BAZEL_GRPC_ADDR"),
+		PprofAddr:         src("PLAID_GOCACHE_PPROF_ADDR"),
 		UploadConcurrency: runtime.NumCPU(),
 	}
 
@@ -346,6 +355,7 @@ var settingNames = map[string]bool{
 	"PLAID_GOCACHE_BAZEL_ADDR":           true,
 	"PLAID_GOCACHE_BAZEL_GRPC_ADDR":      true,
 	"PLAID_GOCACHE_BAZEL_MONITORING":     true,
+	"PLAID_GOCACHE_PPROF_ADDR":           true,
 	"PLAID_GOCACHE_DISABLE_BAZEL_VERIFY": true,
 	"PLAID_GOCACHE_DISABLE_EVICTION":     true,
 	"PLAID_GOCACHE_DISABLE_DAEMON":       true,

internal/config/config_test.go

@@ -107,6 +107,9 @@ func TestLoadDefaults(t *testing.T) {
 	if c.BazelAddr != "" {
 		t.Fatalf("BazelAddr = %q by default, want it empty", c.BazelAddr)
 	}
+	if c.PprofAddr != "" {
+		t.Fatalf("PprofAddr = %q by default, want it empty", c.PprofAddr)
+	}
 	if c.DisableBazelVerify {
 		t.Fatalf("DisableBazelVerify = true by default, want uploads verified")
 	}
@@ -290,6 +293,8 @@ func clearEnv(t *testing.T) {
 		"PLAID_GOCACHE_COMPACT_AFTER",
 		"PLAID_GOCACHE_BAZEL_ADDR",
 		"PLAID_GOCACHE_BAZEL_GRPC_ADDR",
+		"PLAID_GOCACHE_BAZEL_MONITORING",
+		"PLAID_GOCACHE_PPROF_ADDR",
 		"PLAID_GOCACHE_DISABLE_BAZEL_VERIFY",
 		"XDG_CACHE_HOME",
 	} {
@@ -306,16 +311,16 @@ func clearEnv(t *testing.T) {
 	t.Setenv("XDG_CONFIG_HOME", t.TempDir())
 }
 
-// TestLoadBazelSettings pins that the Bazel listener is configurable from the
-// environment as well as from the serve flag, since a supervised daemon is
-// started from a unit file rather than by hand.
-func TestLoadBazelSettings(t *testing.T) {
+// TestLoadListenerSettings pins that optional TCP listeners are configurable
+// from the environment as well as from the serve flag.
+func TestLoadListenerSettings(t *testing.T) {
 	clearEnv(t)
 	t.Setenv("PLAID_GOCACHE_DIR", t.TempDir())
 	t.Setenv("PLAID_GOCACHE_BAZEL_ADDR", "localhost:9095")
 	t.Setenv("PLAID_GOCACHE_BAZEL_GRPC_ADDR", "localhost:9096")
 	t.Setenv("PLAID_GOCACHE_BAZEL_MONITORING", "1")
 	t.Setenv("PLAID_GOCACHE_DISABLE_BAZEL_VERIFY", "1")
+	t.Setenv("PLAID_GOCACHE_PPROF_ADDR", "127.0.0.1:6060")
 
 	c, err := Load()
 	if err != nil {
@@ -327,6 +332,9 @@ func TestLoadBazelSettings(t *testing.T) {
 	if c.BazelGRPCAddr != "localhost:9096" {
 		t.Fatalf("BazelGRPCAddr = %q, want %q", c.BazelGRPCAddr, "localhost:9096")
 	}
+	if c.PprofAddr != "127.0.0.1:6060" {
+		t.Fatalf("PprofAddr = %q, want %q", c.PprofAddr, "127.0.0.1:6060")
+	}
 	if !c.BazelMonitoring {
 		t.Fatalf("BazelMonitoring = false, want true")
 	}
@@ -352,15 +360,16 @@ func TestMonitoringIsOffUnlessAsked(t *testing.T) {
 	}
 }
 
-// TestBazelSettingsAreValidFileKeys pins that every Bazel setting can be written to
-// the configuration file. An unknown key there is a hard error, so a setting
-// missing from the accepted set is one a user cannot persist.
-func TestBazelSettingsAreValidFileKeys(t *testing.T) {
+// TestListenerSettingsAreValidFileKeys pins that every optional TCP listener
+// setting can be written to the configuration file. An unknown key there is a
+// hard error, so a setting missing from the accepted set is one a user cannot
+// persist.
+func TestListenerSettingsAreValidFileKeys(t *testing.T) {
 	clearEnv(t)
 	dir := t.TempDir()
 	t.Setenv("PLAID_GOCACHE_DIR", dir)
 	path := filepath.Join(dir, "config")
-	if err := os.WriteFile(path, []byte("bazel-addr = localhost:9096\nbazel-grpc-addr = localhost:9097\nbazel-monitoring = 1\ndisable-bazel-verify = 1\n"), 0o600); err != nil {
+	if err := os.WriteFile(path, []byte("bazel-addr = localhost:9096\nbazel-grpc-addr = localhost:9097\nbazel-monitoring = 1\npprof-addr = 127.0.0.1:6060\ndisable-bazel-verify = 1\n"), 0o600); err != nil {
 		t.Fatalf("WriteFile: %v", err)
 	}
 	t.Setenv(configFileEnvVar, path)
@@ -375,6 +384,9 @@ func TestBazelSettingsAreValidFileKeys(t *testing.T) {
 	if c.BazelGRPCAddr != "localhost:9097" {
 		t.Fatalf("BazelGRPCAddr = %q, want %q", c.BazelGRPCAddr, "localhost:9097")
 	}
+	if c.PprofAddr != "127.0.0.1:6060" {
+		t.Fatalf("PprofAddr = %q, want %q", c.PprofAddr, "127.0.0.1:6060")
+	}
 	if !c.BazelMonitoring {
 		t.Fatalf("BazelMonitoring = false, want true")
 	}