diff --git a/README.md b/README.md index cb8bbf6..8adf561 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,7 @@ Bazel's two keyspaces map onto what the index already stores. A CAS blob is addr Bazel traffic shows up in `plaid-cache status` and `plaid-cache stats` beside the toolchain's, since both go through the same tiers. + #### Choosing a protocol The two protocols reach the same store, so the choice is about what a client can say over each. @@ -207,6 +208,30 @@ Bazel treats the resulting dangling reference as a failed download rather than a If you set a lifecycle rule on the bucket, expiring action records earlier than output bodies keeps the dangling reference on the harmless side: an action record with no body is a miss, where a body with no action record is merely unreferenced. +### Runtime profiling + +Go runtime profiles are available only from a separate, opt-in loopback +listener: + +```sh +plaid-cache serve -pprof-addr 127.0.0.1:6060 +``` + +The listener serves the standard `/debug/pprof/` endpoints. For example: + +```sh +go tool pprof http://127.0.0.1:6060/debug/pprof/heap +go tool pprof 'http://127.0.0.1:6060/debug/pprof/profile?seconds=30' +curl http://127.0.0.1:6060/debug/pprof/goroutine +curl http://127.0.0.1:6060/debug/pprof/mutex +curl http://127.0.0.1:6060/debug/pprof/block +curl -o trace.out 'http://127.0.0.1:6060/debug/pprof/trace?seconds=5' +``` + +`PLAID_GOCACHE_PPROF_ADDR` is the equivalent environment or configuration-file +setting. It defaults to empty, accepts only loopback IP addresses, has no +authentication, and never adds profiling routes to the Bazel HTTP listener. + ### Monitoring a shared daemon `plaid-cache status` reads the local daemon over a unix socket, which is the right answer for a cache on the machine you are sitting at and no answer at all for one serving a room full of builders. `-bazel-monitoring` adds two read-only routes to the Bazel HTTP address for that case: @@ -539,6 +564,7 @@ be a surprising amount of reach for this one to have. | `PLAID_GOCACHE_COMPACT_AFTER` | Pruned entries that must accumulate before the index is compacted. Deletes in an LSM are writes, so pruning grows the index until a compaction reclaims it. | `1000` | | `PLAID_GOCACHE_BAZEL_ADDR` | Address for the Bazel HTTP remote cache, e.g. `localhost:9095`. Empty serves it not at all. | empty | | `PLAID_GOCACHE_BAZEL_GRPC_ADDR` | Address for the Bazel gRPC remote cache, e.g. `localhost:9096`. Empty serves it not at all. | empty | +| `PLAID_GOCACHE_PPROF_ADDR` | Loopback address for the separate Go runtime-profiling listener, e.g. `127.0.0.1:6060`. Empty serves it not at all. | empty | | `PLAID_GOCACHE_BAZEL_MONITORING` | `1` also serves `/status` and `/metrics` on the Bazel HTTP address, for `plaid-cache status -from` and for a Prometheus scrape. Off by default: they describe the host rather than the cache's contents. | unset | | `PLAID_GOCACHE_DISABLE_BAZEL_VERIFY` | `1` stops both Bazel listeners from checking that an uploaded CAS body hashes to the digest naming it, and lets a gRPC client name a digest function this cache cannot compute. For clients whose digest function is not SHA-256. | unset | | `PLAID_GOCACHE_DISABLE_EVICTION` | `1` disables eviction entirely. | unset | diff --git a/cmd/plaid-cache/commands.go b/cmd/plaid-cache/commands.go index 3663391..87fde62 100644 --- a/cmd/plaid-cache/commands.go +++ b/cmd/plaid-cache/commands.go @@ -103,7 +103,7 @@ func openStores(ctx context.Context, cfg *config.Config) (*stores, error) { // runServe runs the daemon in the foreground. func (a *app) runServe(ctx context.Context) int { var limits limitFlags - var bazelAddr, bazelGRPCAddr string + var bazelAddr, bazelGRPCAddr, pprofAddr string var bazelMonitoring bool register := func(fs *flag.FlagSet) { limits.register(fs) @@ -113,6 +113,8 @@ func (a *app) runServe(ctx context.Context) int { "also serve Bazel's gRPC remote-cache protocol on this address, e.g. localhost:9096 (default: PLAID_GOCACHE_BAZEL_GRPC_ADDR)") fs.BoolVar(&bazelMonitoring, "bazel-monitoring", false, "also serve "+bazel.StatusPath+" and "+bazel.MetricsPath+" on the Bazel HTTP address (default: PLAID_GOCACHE_BAZEL_MONITORING)") + fs.StringVar(&pprofAddr, "pprof-addr", "", + "serve Go runtime profiles on this address, e.g. 127.0.0.1:6060 (default: PLAID_GOCACHE_PPROF_ADDR)") } if _, err := a.parseFlags("serve", register, a.args[1:]); err != nil { a.errf("plaid-cache: %v\n", err) @@ -132,6 +134,9 @@ func (a *app) runServe(ctx context.Context) int { if bazelGRPCAddr != "" { cfg.BazelGRPCAddr = bazelGRPCAddr } + if pprofAddr != "" { + cfg.PprofAddr = pprofAddr + } // The flag can turn monitoring on and not off, matching the addresses above: // a flag given is a decision, a flag omitted is silence, and silence must // leave the environment's answer standing. @@ -170,12 +175,12 @@ func (a *app) runServe(ctx context.Context) int { }) logf("serving on %s (pid %d)", cfg.SocketPath(), os.Getpid()) - // The Bazel listener runs beside the socket rather than instead of it, and - // it holds the index the deferred closes above release. Stopping the daemon - // and waiting for it therefore has to happen before those run, which is - // what the ordering of these two defers buys. - var bazelWG sync.WaitGroup - defer bazelWG.Wait() + // Optional TCP listeners run beside the socket rather than instead of it, + // and they hold the index the deferred closes above release. Stopping the + // daemon and waiting for them therefore has to happen before those run, + // which is what the ordering of these two defers buys. + var listenerWG sync.WaitGroup + defer listenerWG.Wait() defer srv.Stop() if cfg.BazelAddr != "" { @@ -196,9 +201,9 @@ func (a *app) runServe(ctx context.Context) int { logf("serving monitoring on http://%s%s and http://%s%s", bazelLn.Addr(), bazel.StatusPath, bazelLn.Addr(), bazel.MetricsPath) } - bazelWG.Add(1) + listenerWG.Add(1) go func() { - defer bazelWG.Done() + defer listenerWG.Done() if err := srv.ServeBazel(ctx, bazelLn); err != nil && !errors.Is(err, context.Canceled) { logf("bazel listener: %v", err) } @@ -212,15 +217,31 @@ func (a *app) runServe(ctx context.Context) int { return exitError } logf("serving the Bazel gRPC cache on grpc://%s", grpcLn.Addr()) - bazelWG.Add(1) + listenerWG.Add(1) go func() { - defer bazelWG.Done() + defer listenerWG.Done() if err := srv.ServeBazelGRPC(ctx, grpcLn); err != nil && !errors.Is(err, context.Canceled) { logf("bazel grpc listener: %v", err) } }() } + if cfg.PprofAddr != "" { + pprofLn, lerr := daemon.ListenPprof(cfg.PprofAddr) + if lerr != nil { + a.errf("plaid-cache: pprof listener: %v\n", lerr) + return exitError + } + logf("serving Go runtime profiles on http://%s/debug/pprof/", pprofLn.Addr()) + listenerWG.Add(1) + go func() { + defer listenerWG.Done() + if err := srv.ServePprof(ctx, pprofLn); err != nil && !errors.Is(err, context.Canceled) { + logf("pprof listener: %v", err) + } + }() + } + if err := srv.Serve(ctx, ln); err != nil && !errors.Is(err, context.Canceled) { a.errf("plaid-cache: %v\n", err) return exitError diff --git a/cmd/plaid-cache/flags_test.go b/cmd/plaid-cache/flags_test.go index 55cf86e..81cb0c8 100644 --- a/cmd/plaid-cache/flags_test.go +++ b/cmd/plaid-cache/flags_test.go @@ -5,6 +5,7 @@ package main import ( "bytes" + "context" "strings" "testing" "time" @@ -143,6 +144,35 @@ func TestGCFlagsAreUsageErrorsNotFailures(t *testing.T) { } } +// TestServePprofAddressFlagOverridesEnvironment pins that an explicit profile +// address wins over the environment, matching the other optional listeners. +func TestServePprofAddressFlagOverridesEnvironment(t *testing.T) { + a, _, errb := newApp(t, "serve", "-pprof-addr", "not-an-address") + t.Setenv("PLAID_GOCACHE_PPROF_ADDR", "127.0.0.1:0") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan int, 1) + go func() { done <- a.runServe(ctx) }() + + select { + case code := <-done: + if code != exitError { + t.Fatalf("runServe(-pprof-addr) = %d, want %d (stderr: %s)", code, exitError, errb) + } + if !strings.Contains(errb.String(), "pprof listener") { + t.Fatalf("stderr = %q, want the pprof listener error", errb) + } + case <-time.After(5 * time.Second): + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("runServe did not return after cancelling its context") + } + t.Fatal("runServe did not use the explicit pprof address") + } +} + // TestGCAppliesFlagsWithoutADaemon pins that the flags work on the standalone // path too, where this process owns the index. func TestGCAppliesFlagsWithoutADaemon(t *testing.T) { diff --git a/cmd/plaid-cache/main.go b/cmd/plaid-cache/main.go index d2c78d4..f518c52 100644 --- a/cmd/plaid-cache/main.go +++ b/cmd/plaid-cache/main.go @@ -162,6 +162,12 @@ cache already holds, so an action that re-runs stops re-uploading outputs the cache has. Both may be served at once, and a build that uses either reads what the other stored. +-pprof-addr serves Go runtime profiles on its own loopback address rather than +adding them to the Bazel listener. It is off unless asked for and has no +authentication: + plaid-cache serve -pprof-addr 127.0.0.1:6060 + go tool pprof http://127.0.0.1:6060/debug/pprof/heap + -bazel-monitoring adds two routes to the HTTP address — /status and /metrics — so a daemon serving a room full of builders can be read without a shell on its host. They are off unless asked for, because they describe the host rather than diff --git a/cmd/plaid-cache/main_test.go b/cmd/plaid-cache/main_test.go index bc6ba23..0a547e2 100644 --- a/cmd/plaid-cache/main_test.go +++ b/cmd/plaid-cache/main_test.go @@ -42,7 +42,8 @@ func clearCacheEnv(t *testing.T) { "PLAID_GOCACHE_S3_ENDPOINT_URL", "PLAID_GOCACHE_MIN_UPLOAD_SIZE", "PLAID_GOCACHE_UPLOAD_CONCURRENCY", "PLAID_GOCACHE_IDLE_TIMEOUT", "PLAID_GOCACHE_EVICT_INTERVAL", "PLAID_GOCACHE_DISABLE_EVICTION", - "PLAID_GOCACHE_DISABLE_DAEMON", "PLAID_GOCACHE_LOG", "PLAID_GOCACHE_COMPACT_AFTER", "XDG_CACHE_HOME", + "PLAID_GOCACHE_DISABLE_DAEMON", "PLAID_GOCACHE_LOG", "PLAID_GOCACHE_COMPACT_AFTER", + "PLAID_GOCACHE_PPROF_ADDR", "XDG_CACHE_HOME", } { t.Setenv(n, "") } diff --git a/internal/config/config.go b/internal/config/config.go index 22d750a..1b52a83 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -171,6 +171,14 @@ type Config struct { // bind applies, and for the same reason. BazelGRPCAddr string + // PprofAddr is the loopback TCP address that serves Go runtime profiles, + // e.g. "127.0.0.1:6060". Empty, the default, serves no profiles. + // + // Profiles expose process runtime data and have no authentication, so the + // listener accepts only loopback IP addresses. It is separate from + // BazelAddr so enabling diagnostics cannot add routes to the cache listener. + PprofAddr string + // BazelMonitoring serves the two monitoring routes — /status and /metrics — // on the Bazel HTTP listener, so that `plaid-cache status -from` and a // Prometheus scrape can read a daemon an operator has no shell on. False, @@ -252,6 +260,7 @@ func Load() (*Config, error) { S3EndpointURL: src("PLAID_GOCACHE_S3_ENDPOINT_URL"), BazelAddr: src("PLAID_GOCACHE_BAZEL_ADDR"), BazelGRPCAddr: src("PLAID_GOCACHE_BAZEL_GRPC_ADDR"), + PprofAddr: src("PLAID_GOCACHE_PPROF_ADDR"), UploadConcurrency: runtime.NumCPU(), } @@ -346,6 +355,7 @@ var settingNames = map[string]bool{ "PLAID_GOCACHE_BAZEL_ADDR": true, "PLAID_GOCACHE_BAZEL_GRPC_ADDR": true, "PLAID_GOCACHE_BAZEL_MONITORING": true, + "PLAID_GOCACHE_PPROF_ADDR": true, "PLAID_GOCACHE_DISABLE_BAZEL_VERIFY": true, "PLAID_GOCACHE_DISABLE_EVICTION": true, "PLAID_GOCACHE_DISABLE_DAEMON": true, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6fc4d18..713d52e 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -107,6 +107,9 @@ func TestLoadDefaults(t *testing.T) { if c.BazelAddr != "" { t.Fatalf("BazelAddr = %q by default, want it empty", c.BazelAddr) } + if c.PprofAddr != "" { + t.Fatalf("PprofAddr = %q by default, want it empty", c.PprofAddr) + } if c.DisableBazelVerify { t.Fatalf("DisableBazelVerify = true by default, want uploads verified") } @@ -290,6 +293,8 @@ func clearEnv(t *testing.T) { "PLAID_GOCACHE_COMPACT_AFTER", "PLAID_GOCACHE_BAZEL_ADDR", "PLAID_GOCACHE_BAZEL_GRPC_ADDR", + "PLAID_GOCACHE_BAZEL_MONITORING", + "PLAID_GOCACHE_PPROF_ADDR", "PLAID_GOCACHE_DISABLE_BAZEL_VERIFY", "XDG_CACHE_HOME", } { @@ -306,16 +311,16 @@ func clearEnv(t *testing.T) { t.Setenv("XDG_CONFIG_HOME", t.TempDir()) } -// TestLoadBazelSettings pins that the Bazel listener is configurable from the -// environment as well as from the serve flag, since a supervised daemon is -// started from a unit file rather than by hand. -func TestLoadBazelSettings(t *testing.T) { +// TestLoadListenerSettings pins that optional TCP listeners are configurable +// from the environment as well as from the serve flag. +func TestLoadListenerSettings(t *testing.T) { clearEnv(t) t.Setenv("PLAID_GOCACHE_DIR", t.TempDir()) t.Setenv("PLAID_GOCACHE_BAZEL_ADDR", "localhost:9095") t.Setenv("PLAID_GOCACHE_BAZEL_GRPC_ADDR", "localhost:9096") t.Setenv("PLAID_GOCACHE_BAZEL_MONITORING", "1") t.Setenv("PLAID_GOCACHE_DISABLE_BAZEL_VERIFY", "1") + t.Setenv("PLAID_GOCACHE_PPROF_ADDR", "127.0.0.1:6060") c, err := Load() if err != nil { @@ -327,6 +332,9 @@ func TestLoadBazelSettings(t *testing.T) { if c.BazelGRPCAddr != "localhost:9096" { t.Fatalf("BazelGRPCAddr = %q, want %q", c.BazelGRPCAddr, "localhost:9096") } + if c.PprofAddr != "127.0.0.1:6060" { + t.Fatalf("PprofAddr = %q, want %q", c.PprofAddr, "127.0.0.1:6060") + } if !c.BazelMonitoring { t.Fatalf("BazelMonitoring = false, want true") } @@ -352,15 +360,16 @@ func TestMonitoringIsOffUnlessAsked(t *testing.T) { } } -// TestBazelSettingsAreValidFileKeys pins that every Bazel setting can be written to -// the configuration file. An unknown key there is a hard error, so a setting -// missing from the accepted set is one a user cannot persist. -func TestBazelSettingsAreValidFileKeys(t *testing.T) { +// TestListenerSettingsAreValidFileKeys pins that every optional TCP listener +// setting can be written to the configuration file. An unknown key there is a +// hard error, so a setting missing from the accepted set is one a user cannot +// persist. +func TestListenerSettingsAreValidFileKeys(t *testing.T) { clearEnv(t) dir := t.TempDir() t.Setenv("PLAID_GOCACHE_DIR", dir) path := filepath.Join(dir, "config") - if err := os.WriteFile(path, []byte("bazel-addr = localhost:9096\nbazel-grpc-addr = localhost:9097\nbazel-monitoring = 1\ndisable-bazel-verify = 1\n"), 0o600); err != nil { + if err := os.WriteFile(path, []byte("bazel-addr = localhost:9096\nbazel-grpc-addr = localhost:9097\nbazel-monitoring = 1\npprof-addr = 127.0.0.1:6060\ndisable-bazel-verify = 1\n"), 0o600); err != nil { t.Fatalf("WriteFile: %v", err) } t.Setenv(configFileEnvVar, path) @@ -375,6 +384,9 @@ func TestBazelSettingsAreValidFileKeys(t *testing.T) { if c.BazelGRPCAddr != "localhost:9097" { t.Fatalf("BazelGRPCAddr = %q, want %q", c.BazelGRPCAddr, "localhost:9097") } + if c.PprofAddr != "127.0.0.1:6060" { + t.Fatalf("PprofAddr = %q, want %q", c.PprofAddr, "127.0.0.1:6060") + } if !c.BazelMonitoring { t.Fatalf("BazelMonitoring = false, want true") } diff --git a/internal/daemon/pprof.go b/internal/daemon/pprof.go new file mode 100644 index 0000000..9de12a5 --- /dev/null +++ b/internal/daemon/pprof.go @@ -0,0 +1,98 @@ +// Copyright 2026 The plaid-cache authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package daemon + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/http/pprof" + "time" +) + +// Pprof listener timeouts bound setup and shutdown without limiting a profile +// capture whose duration is selected by the caller. +const ( + pprofReadHeaderTimeout = 30 * time.Second + pprofIdleTimeout = 10 * time.Minute + pprofShutdownGrace = 30 * time.Second +) + +// ListenPprof binds the loopback address used for Go runtime profiles. +// +// Pprof endpoints have no authentication and can expose process runtime data, +// so accepting a hostname or an unspecified address here would make a +// configuration typo into an externally reachable diagnostics service. +func ListenPprof(addr string) (net.Listener, error) { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, fmt.Errorf("ListenPprof: split address: %w", err) + } + ip := net.ParseIP(host) + if ip == nil || !ip.IsLoopback() { + return nil, fmt.Errorf("ListenPprof: address %q is not a loopback IP address", addr) + } + ln, err := net.Listen("tcp", addr) + if err != nil { + return nil, fmt.Errorf("ListenPprof: %w", err) + } + return ln, nil +} + +// ServePprof serves Go runtime profiles on ln until the daemon is asked to stop +// or the context is cancelled. +// +// The profiling listener is separate from the Bazel HTTP listener: profiles +// expose process runtime data, while a cache address must continue to serve only +// cache routes. Like the other optional TCP listeners, it keeps the daemon +// alive for as long as it is configured to serve. +func (s *Server) ServePprof(ctx context.Context, ln net.Listener) error { + s.enter() + defer s.leave() + + srv := &http.Server{ + Handler: newPprofHandler(), + ReadHeaderTimeout: pprofReadHeaderTimeout, + IdleTimeout: pprofIdleTimeout, + // Request contexts derive from ctx so stopping the daemon also stops an + // in-flight profile capture. + BaseContext: func(net.Listener) context.Context { return ctx }, + } + + done := make(chan struct{}) + defer close(done) + go func() { + select { + case <-ctx.Done(): + case <-s.stopped: + case <-done: + return + } + // A profile capture is allowed to finish, but one that cannot do so must + // not keep the cache process alive indefinitely. + sctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), pprofShutdownGrace) + defer cancel() + _ = srv.Shutdown(sctx) + }() + + if err := srv.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) { + return fmt.Errorf("ServePprof: %w", err) + } + return nil +} + +// newPprofHandler creates the fixed pprof route set without registering it on +// the process-wide default mux, which could otherwise leak those routes onto an +// unrelated HTTP server in the same process. +func newPprofHandler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + return mux +} diff --git a/internal/daemon/pprof_test.go b/internal/daemon/pprof_test.go new file mode 100644 index 0000000..9c1d677 --- /dev/null +++ b/internal/daemon/pprof_test.go @@ -0,0 +1,93 @@ +// Copyright 2026 The plaid-cache authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package daemon + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + "time" +) + +// startPprof runs ServePprof on a loopback listener and returns its base URL. +// The daemon is stopped and the goroutine joined at test end, which also pins +// that stopping the daemon reaches the profiling listener. +func startPprof(t *testing.T, s *Server) string { + t.Helper() + ln, err := ListenPprof("127.0.0.1:0") + if err != nil { + t.Fatalf("Listen: %v", err) + } + done := make(chan error, 1) + go func() { done <- s.ServePprof(context.Background(), ln) }() + t.Cleanup(func() { + s.Stop() + select { + case err := <-done: + if err != nil { + t.Errorf("ServePprof: %v", err) + } + case <-time.After(serveStopTimeout): + t.Errorf("ServePprof did not return within %v", serveStopTimeout) + } + }) + return "http://" + ln.Addr().String() +} + +// TestListenPprofRejectsNonLoopbackAddresses pins that unauthenticated +// profiling endpoints cannot be exposed by choosing an all-interface address. +func TestListenPprofRejectsNonLoopbackAddresses(t *testing.T) { + for _, addr := range []string{"0.0.0.0:0", "[::]:0", "localhost:0"} { + ln, err := ListenPprof(addr) + if err == nil { + _ = ln.Close() + t.Fatalf("ListenPprof(%q) succeeded, want a loopback refusal", addr) + } + } +} + +// TestServePprofServesTheRuntimeProfileIndex pins that the separate loopback +// listener exposes the standard runtime profiles without adding them to Bazel. +func TestServePprofServesTheRuntimeProfileIndex(t *testing.T) { + cfg := newTestConfig(t) + s := newTestServer(t, cfg) + base := startPprof(t, s) + + resp, err := http.Get(base + "/debug/pprof/") + if err != nil { + t.Fatalf("GET pprof index: %v", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET pprof index = %d, want 200", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read pprof index: %v", err) + } + for _, profile := range []string{"heap", "goroutine", "mutex", "block", "profile", "trace"} { + if !strings.Contains(string(body), profile) { + t.Fatalf("pprof index does not list %q: %s", profile, body) + } + } +} + +// TestServeBazelDoesNotServePprof pins that enabling the profiling listener +// cannot broaden the Bazel HTTP handler with process diagnostics. +func TestServeBazelDoesNotServePprof(t *testing.T) { + cfg := newTestConfig(t) + s := newTestServer(t, cfg) + base := startBazel(t, s) + + resp, err := http.Get(base + "/debug/pprof/") + if err != nil { + t.Fatalf("GET pprof path on Bazel listener: %v", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("GET pprof path on Bazel listener = %d, want 404", resp.StatusCode) + } +}