fix(engine): fail closed on invalid epochs

cmd/ob/ops_contract_test.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/labstack/onebox/internal/app"
 	"github.com/labstack/onebox/internal/transport"
 )
 
@@ -179,6 +180,8 @@ services: {postgres: 17}
 	}
 	fake := &transport.Fake{HostName: "example.invalid", Dynamic: func(command string) (transport.Result, bool) {
 		switch {
+		case strings.HasPrefix(command, ": ob-epoch-probe;"):
+			return transport.Result{ExitCode: app.ProbeAbsent}, true
 		case strings.Contains(command, "/_host/owner"):
 			return transport.Result{Stdout: "shop\n"}, true
 		case strings.Contains(command, " logs "):

internal/engine/backup_lock.go

@@ -110,7 +110,13 @@ func (e *Engine) AcquireBackupLock(ctx context.Context, service, operationID str
 			return 0, err
 		}
 		if res.ExitCode == 0 {
+			if e.backupLockVals == nil {
+				e.backupLockVals = make(map[string]string)
+				e.backupFenceVals = make(map[string]string)
+			}
+			e.backupLockVals[service] = lockValue
 			if err := e.writeBackupFence(ctx, service, operationID, epoch, lockValue); err != nil {
+				e.ReleaseBackupLock(service)
 				return 0, err
 			}
 			return epoch, nil
@@ -159,17 +165,14 @@ func (e *Engine) AcquireBackupLock(ctx context.Context, service, operationID str
 }
 
 func (e *Engine) nextBackupEpoch(ctx context.Context, service string) (int, error) {
-	result, err := e.T.Run(ctx, "cat "+q(e.backupEpochPath(service))+" 2>/dev/null || echo 0")
-	if err != nil {
-		return 0, err
-	}
-	previous, _ := strconv.Atoi(strings.TrimSpace(result.Stdout))
-	return previous + 1, nil
+	return e.nextEpoch(ctx, e.backupEpochPath(service))
 }
 
 func (e *Engine) writeBackupFence(ctx context.Context, service, operationID string, epoch int, lockValue string) error {
 	fenceValue := operationID + " " + strconv.Itoa(epoch)
-	command := `if [ "$(cat ` + q(e.backupLockPath(service)) + ` 2>/dev/null)" = ` + q(lockValue) + ` ]; then echo ` + strconv.Itoa(epoch) + ` > ` + q(e.backupEpochPath(service)) + ` && echo ` + q(fenceValue) + ` > ` + q(e.backupFencePath(service)) + `; else echo ob-backup-lock-lost >&2; exit 96; fi`
+	command := `if [ "$(cat ` + q(e.backupLockPath(service)) + ` 2>/dev/null)" = ` + q(lockValue) + ` ]; then ` +
+		atomicEpochWriteCmd(e.backupEpochPath(service), epoch) + `; echo ` + q(fenceValue) + ` > ` + q(e.backupFencePath(service)) +
+		`; else echo ob-backup-lock-lost >&2; exit 96; fi`
 	result, err := e.T.Run(ctx, command)
 	if err != nil {
 		return err

internal/engine/backup_lock_test.go

@@ -81,7 +81,7 @@ func TestBackupLockHonorsCancellation(t *testing.T) {
 }
 
 func TestBackupLockReclaimsStaleHolderWithNewFence(t *testing.T) {
-	holder := `{"owner":"operator","operation_id":"backup-old","service":"database","epoch":4,"ttl_s":10,"acquired_at":"2026-08-07T11:00:00Z"}`
+	holder := `{"owner":"operator","operation_id":"backup-same","service":"database","epoch":4,"ttl_s":10,"acquired_at":"2026-08-07T11:00:00Z"}`
 	createAttempts := 0
 	fake := &transport.Fake{Dynamic: func(command string) (transport.Result, bool) {
 		switch {
@@ -91,25 +91,25 @@ func TestBackupLockReclaimsStaleHolderWithNewFence(t *testing.T) {
 				return transport.Result{ExitCode: 1}, true
 			}
 			return transport.Result{}, true
-		case strings.HasPrefix(command, "cat ") && strings.Contains(command, "database.epoch"):
+		case strings.Contains(command, "cat ") && strings.Contains(command, "database.epoch"):
 			return transport.Result{Stdout: "4\n"}, true
 		case strings.HasPrefix(command, "cat ") && strings.Contains(command, "database.lock"):
 			return transport.Result{Stdout: holder + "\n"}, true
 		case strings.HasPrefix(command, "if [ -L ") && strings.Contains(command, "database.lock"):
-			return transport.Result{Stdout: "11\n"}, true
+			return transport.Result{Stdout: "1\n"}, true
 		}
 		return transport.Result{}, false
 	}}
 	engine := backupLockTestEngine(fake)
 
-	epoch, err := engine.AcquireBackupLock(context.Background(), "database", "backup-new", 0)
+	epoch, err := engine.AcquireBackupLock(context.Background(), "database", "backup-same", 0)
 	if err != nil {
 		t.Fatalf("reclaim stale backup lock: %v", err)
 	}
 	if epoch != 5 || createAttempts != 2 {
 		t.Fatalf("reclaimed epoch/attempts = %d/%d, want 5/2", epoch, createAttempts)
 	}
-	if got := engine.backupFenceVals["database"]; got != "backup-new 5" {
+	if got := engine.backupFenceVals["database"]; got != "backup-same 5" || got == "backup-same 4" {
 		t.Fatalf("backup fence = %q", got)
 	}
 }

internal/engine/epoch.go

@@ -0,0 +1,62 @@
+package engine
+
+import (
+	"context"
+	"fmt"
+	"path"
+	"strconv"
+	"strings"
+
+	"github.com/labstack/onebox/internal/app"
+)
+
+// nextEpoch reads one durable fencing authority. Absence is the only state
+// that means zero: an unreadable or malformed value must never reissue an epoch
+// a stale runner may still hold.
+func (e *Engine) nextEpoch(ctx context.Context, epochPath string) (int, error) {
+	result, err := e.T.Run(ctx, epochProbeCmd(epochPath))
+	if err != nil {
+		return 0, err
+	}
+	switch result.ExitCode {
+	case 0:
+		// Parsed below.
+	case app.ProbeAbsent:
+		return 1, nil
+	case app.ProbeUnreadable:
+		return 0, fmt.Errorf("epoch file %s exists but cannot be read", epochPath)
+	case app.ProbeNotRegular:
+		return 0, fmt.Errorf("epoch file %s is not a regular file", epochPath)
+	case app.ProbeUndetermined:
+		return 0, fmt.Errorf("epoch file %s cannot be observed because an ancestor directory is not searchable", epochPath)
+	case app.ProbeStatePathNotDirectory:
+		return 0, fmt.Errorf("epoch file %s cannot be read because an ancestor path is not a directory", epochPath)
+	default:
+		return 0, fmt.Errorf("read epoch file %s failed (exit %d): %s", epochPath, result.ExitCode, strings.TrimSpace(result.Stderr))
+	}
+
+	raw := strings.TrimSpace(result.Stdout)
+	previous, err := strconv.Atoi(raw)
+	if err != nil || previous < 0 || previous == int(^uint(0)>>1) {
+		return 0, fmt.Errorf("epoch file %s contains invalid value %q", epochPath, raw)
+	}
+	return previous + 1, nil
+}
+
+// epochProbeCmd distinguishes a file that is genuinely absent from one hidden
+// by permissions or replaced with another kind of filesystem object.
+func epochProbeCmd(epochPath string) string {
+	p := q(epochPath)
+	return ": ob-epoch-probe; if [ ! -e " + p + " ] && [ ! -L " + p + " ]; then " +
+		app.UndeterminedArm(epochPath) + "exit " + strconv.Itoa(app.ProbeAbsent) + "; fi; " +
+		"if [ ! -f " + p + " ] || [ -L " + p + " ]; then exit " + strconv.Itoa(app.ProbeNotRegular) + "; fi; " +
+		"if [ ! -r " + p + " ]; then exit " + strconv.Itoa(app.ProbeUnreadable) + "; fi; cat " + p
+}
+
+// atomicEpochWriteCmd writes beside the epoch and renames over it. A killed
+// shell can leave a disposable temp file, never a truncated authority.
+func atomicEpochWriteCmd(epochPath string, epoch int) string {
+	template := epochPath + ".tmp.XXXXXX"
+	return "set -eu; test -d " + q(path.Dir(epochPath)) + "; umask 077; tmp=$(mktemp " + q(template) + "); " +
+		`trap 'rm -f "$tmp"' 0 1 2 15; printf '%s\n' ` + strconv.Itoa(epoch) + ` > "$tmp"; chmod 600 "$tmp"; mv -f "$tmp" ` + q(epochPath) + `; trap - 0 1 2 15`
+}