feat(database): add infra-error classifier and read-retry util

Adds a shared classifier (isInfrastructureError / looksLikeConnectivityError)
recognising connection-blip failures (P1001/P1002/P1008/P1017, ECONNRESET,
"connection terminated", "server has closed the connection"), and
withInfraRetry — a retry helper gated by an enabled kill-switch (default off)
and the existing TokenBucketRetryBudget.

Only for operations safe to run more than once (reads, or writes made
idempotent); it never authorises retrying a bare non-idempotent write.

TRI-13553

Author: d-cs

internal-packages/database/src/index.ts

@@ -1,3 +1,5 @@
 export * from "../generated/prisma";
 export * from "./boundedIn";
+export * from "./infraError";
+export * from "./infraRetry";
 export * from "./transaction";

internal-packages/database/src/infraError.test.ts

@@ -0,0 +1,113 @@
+import { describe, expect, it } from "vitest";
+import { Prisma } from "../generated/prisma";
+import {
+  isInfrastructureError,
+  isRetryableInfrastructureError,
+  looksLikeConnectivityError,
+} from "./infraError";
+
+const known = (code: string, message = "") =>
+  new Prisma.PrismaClientKnownRequestError(message, { code, clientVersion: "6.14.0" });
+
+describe("isInfrastructureError", () => {
+  it("treats connection-level Prisma codes as infrastructure errors", () => {
+    for (const code of ["P1001", "P1002", "P1008", "P1017"]) {
+      expect(isInfrastructureError(known(code, "boom"))).toBe(true);
+    }
+  });
+
+  it("does not treat query/validation errors as infrastructure errors", () => {
+    expect(isInfrastructureError(known("P2025", "record not found"))).toBe(false);
+    expect(isInfrastructureError(known("P2002", "unique constraint"))).toBe(false);
+  });
+
+  it("treats P2010 as infrastructure only when the message looks like connectivity loss", () => {
+    expect(isInfrastructureError(known("P2010", "Connection terminated unexpectedly"))).toBe(true);
+    expect(isInfrastructureError(known("P2010", "syntax error at or near"))).toBe(false);
+  });
+
+  it("treats init / panic / unknown request errors as infrastructure errors", () => {
+    expect(
+      isInfrastructureError(new Prisma.PrismaClientInitializationError("no db", "6.14.0"))
+    ).toBe(true);
+  });
+
+  it("recognises raw connectivity errno / messages", () => {
+    expect(isInfrastructureError({ code: "ECONNRESET" })).toBe(true);
+    expect(isInfrastructureError(new Error("server has closed the connection"))).toBe(true);
+    expect(isInfrastructureError(new Error("column does not exist"))).toBe(false);
+  });
+});
+
+describe("looksLikeConnectivityError", () => {
+  it("matches known errno codes and message fragments", () => {
+    expect(looksLikeConnectivityError({ code: "EHOSTUNREACH" })).toBe(true);
+    expect(looksLikeConnectivityError(new Error("Can't reach database server"))).toBe(true);
+    expect(looksLikeConnectivityError(new Error("relation does not exist"))).toBe(false);
+  });
+});
+
+describe("isRetryableInfrastructureError", () => {
+  it("retries connection-level codes and connectivity errnos/messages", () => {
+    for (const code of ["P1001", "P1002", "P1008", "P1017"]) {
+      expect(isRetryableInfrastructureError(known(code, "boom"))).toBe(true);
+    }
+    expect(isRetryableInfrastructureError({ code: "ECONNRESET" })).toBe(true);
+    expect(isRetryableInfrastructureError(new Error("server has closed the connection"))).toBe(
+      true
+    );
+  });
+
+  it("does not retry query/validation errors", () => {
+    expect(isRetryableInfrastructureError(known("P2025", "record not found"))).toBe(false);
+    expect(isRetryableInfrastructureError(new Error("column does not exist"))).toBe(false);
+  });
+
+  it("retries an init error only with a connectivity signal (not a permanent one)", () => {
+    expect(
+      isRetryableInfrastructureError(
+        new Prisma.PrismaClientInitializationError("Can't reach database server", "6.14.0", "P1001")
+      )
+    ).toBe(true);
+    expect(
+      isRetryableInfrastructureError(
+        new Prisma.PrismaClientInitializationError(
+          "Authentication failed against database server",
+          "6.14.0",
+          "P1000"
+        )
+      )
+    ).toBe(false);
+  });
+
+  it("never retries a Rust-engine panic", () => {
+    expect(
+      isRetryableInfrastructureError(new Prisma.PrismaClientRustPanicError("panic", "6.14.0"))
+    ).toBe(false);
+  });
+
+  it("never retries pool exhaustion (P2024), even though its message looks like connectivity", () => {
+    const poolMsg = "Timed out fetching a new connection from the connection pool";
+    expect(isRetryableInfrastructureError(known("P2024", poolMsg))).toBe(false);
+    expect(isRetryableInfrastructureError(new Error(poolMsg))).toBe(false);
+    // The broad classifier still flags it (used for logging, not retry).
+    expect(isInfrastructureError(new Error(poolMsg))).toBe(true);
+  });
+
+  it("retries an unknown-request error only with a connectivity signal", () => {
+    expect(
+      isRetryableInfrastructureError(
+        new Prisma.PrismaClientUnknownRequestError("connection terminated unexpectedly", {
+          clientVersion: "6.14.0",
+        })
+      )
+    ).toBe(true);
+    expect(
+      isRetryableInfrastructureError(
+        new Prisma.PrismaClientUnknownRequestError("unexpected engine failure", {
+          clientVersion: "6.14.0",
+        })
+      )
+    ).toBe(false);
+  });
+});

internal-packages/database/src/infraError.ts

@@ -0,0 +1,101 @@
+import { Prisma } from "../generated/prisma";
+
+// Prisma connectivity / infrastructure error codes — connection-level failures,
+// not query- or validation-level ones (e.g. P1001 "Can't reach database server").
+const INFRASTRUCTURE_PRISMA_CODES = new Set(["P1001", "P1002", "P1008", "P1017"]);
+
+const CONNECTIVITY_ERRNO = new Set([
+  "ECONNREFUSED",
+  "ENOTFOUND",
+  "ETIMEDOUT",
+  "ECONNRESET",
+  "EHOSTUNREACH",
+  "EPIPE",
+]);
+
+const CONNECTIVITY_MESSAGE =
+  /ECONNREFUSED|ENOTFOUND|ETIMEDOUT|ECONNRESET|EHOSTUNREACH|database not reachable|can't reach database|connection terminated|server has closed the connection|timed out fetching a new connection/i;
+
+// Connection-pool exhaustion (P2024). Matched only to EXCLUDE it from retry:
+// retrying against an already-exhausted pool deepens the contention rather than
+// riding out a blip (the transaction-start retry gate excludes it for the same reason).
+const POOL_EXHAUSTION_MESSAGE = /timed out fetching a new connection/i;
+
+/** True for an errno/message that looks like a lost or unreachable connection. */
+export function looksLikeConnectivityError(error: unknown): boolean {
+  const e = error as { code?: unknown; message?: unknown };
+  if (typeof e?.code === "string" && CONNECTIVITY_ERRNO.has(e.code)) {
+    return true;
+  }
+  return typeof e?.message === "string" && CONNECTIVITY_MESSAGE.test(e.message);
+}
+
+/**
+ * True when `error` is a Prisma infrastructure/connectivity failure (DB
+ * unreachable, timed out, connection dropped) rather than a query- or
+ * validation-level error. Broad by design (matches the classifier used for
+ * logging); for the retry decision use {@link isRetryableInfrastructureError}.
+ */
+export function isInfrastructureError(error: unknown): boolean {
+  if (
+    error instanceof Prisma.PrismaClientInitializationError ||
+    error instanceof Prisma.PrismaClientRustPanicError ||
+    error instanceof Prisma.PrismaClientUnknownRequestError
+  ) {
+    return true;
+  }
+
+  if (error instanceof Prisma.PrismaClientKnownRequestError) {
+    if (INFRASTRUCTURE_PRISMA_CODES.has(error.code)) {
+      return true;
+    }
+    return error.code === "P2010" && looksLikeConnectivityError(error);
+  }
+
+  return looksLikeConnectivityError(error);
+}
+
+/**
+ * True when `error` is a *transient* infrastructure failure worth retrying — a
+ * genuine connectivity blip, not a permanent one. Narrower than
+ * {@link isInfrastructureError}: an initialization or unknown-request error
+ * counts only when it carries a connectivity signal (so a bad-URL / auth /
+ * database-selection failure is NOT retried), and a Rust-engine panic is never
+ * retried. This is the default retry gate for `withInfraRetry`.
+ */
+export function isRetryableInfrastructureError(error: unknown): boolean {
+  // Never retry pool exhaustion (P2024): another attempt only competes for the
+  // same exhausted pool. Checked before the connectivity fallbacks because its
+  // message otherwise matches CONNECTIVITY_MESSAGE.
+  const message = (error as { message?: unknown })?.message;
+  if (
+    (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2024") ||
+    (typeof message === "string" && POOL_EXHAUSTION_MESSAGE.test(message))
+  ) {
+    return false;
+  }
+
+  if (error instanceof Prisma.PrismaClientRustPanicError) {
+    return false;
+  }
+
+  if (error instanceof Prisma.PrismaClientInitializationError) {
+    return (
+      (typeof error.errorCode === "string" && INFRASTRUCTURE_PRISMA_CODES.has(error.errorCode)) ||
+      looksLikeConnectivityError(error)
+    );
+  }
+
+  if (error instanceof Prisma.PrismaClientUnknownRequestError) {
+    return looksLikeConnectivityError(error);
+  }
+
+  if (error instanceof Prisma.PrismaClientKnownRequestError) {
+    if (INFRASTRUCTURE_PRISMA_CODES.has(error.code)) {
+      return true;
+    }
+    return error.code === "P2010" && looksLikeConnectivityError(error);
+  }
+
+  return looksLikeConnectivityError(error);
+}