test(firestore): retry failed system tests instead of the whole suite

[MarkDuckworth]

The system tests exercise real RPCs over the network, so an individual test can occasionally fail for reasons unrelated to the code under test, such as a dropped connection in the test environment. Previously a single such failure meant re-running the entire suite.

Mocha now retries a failed system test up to 3 times. Retries are scoped to the failing test, so passing tests still run exactly once. Unit and conformance tests are deterministic and are never retried, so genuine flakiness there stays visible. Set MOCHA_RETRIES=0 to disable retries when investigating a failure.

scripts/mocha-retry.js logs every retried attempt, along with the error that caused it, and prints a summary at the end of the run, so that retries do not silently hide flakiness. It also caps the total number of retries per run (MOCHA_MAX_TOTAL_RETRIES, default 15). Widespread failures usually indicate a systemic problem, such as expired credentials or a misconfigured target, rather than flakiness, and retrying every test would multiply the time taken to surface it.

Note that Mocha does not retry failures originating in before/after hooks, only those in tests and their beforeEach/afterEach hooks.

[gemini-code-assist]

Code Review

This pull request introduces a test retry mechanism for Firestore system tests by updating .mocharc.js and adding a custom Mocha runner patch in mocha-retry.js to log retries, print a summary, and implement a circuit breaker. The review feedback highlights two important issues: first, spreading config.require directly can cause a crash if it is configured as a single string rather than an array; second, using module-level globals for retry state can lead to state leakage across runs in watch mode, which can be resolved by encapsulating the state within the patched run method.

[gemini-code-assist]

If config.require is configured as a single string (e.g., config.require = 'ts-node/register') rather than an array, spreading it with ...(config.require || []) will spread the string character-by-character. This will result in Mocha attempting to require each individual character as a module (e.g., requiring 't', 's', etc.), causing the test run to crash with a Cannot find module error.

We should safely normalize config.require to an array first.

  const requires = Array.isArray(config.require)
    ? config.require
    : config.require
    ? [config.require]
    : [];
  config.require = [
    ...requires,
    require.resolve('./scripts/mocha-retry.js'),
  ];

[gemini-code-assist]

Defining totalRetries, retriesDisabled, and retried as module-level globals means that their state will persist across multiple test runs if Mocha is executed programmatically or in watch mode (mocha --watch). In watch mode, the module is only loaded once, so subsequent test runs will inherit the dirty state from previous runs (e.g., the retry budget might remain permanently exhausted, and the summary will accumulate tests from previous runs).

To prevent this, we should encapsulate the retry state, helper functions, and summary printer inside the patched Mocha.Runner.prototype.run method so that a fresh state is created for each individual test run.

// Patch Runner#run so that listeners are attached to whichever Runner instance
// Mocha creates, before the run starts.
const originalRun = Mocha.Runner.prototype.run;
Mocha.Runner.prototype.run = function (fn, opts) {
  /** Retries consumed so far across this run. */
  let totalRetries = 0;

  /** Whether the circuit breaker has tripped. */
  let retriesDisabled = false;

  /**
   * Tests that failed at least one attempt, keyed by full title.
   * @type {Map<string, {attempts: number, passed: boolean, errors: string[]}>}
   */
  const retried = new Map();

  /**
   * @param {Mocha.Test} test
   * @return {{attempts: number, passed: boolean, errors: string[]}}
   */
  const entryFor = (test) => {
    const title = test.fullTitle();
    let entry = retried.get(title);
    if (!entry) {
      entry = {attempts: 0, passed: false, errors: []};
      retried.set(title, entry);
    }
    return entry;
  };

  const printSummary = () => {
    if (retried.size === 0) return;

    const flaky = [...retried].filter(([, e]) => e.passed);
    const broken = [...retried].filter(([, e]) => !e.passed);
    const rule = '-'.repeat(70);

    console.log(`\n  Retry summary\n  ${rule}`);
    if (flaky.length > 0) {
      console.log(`  ${flaky.length} test(s) passed only after being retried:`);
      for (const [title, entry] of flaky) {
        console.log(`    FLAKY: ${title} (passed on attempt ${entry.attempts})`);
        entry.errors.forEach((err, i) => {
          console.log(`             attempt ${i + 1}: ${err}`);
        });
      }
    }
    if (broken.length > 0) {
      console.log(`  ${broken.length} test(s) failed every attempt:`);
      for (const [title, entry] of broken) {
        console.log(`    FAILED: ${title} (${entry.attempts} attempt(s))`);
      }
    }
    if (retriesDisabled) {
      console.log(
        `  NOTE: the retry budget (${MAX_TOTAL_RETRIES}) was exhausted, so later ` +
          'tests were not retried at all.',
      );
    }
    console.log(`  ${rule}\n`);
  };

  this.on(EVENT_TEST_RETRY, (test, err) => {
    // `currentRetry()` is 0-based and still reflects the attempt that just
    // failed at the time this event fires.
    const attempt = test.currentRetry() + 1;
    const total = test.retries() + 1;
    const entry = entryFor(test);
    entry.attempts = attempt;
    entry.errors.push(summarizeError(err));
    console.log(
      `  [retry] attempt ${attempt}/${total} failed, retrying: ${test.fullTitle()}\n` +
        `          ${summarizeError(err)}`,
    );

    if (++totalRetries === MAX_TOTAL_RETRIES && !retriesDisabled) {
      retriesDisabled = true;
      // So many tests are failing that this is unlikely to be an isolated
      // problem. Stop retrying so the run fails fast instead of spending
      // several times the usual duration to reach the same conclusion. The
      // already-queued retry of the current test still runs.
      this.suite.eachTest(t => t.retries(0));
      console.log(
        `  [retry] retry budget of ${MAX_TOTAL_RETRIES} exhausted; disabling ` +
          'retries for the remaining tests. Widespread failures usually mean a ' +
          'systemic problem (credentials, target configuration, connectivity) ' +
          'rather than flakiness. Set MOCHA_MAX_TOTAL_RETRIES to change this ' +
          'budget.',
      );
    }
  });

  this.on(EVENT_TEST_PASS, test => {
    if (test.currentRetry() === 0) return;
    const entry = entryFor(test);
    entry.attempts = test.currentRetry() + 1;
    entry.passed = true;
    console.log(
      `  [retry] FLAKY: passed on attempt ${entry.attempts}/${test.retries() + 1}: ` +
        test.fullTitle(),
    );
  });

  this.on(EVENT_TEST_FAIL, test => {
    // Only interested in tests that exhausted their retries; a test with no
    // retries left is reported normally by the reporter.
    if (!test.currentRetry || test.currentRetry() === 0) return;
    const entry = entryFor(test);
    entry.attempts = test.currentRetry() + 1;
    console.log(
      `  [retry] exhausted ${entry.attempts} attempt(s), giving up: ${test.fullTitle()}`,
    );
  });

  this.once(EVENT_RUN_END, printSummary);

  return originalRun.call(this, fn, opts);
};