diff --git a/runner/eval-cli.ts b/runner/eval-cli.ts index 3b50c6fe..9011c309 100644 --- a/runner/eval-cli.ts +++ b/runner/eval-cli.ts @@ -7,6 +7,7 @@ import { DEFAULT_MAX_TEST_REPAIR_ATTEMPTS, DEFAULT_MODEL_NAME, DEFAULT_PROMPT_TIMEOUT_RETRIES, + REPORTS_ROOT_DIR, } from './configuration/constants.js'; import {generateCodeAndAssess} from './orchestration/generate.js'; import {logReportToConsole, writeReportToDisk} from './reporting/report-logging.js'; @@ -236,7 +237,7 @@ async function handler(cliArgs: Arguments): Promise { }); logReportToConsole(runInfo); - await writeReportToDisk(runInfo, runInfo.details.summary.environmentId); + await writeReportToDisk(runInfo, runInfo.details.summary.environmentId, REPORTS_ROOT_DIR); } catch (error: unknown) { if (error instanceof UserFacingError) { console.error(chalk.red(error.message)); diff --git a/runner/index.ts b/runner/index.ts index b2f45280..9ee2e212 100644 --- a/runner/index.ts +++ b/runner/index.ts @@ -51,3 +51,4 @@ export {type ServeTestingResult} from './workers/serve-testing/worker-types.js'; export {replaceAtReferencesInPrompt} from './utils/prompt-at-references.js'; export {extractRubrics} from './utils/extract-rubrics.js'; export {combineReports} from './utils/combine-reports.mjs'; +export {writeReportToDisk} from './reporting/report-logging.js'; diff --git a/runner/reporting/report-logging.ts b/runner/reporting/report-logging.ts index 84aad8bf..8a938fa1 100644 --- a/runner/reporting/report-logging.ts +++ b/runner/reporting/report-logging.ts @@ -2,7 +2,7 @@ import {join} from 'path'; import chalk from 'chalk'; import boxen from 'boxen'; import {IndividualAssessmentState, RunInfo, ScoreBucket} from '../shared-interfaces.js'; -import {DEFAULT_AUTORATER_MODEL_NAME, REPORTS_ROOT_DIR} from '../configuration/constants.js'; +import {DEFAULT_AUTORATER_MODEL_NAME} from '../configuration/constants.js'; import {calculateBuildAndCheckStats} from '../ratings/stats.js'; import {safeWriteFile} from '../file-system-utils.js'; import {BuildResultStatus} from '../workers/builder/builder-types.js'; @@ -17,7 +17,6 @@ import { } from './format.js'; import {Environment} from '../configuration/environment.js'; import {groupSimilarReports} from '../orchestration/grouping.js'; -import {LocalExecutor} from '../orchestration/executors/local-executor.js'; /** * Generates a structured report on fs, based on the assessment run information. @@ -38,14 +37,19 @@ import {LocalExecutor} from '../orchestration/executors/local-executor.js'; * * @param runInfo An object containing all details and results of the assessment run. * @param id ID of the environment that was used for the eval. + * @param reportsRootDir Root directory where the reports are written to. * @returns The original `runInfo` object, allowing for chaining. */ -export async function writeReportToDisk(runInfo: RunInfo, id: string): Promise { +export async function writeReportToDisk( + runInfo: RunInfo, + id: string, + reportsRootDir: string, +): Promise { // Sanitize report name: allow only a-z, A-Z, 0-9, and hyphens. Replace others with a hyphen. const sanitizedReportName = runInfo.details.reportName.replace(/[^a-zA-Z0-9-]/g, '-'); const {results} = runInfo; - const reportBaseDir = join(REPORTS_ROOT_DIR, id, sanitizedReportName); + const reportBaseDir = join(reportsRootDir, id, sanitizedReportName); // Write `summary.json` file, which contains **all** available info. const summaryJsonPath = join(reportBaseDir, 'summary.json');