ApiCore/ApiCore-Backend/scripts/eval-voice.ts

/**
 * Evaluates voice extraction against the gold set (test/fixtures/voice/gold).
 *
 *   npm run eval:voice                      offline, replays the expected answers, no network, free
 *   npm run eval:voice -- --live --yes      real Claude calls, costs money
 *   npm run eval:voice -- --live --yes --model=claude-haiku-5-5
 *
 * Live mode needs ANTHROPIC_API_KEY in the environment AND both --live and
 * --yes; it is never the default and never runs in CI (no workflow calls
 * this script). This is an operator tool, not a request path: it lives
 * outside src/, so the network guard (src/common/no-network-outside-loops)
 * does not scan it, and it talks to the adapter only through the
 * StructuredModel port.
 *
 * Exit code: 0 pass, 1 below the thresholds (precision < 98 % or recall <
 * 85 %, see EVAL_THRESHOLDS), 2 usage or setup error.
 */
import { join } from 'node:path';
import { anthropicApiKey } from '../src/voice/anthropic-config';
import { formatReport } from '../src/voice/eval/eval-report';
import { loadGoldCases } from '../src/voice/eval/gold-case';
import { estimateCostUsd } from '../src/voice/eval/voice-prices';
import {
  offlineModelFor,
  runEval,
  summarize,
} from '../src/voice/eval/voice-eval';
import { AnthropicStructuredModel } from '../src/voice/extraction/anthropic-structured-model';
import { FAKE_USAGE } from '../src/voice/extraction/fake-structured-model';
import { DEFAULT_VOICE_MODEL, VOICE_MODELS } from '../src/voice/voice-config';

const GOLD_DIR = join(__dirname, '..', 'test', 'fixtures', 'voice', 'gold');
const EXIT_PASS = 0;
const EXIT_BELOW_THRESHOLD = 1;
const EXIT_USAGE = 2;

interface Options {
  live: boolean;
  yes: boolean;
  model: string;
}

function parseArgs(args: readonly string[]): Options | string {
  let live = false;
  let yes = false;
  let model: string = DEFAULT_VOICE_MODEL;
  for (const arg of args) {
    if (arg === '--live') {
      live = true;
    } else if (arg === '--yes') {
      yes = true;
    } else if (arg.startsWith('--model=')) {
      model = arg.slice('--model='.length);
    } else {
      return `Unknown argument: ${arg}`;
    }
  }
  if (!(VOICE_MODELS as readonly string[]).includes(model)) {
    return `--model must be one of ${VOICE_MODELS.join(', ')}`;
  }
  return { live, yes, model };
}

function fail(message: string): never {
  console.error(message);
  process.exit(EXIT_USAGE);
}

async function main(): Promise<number> {
  const options = parseArgs(process.argv.slice(2));
  if (typeof options === 'string') {
    return fail(options);
  }
  const cases = loadGoldCases(GOLD_DIR);
  if (!options.live) {
    const results = await runEval(
      cases,
      offlineModelFor(options.model),
      options.model,
    );
    console.log(
      formatReport(results, summarize(results), {
        mode: 'offline',
        model: options.model,
      }),
    );
    return summarizeExit(results);
  }
  const key = anthropicApiKey(process.env);
  if (key === null) {
    return fail('--live needs a valid ANTHROPIC_API_KEY in the environment.');
  }
  const estimate = estimateCostUsd(options.model, {
    input: FAKE_USAGE.input * cases.length,
    output: FAKE_USAGE.output * cases.length,
    cacheRead: 0,
    cacheWrite: 0,
  });
  console.warn(
    `WARNING: ${cases.length} real Claude calls on ${options.model}, roughly ${
      estimate === null ? 'an unknown amount' : `$${estimate.toFixed(2)}`
    } of API cost.`,
  );
  if (!options.yes) {
    return fail('Add --yes to confirm and run the live evaluation.');
  }
  const live = new AnthropicStructuredModel(key);
  const results = await runEval(cases, () => live, options.model);
  console.log(
    formatReport(results, summarize(results), {
      mode: 'live',
      model: options.model,
    }),
  );
  return summarizeExit(results);
}

function summarizeExit(results: Parameters<typeof summarize>[0]): number {
  return summarize(results).failures.length === 0
    ? EXIT_PASS
    : EXIT_BELOW_THRESHOLD;
}

main().then(
  (code) => process.exit(code),
  (error: unknown) => {
    console.error(error instanceof Error ? error.message : 'eval failed');
    process.exit(EXIT_USAGE);
  },
);