ApiCore/ApiCore-Backend/src/voice/eval/voice-eval.spec.ts

import { existsSync, readFileSync, readdirSync } from 'node:fs';
import { join } from 'node:path';
import { formatReport } from './eval-report';
import { loadGoldCases, parseGoldCase, type GoldCase } from './gold-case';
import {
  EVAL_THRESHOLDS,
  offlineModelFor,
  runCase,
  runEval,
  summarize,
} from './voice-eval';
import { estimateCostUsd } from './voice-prices';
import { FakeStructuredModel } from '../extraction/fake-structured-model';

const GOLD_DIR = join(
  __dirname,
  '..',
  '..',
  '..',
  'test',
  'fixtures',
  'voice',
  'gold',
);
const MODEL = 'claude-sonnet-5-5';

const VALID = {
  id: 'g99-sample',
  title: 'Sample',
  transcript: 'Vier Brutwaben und Königin gesehen.',
  expected: [{ field: 'broodFrames', value: 4, quote: 'Vier Brutwaben' }],
  unclear: [],
  cannedExtra: [],
  expectedDropped: 0,
};

describe('gold set', () => {
  it('holds 30 files and every one matches the schema', () => {
    const files = readdirSync(GOLD_DIR).filter((name) =>
      name.endsWith('.json'),
    );

    expect(files).toHaveLength(30);
    for (const file of files) {
      const parsed = parseGoldCase(
        JSON.parse(readFileSync(join(GOLD_DIR, file), 'utf8')) as unknown,
      );
      expect({ file, errors: parsed.ok ? [] : parsed.errors }).toEqual({
        file,
        errors: [],
      });
    }
  });

  it('has unique ids that match the file names', () => {
    const files = readdirSync(GOLD_DIR)
      .filter((name) => name.endsWith('.json'))
      .map((name) => name.replace(/\.json$/, ''))
      .sort();

    expect(loadGoldCases(GOLD_DIR).map((each) => each.id)).toEqual(files);
  });

  it('covers corrections, decimals, weather, noise and an empty answer', () => {
    const cases = loadGoldCases(GOLD_DIR);

    expect(cases.some((c) => c.expected.some((e) => e.corrected))).toBe(true);
    expect(cases.some((c) => /\d,\d/.test(c.transcript))).toBe(true);
    expect(cases.some((c) => /Grad/.test(c.transcript))).toBe(true);
    expect(cases.some((c) => c.expected.length === 0)).toBe(true);
    expect(cases.some((c) => c.cannedExtra.length > 0)).toBe(true);
  });
});

describe('parseGoldCase', () => {
  it('accepts a valid case', () => {
    expect(parseGoldCase(VALID).ok).toBe(true);
  });

  it.each([
    ['an unknown key', { ...VALID, extra: 1 }],
    ['a bad id', { ...VALID, id: 'sample' }],
    [
      'a quote that is not in the transcript',
      {
        ...VALID,
        expected: [{ field: 'broodFrames', value: 4, quote: 'Fünf Brutwaben' }],
      },
    ],
    [
      'a value outside the field range',
      {
        ...VALID,
        expected: [
          { field: 'broodFrames', value: 400, quote: 'Vier Brutwaben' },
        ],
      },
    ],
    [
      'an unknown field',
      {
        ...VALID,
        expected: [{ field: 'weather', value: 'x', quote: 'Vier Brutwaben' }],
      },
    ],
    [
      'the same field twice',
      {
        ...VALID,
        expected: [
          { field: 'broodFrames', value: 4, quote: 'Vier Brutwaben' },
          { field: 'broodFrames', value: 4, quote: 'Vier Brutwaben' },
        ],
      },
    ],
    ['a negative dropped count', { ...VALID, expectedDropped: -1 }],
  ])('rejects %s', (_label, raw) => {
    expect(parseGoldCase(raw).ok).toBe(false);
  });
});

describe('offline eval over the gold set', () => {
  const cases = loadGoldCases(GOLD_DIR);

  it('finds every expected suggestion and nothing else, and counts what validation drops', async () => {
    const results = await runEval(cases, offlineModelFor(MODEL), MODEL);

    for (const [index, result] of results.entries()) {
      expect({
        id: result.id,
        outcome: result.outcome,
        unsaid: result.unsaid,
        dropped: result.dropped,
      }).toEqual({
        id: cases[index].id,
        outcome: 'ok',
        unsaid: [],
        dropped: cases[index].expectedDropped,
      });
      expect(
        Object.values(result.perField).every((each) => each.fp + each.fn === 0),
      ).toBe(true);
    }
    const summary = summarize(results);
    expect(summary).toMatchObject({
      cases: 30,
      failedCalls: 0,
      precision: 1,
      recall: 1,
      unsaid: 0,
      failures: [],
    });
    expect(summary.dropped).toBe(
      cases.reduce((sum, each) => sum + each.expectedDropped, 0),
    );
    expect(summary.costUsd).toBeGreaterThan(0);
  });

  it('prints a report with the verdict', async () => {
    const results = await runEval(cases, offlineModelFor(MODEL), MODEL);

    const report = formatReport(results, summarize(results), {
      mode: 'offline',
      model: MODEL,
    });

    expect(report).toContain('RESULT: PASS');
    expect(report).toContain('simulated');
    expect(report).toContain(cases[0].id);
  });
});

describe('scoring', () => {
  const gold: GoldCase = {
    ...(
      parseGoldCase({
        ...VALID,
        transcript: 'Vier Brutwaben, Königin gesehen, Stifte da.',
        expected: [
          { field: 'broodFrames', value: 4, quote: 'Vier Brutwaben' },
          { field: 'queenSeen', value: true, quote: 'Königin gesehen' },
        ],
      }) as { ok: true; value: GoldCase }
    ).value,
  };

  async function score(findings: object[]) {
    const model = new FakeStructuredModel([
      FakeStructuredModel.ok({ findings, unclear: [] }, MODEL),
    ]);
    return runCase(gold, model, MODEL);
  }

  it('counts a wrong value as a miss and as an unsaid adoption', async () => {
    const result = await score([
      {
        field: 'broodFrames',
        value: 4,
        quote: 'Vier Brutwaben',
        corrected: false,
      },
      {
        field: 'queenSeen',
        value: false,
        quote: 'Königin gesehen',
        corrected: false,
      },
    ]);

    expect(result.perField.broodFrames).toEqual({ tp: 1, fp: 0, fn: 0 });
    expect(result.perField.queenSeen).toEqual({ tp: 0, fp: 1, fn: 1 });
    expect(result.unsaid).toEqual(['queenSeen']);
  });

  it('counts a suggestion nobody expected as an unsaid adoption', async () => {
    const result = await score([
      {
        field: 'broodFrames',
        value: 4,
        quote: 'Vier Brutwaben',
        corrected: false,
      },
      {
        field: 'queenSeen',
        value: true,
        quote: 'Königin gesehen',
        corrected: false,
      },
      {
        field: 'eggsPresent',
        value: true,
        quote: 'Stifte da',
        corrected: false,
      },
    ]);

    expect(result.perField.eggsPresent).toEqual({ tp: 0, fp: 1, fn: 0 });
    expect(result.unsaid).toEqual(['eggsPresent']);
  });

  it('scores a refused, failed or malformed answer as all missed', async () => {
    const refused = await runCase(
      gold,
      new FakeStructuredModel([
        {
          kind: 'refused',
          usage: { input: 10, output: 0, cacheRead: 0, cacheWrite: 0 },
          model: MODEL,
        },
      ]),
      MODEL,
    );
    const failed = await runCase(
      gold,
      new FakeStructuredModel([{ kind: 'error', code: 'AUTH' }]),
      MODEL,
    );
    const malformed = await runCase(
      gold,
      new FakeStructuredModel([FakeStructuredModel.ok('no json', MODEL)]),
      MODEL,
    );

    expect(refused.outcome).toBe('refused');
    expect(failed.outcome).toBe('error');
    expect(malformed.outcome).toBe('invalid');
    for (const result of [refused, failed, malformed]) {
      expect(result.perField.broodFrames).toEqual({ tp: 0, fp: 0, fn: 1 });
    }
    expect(summarize([refused]).recall).toBe(0);
  });

  it('fails the verdict below the thresholds', async () => {
    const missed = await score([]);
    const summary = summarize([missed]);

    expect(summary.recall).toBeLessThan(EVAL_THRESHOLDS.recall);
    expect(summary.failures).toHaveLength(1);

    const wrong = summarize([
      await score([
        {
          field: 'broodFrames',
          value: 3,
          quote: 'Vier Brutwaben',
          corrected: false,
        },
      ]),
    ]);
    expect(wrong.precision).toBeLessThan(EVAL_THRESHOLDS.precision);
  });
});

describe('estimateCostUsd', () => {
  const usage = {
    input: 1_000_000,
    output: 1_000_000,
    cacheRead: 0,
    cacheWrite: 0,
  };

  it('prices tokens per model and never guesses an unknown one', () => {
    expect(estimateCostUsd('claude-sonnet-5-5', usage)).toBe(12);
    expect(estimateCostUsd('claude-haiku-5-5', usage)).toBeCloseTo(0.6);
    expect(estimateCostUsd('some-other-model', usage)).toBeNull();
  });
});

describe('eval:voice stays out of CI', () => {
  it('no workflow runs eval:voice', () => {
    const workflows = join(
      __dirname,
      '..',
      '..',
      '..',
      '..',
      '.gitea',
      'workflows',
    );
    // The backend image builds without the repo root: nothing to check there.
    if (!existsSync(workflows)) {
      return;
    }
    const files = readdirSync(workflows).filter((name) =>
      /\.ya?ml$/.test(name),
    );

    expect(files.length).toBeGreaterThan(0);
    for (const file of files) {
      expect({
        file,
        callsEval: /eval[:-]voice/.test(
          readFileSync(join(workflows, file), 'utf8'),
        ),
      }).toEqual({ file, callsEval: false });
    }
  });
});