feat: Gold-Menge und Auswertungs-Werkzeug für die Sprach-Auswertung (T7 L4 Task 16)
30 erfundene Durchsicht-Transkripte mit erwarteten Vorschlägen (Fallen: Korrekturen, Dezimalzahl, Wetter, Füllwörter, Prompt-Injection, sehr lange Notiz, ungültige Funde). npm run eval:voice prüft offline mit FakeStructuredModel (Präzision/Trefferquote je Feld, Schwellen 98/85 %); --live --yes ruft das echte Modell nur ausdrücklich (≈ 0,21 $ mit Sonnet). Preistabelle Backend und Frontend per Test gleich. Refs #27 Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
Nick.RosenloecherandClaude Opus 5.5 committed
1 parent
82b0c746d1
commit
8b63c50f7a
38 files changed
+1756
-1
No files matched your search
@@ -0,0 +1,65 @@
|
||||
import type { EvalCaseResult, EvalSummary } from './voice-eval';
|
||||
import { percent } from './voice-eval';
|
||||
|
||||
export interface ReportContext {
|
||||
mode: 'offline' | 'live';
|
||||
model: string;
|
||||
}
|
||||
|
||||
function usd(value: number | null): string {
|
||||
return value === null ? 'n/a' : `$${value.toFixed(4)}`;
|
||||
}
|
||||
|
||||
function caseLine(result: EvalCaseResult): string {
|
||||
const counts = Object.values(result.perField).reduce(
|
||||
(sum, each) => ({
|
||||
tp: sum.tp + each.tp,
|
||||
fp: sum.fp + each.fp,
|
||||
fn: sum.fn + each.fn,
|
||||
}),
|
||||
{ tp: 0, fp: 0, fn: 0 },
|
||||
);
|
||||
const tokens = result.usage.input + result.usage.output;
|
||||
return [
|
||||
result.id.padEnd(34),
|
||||
result.outcome.padEnd(9),
|
||||
`tp ${counts.tp}`.padEnd(6),
|
||||
`fp ${counts.fp}`.padEnd(6),
|
||||
`fn ${counts.fn}`.padEnd(6),
|
||||
`dropped ${result.dropped}`.padEnd(10),
|
||||
`tokens ${tokens}`.padEnd(13),
|
||||
usd(result.costUsd),
|
||||
].join(' ');
|
||||
}
|
||||
|
||||
/** The printable report: per case, per field, totals and the verdict. */
|
||||
export function formatReport(
|
||||
results: readonly EvalCaseResult[],
|
||||
summary: EvalSummary,
|
||||
context: ReportContext,
|
||||
): string {
|
||||
const simulated =
|
||||
context.mode === 'offline' ? ' (simulated tokens, no real call)' : '';
|
||||
const fieldLines = Object.entries(summary.perField).map(
|
||||
([field, each]) =>
|
||||
` ${field.padEnd(20)} precision ${percent(each.precision).padStart(8)} recall ${percent(each.recall).padStart(8)} (tp ${each.tp}, fp ${each.fp}, fn ${each.fn})`,
|
||||
);
|
||||
return [
|
||||
`eval:voice mode=${context.mode} model=${context.model}`,
|
||||
'',
|
||||
'Cases',
|
||||
...results.map(caseLine),
|
||||
'',
|
||||
'Per field',
|
||||
...fieldLines,
|
||||
'',
|
||||
`Cases: ${summary.cases}, failed calls: ${summary.failedCalls}`,
|
||||
`Precision: ${percent(summary.precision)} Recall: ${percent(summary.recall)}`,
|
||||
`Unsaid adoptions: ${summary.unsaid} Dropped by validation: ${summary.dropped}`,
|
||||
`Tokens: ${summary.tokens.input} in / ${summary.tokens.output} out${simulated}`,
|
||||
`Estimated cost: ${usd(summary.costUsd)}${simulated}`,
|
||||
summary.failures.length === 0
|
||||
? 'RESULT: PASS'
|
||||
: `RESULT: FAIL - ${summary.failures.join('; ')}`,
|
||||
].join('\n');
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
import { readdirSync, readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import {
|
||||
checkFieldValue,
|
||||
type InspectionVoiceField,
|
||||
} from '../core/inspection-field-specs';
|
||||
import { quoteInText } from '../core/voice-text';
|
||||
|
||||
/** One expected finding: what a correct answer contains for the transcript. */
|
||||
export interface GoldFinding {
|
||||
field: InspectionVoiceField;
|
||||
value: number | boolean | string;
|
||||
quote: string;
|
||||
/** The speaker corrected themselves ("vier, nein fünf"). Default false. */
|
||||
corrected?: boolean;
|
||||
}
|
||||
|
||||
/** A canned finding the offline model adds on top, which validation must drop. */
|
||||
export interface GoldExtraFinding {
|
||||
field: string;
|
||||
value: unknown;
|
||||
quote: string;
|
||||
corrected?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* One invented, anonymous German inspection transcript with the suggestions
|
||||
* a correct extraction yields (spec 6.4/6.5). The offline eval replays
|
||||
* `expected` (plus `cannedExtra`, which must be dropped) as the model's
|
||||
* answer, so the harness is tested without any model.
|
||||
*/
|
||||
export interface GoldCase {
|
||||
id: string;
|
||||
title: string;
|
||||
transcript: string;
|
||||
expected: GoldFinding[];
|
||||
/** Notes the model may put in `unclear`; not scored. */
|
||||
unclear: string[];
|
||||
cannedExtra: GoldExtraFinding[];
|
||||
/** How many findings validation throws away with `cannedExtra` added. */
|
||||
expectedDropped: number;
|
||||
}
|
||||
|
||||
export type GoldParse =
|
||||
{ ok: true; value: GoldCase } | { ok: false; errors: string[] };
|
||||
|
||||
const ID_PATTERN = /^g\d{2}-[a-z0-9-]+$/;
|
||||
const CASE_KEYS = [
|
||||
'id',
|
||||
'title',
|
||||
'transcript',
|
||||
'expected',
|
||||
'unclear',
|
||||
'cannedExtra',
|
||||
'expectedDropped',
|
||||
];
|
||||
const FINDING_KEYS = ['field', 'value', 'quote', 'corrected'];
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function isNonEmptyString(value: unknown): value is string {
|
||||
return typeof value === 'string' && value.trim() !== '';
|
||||
}
|
||||
|
||||
function unknownKeys(
|
||||
record: Record<string, unknown>,
|
||||
allowed: readonly string[],
|
||||
where: string,
|
||||
): string[] {
|
||||
return Object.keys(record)
|
||||
.filter((key) => !allowed.includes(key))
|
||||
.map((key) => `${where}: unknown key "${key}"`);
|
||||
}
|
||||
|
||||
function expectedFindings(
|
||||
raw: unknown,
|
||||
transcript: string,
|
||||
): { errors: string[]; findings: GoldFinding[] } {
|
||||
if (!Array.isArray(raw)) {
|
||||
return { errors: ['expected must be an array'], findings: [] };
|
||||
}
|
||||
const errors: string[] = [];
|
||||
const findings: GoldFinding[] = [];
|
||||
const seen = new Set<string>();
|
||||
raw.forEach((each: unknown, index) => {
|
||||
const where = `expected[${index}]`;
|
||||
if (!isRecord(each)) {
|
||||
errors.push(`${where}: must be an object`);
|
||||
return;
|
||||
}
|
||||
errors.push(...unknownKeys(each, FINDING_KEYS, where));
|
||||
const { field, value, quote, corrected } = each;
|
||||
const checked =
|
||||
typeof field === 'string' ? checkFieldValue(field, value) : null;
|
||||
if (!checked?.ok) {
|
||||
errors.push(`${where}: field/value is not valid for the field specs`);
|
||||
return;
|
||||
}
|
||||
if (!isNonEmptyString(quote) || !quoteInText(quote, transcript)) {
|
||||
errors.push(`${where}: quote does not stand in the transcript`);
|
||||
return;
|
||||
}
|
||||
if (corrected !== undefined && typeof corrected !== 'boolean') {
|
||||
errors.push(`${where}: corrected must be a boolean`);
|
||||
return;
|
||||
}
|
||||
if (seen.has(checked.field)) {
|
||||
errors.push(`${where}: field ${checked.field} appears twice`);
|
||||
return;
|
||||
}
|
||||
seen.add(checked.field);
|
||||
findings.push({
|
||||
field: checked.field,
|
||||
value: checked.value,
|
||||
quote,
|
||||
...(corrected === undefined ? {} : { corrected }),
|
||||
});
|
||||
});
|
||||
return { errors, findings };
|
||||
}
|
||||
|
||||
function extraFindings(raw: unknown): {
|
||||
errors: string[];
|
||||
extras: GoldExtraFinding[];
|
||||
} {
|
||||
if (!Array.isArray(raw)) {
|
||||
return { errors: ['cannedExtra must be an array'], extras: [] };
|
||||
}
|
||||
const errors: string[] = [];
|
||||
const extras: GoldExtraFinding[] = [];
|
||||
raw.forEach((each: unknown, index) => {
|
||||
const where = `cannedExtra[${index}]`;
|
||||
if (
|
||||
!isRecord(each) ||
|
||||
typeof each.field !== 'string' ||
|
||||
!isNonEmptyString(each.quote) ||
|
||||
(each.corrected !== undefined && typeof each.corrected !== 'boolean')
|
||||
) {
|
||||
errors.push(
|
||||
`${where}: needs a string field, a quote and an optional boolean corrected`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
errors.push(...unknownKeys(each, FINDING_KEYS, where));
|
||||
extras.push({
|
||||
field: each.field,
|
||||
value: each.value,
|
||||
quote: each.quote,
|
||||
corrected: each.corrected,
|
||||
});
|
||||
});
|
||||
return { errors, extras };
|
||||
}
|
||||
|
||||
/** The schema every gold file must match; returns every problem found. */
|
||||
export function parseGoldCase(raw: unknown): GoldParse {
|
||||
if (!isRecord(raw)) {
|
||||
return { ok: false, errors: ['gold case must be an object'] };
|
||||
}
|
||||
const errors = unknownKeys(raw, CASE_KEYS, 'case');
|
||||
const { id, title, transcript, unclear, expectedDropped } = raw;
|
||||
if (typeof id !== 'string' || !ID_PATTERN.test(id)) {
|
||||
errors.push('id must look like g01-some-slug');
|
||||
}
|
||||
if (!isNonEmptyString(title)) {
|
||||
errors.push('title must be a non-empty string');
|
||||
}
|
||||
if (!isNonEmptyString(transcript)) {
|
||||
errors.push('transcript must be a non-empty string');
|
||||
}
|
||||
if (
|
||||
!Array.isArray(unclear) ||
|
||||
!unclear.every((each) => typeof each === 'string')
|
||||
) {
|
||||
errors.push('unclear must be an array of strings');
|
||||
}
|
||||
if (!Number.isInteger(expectedDropped) || (expectedDropped as number) < 0) {
|
||||
errors.push('expectedDropped must be a non-negative integer');
|
||||
}
|
||||
const text = typeof transcript === 'string' ? transcript : '';
|
||||
const expected = expectedFindings(raw.expected, text);
|
||||
const extra = extraFindings(raw.cannedExtra);
|
||||
errors.push(...expected.errors, ...extra.errors);
|
||||
if (errors.length > 0) {
|
||||
return { ok: false, errors };
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
value: {
|
||||
id: id as string,
|
||||
title: title as string,
|
||||
transcript: text,
|
||||
expected: expected.findings,
|
||||
unclear: unclear as string[],
|
||||
cannedExtra: extra.extras,
|
||||
expectedDropped: expectedDropped as number,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** All gold files of a directory, sorted by name; throws naming each bad file. */
|
||||
export function loadGoldCases(directory: string): GoldCase[] {
|
||||
const files = readdirSync(directory)
|
||||
.filter((name) => name.endsWith('.json'))
|
||||
.sort();
|
||||
const cases: GoldCase[] = [];
|
||||
const problems: string[] = [];
|
||||
for (const file of files) {
|
||||
const parsed = parseGoldCase(
|
||||
JSON.parse(readFileSync(join(directory, file), 'utf8')) as unknown,
|
||||
);
|
||||
if (parsed.ok) {
|
||||
cases.push(parsed.value);
|
||||
} else {
|
||||
problems.push(...parsed.errors.map((error) => `${file}: ${error}`));
|
||||
}
|
||||
}
|
||||
if (problems.length > 0) {
|
||||
throw new Error(`Invalid gold files:\n${problems.join('\n')}`);
|
||||
}
|
||||
return cases;
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
import { readFileSync, readdirSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { formatReport } from './eval-report';
|
||||
import { loadGoldCases, parseGoldCase, type GoldCase } from './gold-case';
|
||||
import {
|
||||
EVAL_THRESHOLDS,
|
||||
offlineModelFor,
|
||||
runCase,
|
||||
runEval,
|
||||
summarize,
|
||||
} from './voice-eval';
|
||||
import { estimateCostUsd } from './voice-prices';
|
||||
import { FakeStructuredModel } from '../extraction/fake-structured-model';
|
||||
|
||||
const GOLD_DIR = join(
|
||||
__dirname,
|
||||
'..',
|
||||
'..',
|
||||
'..',
|
||||
'test',
|
||||
'fixtures',
|
||||
'voice',
|
||||
'gold',
|
||||
);
|
||||
const MODEL = 'claude-sonnet-5-5';
|
||||
|
||||
const VALID = {
|
||||
id: 'g99-sample',
|
||||
title: 'Sample',
|
||||
transcript: 'Vier Brutwaben und Königin gesehen.',
|
||||
expected: [{ field: 'broodFrames', value: 4, quote: 'Vier Brutwaben' }],
|
||||
unclear: [],
|
||||
cannedExtra: [],
|
||||
expectedDropped: 0,
|
||||
};
|
||||
|
||||
describe('gold set', () => {
|
||||
it('holds 30 files and every one matches the schema', () => {
|
||||
const files = readdirSync(GOLD_DIR).filter((name) =>
|
||||
name.endsWith('.json'),
|
||||
);
|
||||
|
||||
expect(files).toHaveLength(30);
|
||||
for (const file of files) {
|
||||
const parsed = parseGoldCase(
|
||||
JSON.parse(readFileSync(join(GOLD_DIR, file), 'utf8')) as unknown,
|
||||
);
|
||||
expect({ file, errors: parsed.ok ? [] : parsed.errors }).toEqual({
|
||||
file,
|
||||
errors: [],
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it('has unique ids that match the file names', () => {
|
||||
const files = readdirSync(GOLD_DIR)
|
||||
.filter((name) => name.endsWith('.json'))
|
||||
.map((name) => name.replace(/\.json$/, ''))
|
||||
.sort();
|
||||
|
||||
expect(loadGoldCases(GOLD_DIR).map((each) => each.id)).toEqual(files);
|
||||
});
|
||||
|
||||
it('covers corrections, decimals, weather, noise and an empty answer', () => {
|
||||
const cases = loadGoldCases(GOLD_DIR);
|
||||
|
||||
expect(cases.some((c) => c.expected.some((e) => e.corrected))).toBe(true);
|
||||
expect(cases.some((c) => /\d,\d/.test(c.transcript))).toBe(true);
|
||||
expect(cases.some((c) => /Grad/.test(c.transcript))).toBe(true);
|
||||
expect(cases.some((c) => c.expected.length === 0)).toBe(true);
|
||||
expect(cases.some((c) => c.cannedExtra.length > 0)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseGoldCase', () => {
|
||||
it('accepts a valid case', () => {
|
||||
expect(parseGoldCase(VALID).ok).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['an unknown key', { ...VALID, extra: 1 }],
|
||||
['a bad id', { ...VALID, id: 'sample' }],
|
||||
[
|
||||
'a quote that is not in the transcript',
|
||||
{
|
||||
...VALID,
|
||||
expected: [{ field: 'broodFrames', value: 4, quote: 'Fünf Brutwaben' }],
|
||||
},
|
||||
],
|
||||
[
|
||||
'a value outside the field range',
|
||||
{
|
||||
...VALID,
|
||||
expected: [
|
||||
{ field: 'broodFrames', value: 400, quote: 'Vier Brutwaben' },
|
||||
],
|
||||
},
|
||||
],
|
||||
[
|
||||
'an unknown field',
|
||||
{
|
||||
...VALID,
|
||||
expected: [{ field: 'weather', value: 'x', quote: 'Vier Brutwaben' }],
|
||||
},
|
||||
],
|
||||
[
|
||||
'the same field twice',
|
||||
{
|
||||
...VALID,
|
||||
expected: [
|
||||
{ field: 'broodFrames', value: 4, quote: 'Vier Brutwaben' },
|
||||
{ field: 'broodFrames', value: 4, quote: 'Vier Brutwaben' },
|
||||
],
|
||||
},
|
||||
],
|
||||
['a negative dropped count', { ...VALID, expectedDropped: -1 }],
|
||||
])('rejects %s', (_label, raw) => {
|
||||
expect(parseGoldCase(raw).ok).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('offline eval over the gold set', () => {
|
||||
const cases = loadGoldCases(GOLD_DIR);
|
||||
|
||||
it('finds every expected suggestion and nothing else, and counts what validation drops', async () => {
|
||||
const results = await runEval(cases, offlineModelFor(MODEL), MODEL);
|
||||
|
||||
for (const [index, result] of results.entries()) {
|
||||
expect({
|
||||
id: result.id,
|
||||
outcome: result.outcome,
|
||||
unsaid: result.unsaid,
|
||||
dropped: result.dropped,
|
||||
}).toEqual({
|
||||
id: cases[index].id,
|
||||
outcome: 'ok',
|
||||
unsaid: [],
|
||||
dropped: cases[index].expectedDropped,
|
||||
});
|
||||
expect(
|
||||
Object.values(result.perField).every((each) => each.fp + each.fn === 0),
|
||||
).toBe(true);
|
||||
}
|
||||
const summary = summarize(results);
|
||||
expect(summary).toMatchObject({
|
||||
cases: 30,
|
||||
failedCalls: 0,
|
||||
precision: 1,
|
||||
recall: 1,
|
||||
unsaid: 0,
|
||||
failures: [],
|
||||
});
|
||||
expect(summary.dropped).toBe(
|
||||
cases.reduce((sum, each) => sum + each.expectedDropped, 0),
|
||||
);
|
||||
expect(summary.costUsd).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('prints a report with the verdict', async () => {
|
||||
const results = await runEval(cases, offlineModelFor(MODEL), MODEL);
|
||||
|
||||
const report = formatReport(results, summarize(results), {
|
||||
mode: 'offline',
|
||||
model: MODEL,
|
||||
});
|
||||
|
||||
expect(report).toContain('RESULT: PASS');
|
||||
expect(report).toContain('simulated');
|
||||
expect(report).toContain(cases[0].id);
|
||||
});
|
||||
});
|
||||
|
||||
describe('scoring', () => {
|
||||
const gold: GoldCase = {
|
||||
...(
|
||||
parseGoldCase({
|
||||
...VALID,
|
||||
transcript: 'Vier Brutwaben, Königin gesehen, Stifte da.',
|
||||
expected: [
|
||||
{ field: 'broodFrames', value: 4, quote: 'Vier Brutwaben' },
|
||||
{ field: 'queenSeen', value: true, quote: 'Königin gesehen' },
|
||||
],
|
||||
}) as { ok: true; value: GoldCase }
|
||||
).value,
|
||||
};
|
||||
|
||||
async function score(findings: object[]) {
|
||||
const model = new FakeStructuredModel([
|
||||
FakeStructuredModel.ok({ findings, unclear: [] }, MODEL),
|
||||
]);
|
||||
return runCase(gold, model, MODEL);
|
||||
}
|
||||
|
||||
it('counts a wrong value as a miss and as an unsaid adoption', async () => {
|
||||
const result = await score([
|
||||
{
|
||||
field: 'broodFrames',
|
||||
value: 4,
|
||||
quote: 'Vier Brutwaben',
|
||||
corrected: false,
|
||||
},
|
||||
{
|
||||
field: 'queenSeen',
|
||||
value: false,
|
||||
quote: 'Königin gesehen',
|
||||
corrected: false,
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result.perField.broodFrames).toEqual({ tp: 1, fp: 0, fn: 0 });
|
||||
expect(result.perField.queenSeen).toEqual({ tp: 0, fp: 1, fn: 1 });
|
||||
expect(result.unsaid).toEqual(['queenSeen']);
|
||||
});
|
||||
|
||||
it('counts a suggestion nobody expected as an unsaid adoption', async () => {
|
||||
const result = await score([
|
||||
{
|
||||
field: 'broodFrames',
|
||||
value: 4,
|
||||
quote: 'Vier Brutwaben',
|
||||
corrected: false,
|
||||
},
|
||||
{
|
||||
field: 'queenSeen',
|
||||
value: true,
|
||||
quote: 'Königin gesehen',
|
||||
corrected: false,
|
||||
},
|
||||
{
|
||||
field: 'eggsPresent',
|
||||
value: true,
|
||||
quote: 'Stifte da',
|
||||
corrected: false,
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result.perField.eggsPresent).toEqual({ tp: 0, fp: 1, fn: 0 });
|
||||
expect(result.unsaid).toEqual(['eggsPresent']);
|
||||
});
|
||||
|
||||
it('scores a refused, failed or malformed answer as all missed', async () => {
|
||||
const refused = await runCase(
|
||||
gold,
|
||||
new FakeStructuredModel([
|
||||
{
|
||||
kind: 'refused',
|
||||
usage: { input: 10, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
model: MODEL,
|
||||
},
|
||||
]),
|
||||
MODEL,
|
||||
);
|
||||
const failed = await runCase(
|
||||
gold,
|
||||
new FakeStructuredModel([{ kind: 'error', code: 'AUTH' }]),
|
||||
MODEL,
|
||||
);
|
||||
const malformed = await runCase(
|
||||
gold,
|
||||
new FakeStructuredModel([FakeStructuredModel.ok('no json', MODEL)]),
|
||||
MODEL,
|
||||
);
|
||||
|
||||
expect(refused.outcome).toBe('refused');
|
||||
expect(failed.outcome).toBe('error');
|
||||
expect(malformed.outcome).toBe('invalid');
|
||||
for (const result of [refused, failed, malformed]) {
|
||||
expect(result.perField.broodFrames).toEqual({ tp: 0, fp: 0, fn: 1 });
|
||||
}
|
||||
expect(summarize([refused]).recall).toBe(0);
|
||||
});
|
||||
|
||||
it('fails the verdict below the thresholds', async () => {
|
||||
const missed = await score([]);
|
||||
const summary = summarize([missed]);
|
||||
|
||||
expect(summary.recall).toBeLessThan(EVAL_THRESHOLDS.recall);
|
||||
expect(summary.failures).toHaveLength(1);
|
||||
|
||||
const wrong = summarize([
|
||||
await score([
|
||||
{
|
||||
field: 'broodFrames',
|
||||
value: 3,
|
||||
quote: 'Vier Brutwaben',
|
||||
corrected: false,
|
||||
},
|
||||
]),
|
||||
]);
|
||||
expect(wrong.precision).toBeLessThan(EVAL_THRESHOLDS.precision);
|
||||
});
|
||||
});
|
||||
|
||||
describe('estimateCostUsd', () => {
|
||||
const usage = {
|
||||
input: 1_000_000,
|
||||
output: 1_000_000,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
};
|
||||
|
||||
it('prices tokens per model and never guesses an unknown one', () => {
|
||||
expect(estimateCostUsd('claude-sonnet-5-5', usage)).toBe(12);
|
||||
expect(estimateCostUsd('claude-haiku-5-5', usage)).toBeCloseTo(0.6);
|
||||
expect(estimateCostUsd('some-other-model', usage)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('eval:voice stays out of CI', () => {
|
||||
it('no workflow runs eval:voice', () => {
|
||||
const workflows = join(
|
||||
__dirname,
|
||||
'..',
|
||||
'..',
|
||||
'..',
|
||||
'..',
|
||||
'.gitea',
|
||||
'workflows',
|
||||
);
|
||||
const files = readdirSync(workflows).filter((name) =>
|
||||
/\.ya?ml$/.test(name),
|
||||
);
|
||||
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
for (const file of files) {
|
||||
expect({
|
||||
file,
|
||||
callsEval: /eval[:-]voice/.test(
|
||||
readFileSync(join(workflows, file), 'utf8'),
|
||||
),
|
||||
}).toEqual({ file, callsEval: false });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,305 @@
|
||||
import { FIELD_SPECS } from '../core/inspection-field-specs';
|
||||
import {
|
||||
VOICE_EXTRACT_MAX_RETRIES,
|
||||
VOICE_EXTRACT_MAX_TOKENS,
|
||||
VOICE_EXTRACT_TIMEOUT_MS,
|
||||
} from '../extraction/voice-extraction.service';
|
||||
import { FakeStructuredModel } from '../extraction/fake-structured-model';
|
||||
import type {
|
||||
StructuredModel,
|
||||
StructuredOutcome,
|
||||
TokenUsage,
|
||||
} from '../extraction/structured-model.port';
|
||||
import { buildSystemBlocks, buildUserText } from '../extraction/voice-prompt';
|
||||
import { SUGGESTION_SCHEMA } from '../extraction/voice-schema';
|
||||
import { validateSuggestions } from '../extraction/validate-suggestions';
|
||||
import type { GoldCase } from './gold-case';
|
||||
import { estimateCostUsd } from './voice-prices';
|
||||
|
||||
/**
|
||||
* Exit-code thresholds of `npm run eval:voice`, aggregated over all cases.
|
||||
* Precision stands for "nothing unsaid is adopted" (plan target: no unsaid
|
||||
* adoption, boolean accuracy >= 98 %); recall is the hit rate (>= 85 %).
|
||||
*/
|
||||
export const EVAL_THRESHOLDS = { precision: 0.98, recall: 0.85 } as const;
|
||||
|
||||
export type EvalCaseOutcome =
|
||||
'ok' | 'invalid' | 'refused' | 'truncated' | 'error';
|
||||
|
||||
export interface FieldCounts {
|
||||
tp: number;
|
||||
fp: number;
|
||||
fn: number;
|
||||
}
|
||||
|
||||
export interface EvalCaseResult {
|
||||
id: string;
|
||||
outcome: EvalCaseOutcome;
|
||||
perField: Readonly<Record<string, FieldCounts>>;
|
||||
/** Fields suggested although the gold set did not expect them (or with another value). */
|
||||
unsaid: string[];
|
||||
dropped: number;
|
||||
usage: TokenUsage;
|
||||
costUsd: number | null;
|
||||
}
|
||||
|
||||
const NO_USAGE: TokenUsage = {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
};
|
||||
|
||||
/** The answer the offline model replays for a case: the expected findings. */
|
||||
export function cannedResponse(gold: GoldCase): object {
|
||||
return {
|
||||
findings: [
|
||||
...gold.expected.map((each) => ({
|
||||
field: each.field,
|
||||
value: each.value,
|
||||
quote: each.quote,
|
||||
corrected: each.corrected ?? false,
|
||||
})),
|
||||
...gold.cannedExtra.map((each) => ({
|
||||
field: each.field,
|
||||
value: each.value,
|
||||
quote: each.quote,
|
||||
corrected: each.corrected ?? false,
|
||||
})),
|
||||
],
|
||||
unclear: gold.unclear,
|
||||
};
|
||||
}
|
||||
|
||||
/** Offline adapter: one FakeStructuredModel per case, replaying cannedResponse. */
|
||||
export function offlineModelFor(
|
||||
modelId: string,
|
||||
): (gold: GoldCase) => StructuredModel {
|
||||
return (gold) =>
|
||||
new FakeStructuredModel([
|
||||
FakeStructuredModel.ok(cannedResponse(gold), modelId),
|
||||
]);
|
||||
}
|
||||
|
||||
function usageOf(outcome: StructuredOutcome): TokenUsage {
|
||||
return outcome.kind === 'error' ? NO_USAGE : outcome.usage;
|
||||
}
|
||||
|
||||
function outcomeLabel(outcome: StructuredOutcome): EvalCaseOutcome {
|
||||
return outcome.kind === 'ok' ? 'ok' : outcome.kind;
|
||||
}
|
||||
|
||||
/**
|
||||
* Text fields are free wording, so they match by presence; all other fields
|
||||
* (numbers, booleans, feed codes) must carry the expected value.
|
||||
*/
|
||||
function sameValue(
|
||||
field: string,
|
||||
expected: unknown,
|
||||
produced: unknown,
|
||||
): boolean {
|
||||
const spec = FIELD_SPECS[field as keyof typeof FIELD_SPECS];
|
||||
return spec.kind === 'text' || expected === produced;
|
||||
}
|
||||
|
||||
function scoreFields(
|
||||
gold: GoldCase,
|
||||
produced: ReadonlyMap<string, unknown>,
|
||||
): { perField: Record<string, FieldCounts>; unsaid: string[] } {
|
||||
const expected = new Map<string, unknown>(
|
||||
gold.expected.map((each) => [each.field, each.value]),
|
||||
);
|
||||
const perField: Record<string, FieldCounts> = {};
|
||||
const unsaid: string[] = [];
|
||||
for (const field of new Set([...expected.keys(), ...produced.keys()])) {
|
||||
const counts: FieldCounts = { tp: 0, fp: 0, fn: 0 };
|
||||
const wanted = expected.has(field);
|
||||
const given = produced.has(field);
|
||||
if (
|
||||
wanted &&
|
||||
given &&
|
||||
sameValue(field, expected.get(field), produced.get(field))
|
||||
) {
|
||||
counts.tp = 1;
|
||||
} else {
|
||||
counts.fp = Number(given);
|
||||
counts.fn = Number(wanted);
|
||||
if (given) {
|
||||
unsaid.push(field);
|
||||
}
|
||||
}
|
||||
perField[field] = counts;
|
||||
}
|
||||
return { perField, unsaid };
|
||||
}
|
||||
|
||||
function missesEverything(gold: GoldCase): Record<string, FieldCounts> {
|
||||
return Object.fromEntries(
|
||||
gold.expected.map((each) => [each.field, { tp: 0, fp: 0, fn: 1 }]),
|
||||
);
|
||||
}
|
||||
|
||||
/** Runs one gold case through prompt, model and server-side validation. */
|
||||
export async function runCase(
|
||||
gold: GoldCase,
|
||||
model: StructuredModel,
|
||||
modelId: string,
|
||||
): Promise<EvalCaseResult> {
|
||||
const outcome = await model.extract({
|
||||
model: modelId,
|
||||
system: buildSystemBlocks(),
|
||||
userText: buildUserText(gold.transcript),
|
||||
schema: SUGGESTION_SCHEMA,
|
||||
maxTokens: VOICE_EXTRACT_MAX_TOKENS,
|
||||
timeoutMs: VOICE_EXTRACT_TIMEOUT_MS,
|
||||
maxRetries: VOICE_EXTRACT_MAX_RETRIES,
|
||||
effort: 'low',
|
||||
});
|
||||
const usage = usageOf(outcome);
|
||||
const costUsd = estimateCostUsd(modelId, usage);
|
||||
const checked =
|
||||
outcome.kind === 'ok'
|
||||
? validateSuggestions(outcome.data, gold.transcript)
|
||||
: null;
|
||||
if (checked === null) {
|
||||
return {
|
||||
id: gold.id,
|
||||
outcome: outcome.kind === 'ok' ? 'invalid' : outcomeLabel(outcome),
|
||||
perField: missesEverything(gold),
|
||||
unsaid: [],
|
||||
dropped: 0,
|
||||
usage,
|
||||
costUsd,
|
||||
};
|
||||
}
|
||||
const produced = new Map<string, unknown>(
|
||||
checked.suggestions.map((each) => [each.field, each.value]),
|
||||
);
|
||||
return {
|
||||
id: gold.id,
|
||||
outcome: 'ok',
|
||||
...scoreFields(gold, produced),
|
||||
dropped: checked.dropped,
|
||||
usage,
|
||||
costUsd,
|
||||
};
|
||||
}
|
||||
|
||||
export interface EvalSummary {
|
||||
cases: number;
|
||||
failedCalls: number;
|
||||
precision: number;
|
||||
recall: number;
|
||||
unsaid: number;
|
||||
dropped: number;
|
||||
tokens: { input: number; output: number };
|
||||
/** Null when the model has no price in the table. */
|
||||
costUsd: number | null;
|
||||
perField: Readonly<
|
||||
Record<string, FieldCounts & { precision: number; recall: number }>
|
||||
>;
|
||||
/** Human-readable reasons the thresholds are missed; empty = pass. */
|
||||
failures: string[];
|
||||
}
|
||||
|
||||
function ratio(hit: number, total: number): number {
|
||||
return total === 0 ? 1 : hit / total;
|
||||
}
|
||||
|
||||
function sumCounts(parts: readonly FieldCounts[]): FieldCounts {
|
||||
return parts.reduce(
|
||||
(sum, each) => ({
|
||||
tp: sum.tp + each.tp,
|
||||
fp: sum.fp + each.fp,
|
||||
fn: sum.fn + each.fn,
|
||||
}),
|
||||
{ tp: 0, fp: 0, fn: 0 },
|
||||
);
|
||||
}
|
||||
|
||||
function fieldTotals(
|
||||
results: readonly EvalCaseResult[],
|
||||
): EvalSummary['perField'] {
|
||||
const byField = new Map<string, FieldCounts[]>();
|
||||
for (const result of results) {
|
||||
for (const [field, counts] of Object.entries(result.perField)) {
|
||||
byField.set(field, [...(byField.get(field) ?? []), counts]);
|
||||
}
|
||||
}
|
||||
return Object.fromEntries(
|
||||
[...byField.entries()].sort().map(([field, parts]) => {
|
||||
const total = sumCounts(parts);
|
||||
return [
|
||||
field,
|
||||
{
|
||||
...total,
|
||||
precision: ratio(total.tp, total.tp + total.fp),
|
||||
recall: ratio(total.tp, total.tp + total.fn),
|
||||
},
|
||||
];
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function totalCost(results: readonly EvalCaseResult[]): number | null {
|
||||
const costs = results.map((each) => each.costUsd);
|
||||
return costs.every((each) => each !== null)
|
||||
? costs.reduce<number>((sum, each) => sum + each, 0)
|
||||
: null;
|
||||
}
|
||||
|
||||
export function summarize(results: readonly EvalCaseResult[]): EvalSummary {
|
||||
const total = sumCounts(
|
||||
results.flatMap((each) => Object.values(each.perField)),
|
||||
);
|
||||
const precision = ratio(total.tp, total.tp + total.fp);
|
||||
const recall = ratio(total.tp, total.tp + total.fn);
|
||||
const failedCalls = results.filter((each) => each.outcome !== 'ok').length;
|
||||
const failures: string[] = [];
|
||||
if (precision < EVAL_THRESHOLDS.precision) {
|
||||
failures.push(
|
||||
`precision ${percent(precision)} is below ${percent(EVAL_THRESHOLDS.precision)}`,
|
||||
);
|
||||
}
|
||||
if (recall < EVAL_THRESHOLDS.recall) {
|
||||
failures.push(
|
||||
`recall ${percent(recall)} is below ${percent(EVAL_THRESHOLDS.recall)}`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
cases: results.length,
|
||||
failedCalls,
|
||||
precision,
|
||||
recall,
|
||||
unsaid: results.reduce((sum, each) => sum + each.unsaid.length, 0),
|
||||
dropped: results.reduce((sum, each) => sum + each.dropped, 0),
|
||||
tokens: {
|
||||
input: results.reduce(
|
||||
(sum, each) =>
|
||||
sum + each.usage.input + each.usage.cacheRead + each.usage.cacheWrite,
|
||||
0,
|
||||
),
|
||||
output: results.reduce((sum, each) => sum + each.usage.output, 0),
|
||||
},
|
||||
costUsd: totalCost(results),
|
||||
perField: fieldTotals(results),
|
||||
failures,
|
||||
};
|
||||
}
|
||||
|
||||
export function percent(value: number): string {
|
||||
return `${(value * 100).toFixed(1)} %`;
|
||||
}
|
||||
|
||||
/** Runs every case; `modelFor` gives the adapter per case (offline: replay; live: one shared). */
|
||||
export async function runEval(
|
||||
cases: readonly GoldCase[],
|
||||
modelFor: (gold: GoldCase) => StructuredModel,
|
||||
modelId: string,
|
||||
): Promise<EvalCaseResult[]> {
|
||||
const results: EvalCaseResult[] = [];
|
||||
for (const gold of cases) {
|
||||
results.push(await runCase(gold, modelFor(gold), modelId));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { existsSync, readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { VOICE_MODEL_PRICES_USD } from './voice-prices';
|
||||
|
||||
// The frontend cost card prices the same models; both tables must agree.
|
||||
const FRONTEND_PRICES = join(
|
||||
__dirname,
|
||||
'..',
|
||||
'..',
|
||||
'..',
|
||||
'..',
|
||||
'ApiCore-Frontend',
|
||||
'src',
|
||||
'lib',
|
||||
'voice',
|
||||
'prices.ts',
|
||||
);
|
||||
|
||||
describe('voice model prices', () => {
|
||||
// The backend image builds without the frontend folder: nothing to compare.
|
||||
it('equal the frontend price table', () => {
|
||||
if (!existsSync(FRONTEND_PRICES)) {
|
||||
return;
|
||||
}
|
||||
const source = readFileSync(FRONTEND_PRICES, 'utf8');
|
||||
const frontend = Object.fromEntries(
|
||||
[
|
||||
...source.matchAll(
|
||||
/"([a-z0-9-]+)":\s*\{\s*inputPerMillion:\s*([\d.]+),\s*outputPerMillion:\s*([\d.]+)\s*\}/g,
|
||||
),
|
||||
].map((m) => [
|
||||
m[1],
|
||||
{ inputPerMillion: Number(m[2]), outputPerMillion: Number(m[3]) },
|
||||
]),
|
||||
);
|
||||
|
||||
expect(Object.keys(frontend).length).toBeGreaterThan(0);
|
||||
expect(frontend).toEqual(VOICE_MODEL_PRICES_USD);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { TokenUsage } from '../extraction/structured-model.port';
|
||||
|
||||
/**
|
||||
* USD per million tokens for the models of VOICE_MODELS (spec 7.1, Anthropic
|
||||
* model list as of 2026-10-06). Same table as
|
||||
* ApiCore-Frontend/src/lib/voice/prices.ts: change both together. A model
|
||||
* without a price gives no estimate, never a guess.
|
||||
*/
|
||||
export const VOICE_MODEL_PRICES_USD: Readonly<
|
||||
Record<string, { inputPerMillion: number; outputPerMillion: number }>
|
||||
> = {
|
||||
'claude-opus-5-5': { inputPerMillion: 4, outputPerMillion: 20 },
|
||||
'claude-sonnet-5-5': { inputPerMillion: 2, outputPerMillion: 10 },
|
||||
'claude-haiku-5-5': { inputPerMillion: 0.1, outputPerMillion: 0.5 },
|
||||
};
|
||||
|
||||
const TOKENS_PER_MILLION = 1_000_000;
|
||||
|
||||
/** Cache reads and writes are priced like plain input (as the call ledger counts them). */
|
||||
export function estimateCostUsd(
|
||||
model: string,
|
||||
usage: TokenUsage,
|
||||
): number | null {
|
||||
const price = VOICE_MODEL_PRICES_USD[model];
|
||||
if (!price) {
|
||||
return null;
|
||||
}
|
||||
const input = usage.input + usage.cacheRead + usage.cacheWrite;
|
||||
return (
|
||||
(input * price.inputPerMillion + usage.output * price.outputPerMillion) /
|
||||
TOKENS_PER_MILLION
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user