feat: Gold-Menge und Auswertungs-Werkzeug für die Sprach-Auswertung (T7 L4 Task 16)
30 erfundene Durchsicht-Transkripte mit erwarteten Vorschlägen (Fallen: Korrekturen, Dezimalzahl, Wetter, Füllwörter, Prompt-Injection, sehr lange Notiz, ungültige Funde). npm run eval:voice prüft offline mit FakeStructuredModel (Präzision/Trefferquote je Feld, Schwellen 98/85 %); --live --yes ruft das echte Modell nur ausdrücklich (≈ 0,21 $ mit Sonnet). Preistabelle Backend und Frontend per Test gleich. Refs #27 Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
Nick.RosenloecherandClaude Opus 5.5 committed
1 parent
82b0c746d1
commit
8b63c50f7a
38 files changed
+1756
-1
No files matched your search
@@ -0,0 +1,334 @@
|
||||
import { readFileSync, readdirSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { formatReport } from './eval-report';
|
||||
import { loadGoldCases, parseGoldCase, type GoldCase } from './gold-case';
|
||||
import {
|
||||
EVAL_THRESHOLDS,
|
||||
offlineModelFor,
|
||||
runCase,
|
||||
runEval,
|
||||
summarize,
|
||||
} from './voice-eval';
|
||||
import { estimateCostUsd } from './voice-prices';
|
||||
import { FakeStructuredModel } from '../extraction/fake-structured-model';
|
||||
|
||||
const GOLD_DIR = join(
|
||||
__dirname,
|
||||
'..',
|
||||
'..',
|
||||
'..',
|
||||
'test',
|
||||
'fixtures',
|
||||
'voice',
|
||||
'gold',
|
||||
);
|
||||
const MODEL = 'claude-sonnet-5-5';
|
||||
|
||||
const VALID = {
|
||||
id: 'g99-sample',
|
||||
title: 'Sample',
|
||||
transcript: 'Vier Brutwaben und Königin gesehen.',
|
||||
expected: [{ field: 'broodFrames', value: 4, quote: 'Vier Brutwaben' }],
|
||||
unclear: [],
|
||||
cannedExtra: [],
|
||||
expectedDropped: 0,
|
||||
};
|
||||
|
||||
describe('gold set', () => {
|
||||
it('holds 30 files and every one matches the schema', () => {
|
||||
const files = readdirSync(GOLD_DIR).filter((name) =>
|
||||
name.endsWith('.json'),
|
||||
);
|
||||
|
||||
expect(files).toHaveLength(30);
|
||||
for (const file of files) {
|
||||
const parsed = parseGoldCase(
|
||||
JSON.parse(readFileSync(join(GOLD_DIR, file), 'utf8')) as unknown,
|
||||
);
|
||||
expect({ file, errors: parsed.ok ? [] : parsed.errors }).toEqual({
|
||||
file,
|
||||
errors: [],
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it('has unique ids that match the file names', () => {
|
||||
const files = readdirSync(GOLD_DIR)
|
||||
.filter((name) => name.endsWith('.json'))
|
||||
.map((name) => name.replace(/\.json$/, ''))
|
||||
.sort();
|
||||
|
||||
expect(loadGoldCases(GOLD_DIR).map((each) => each.id)).toEqual(files);
|
||||
});
|
||||
|
||||
it('covers corrections, decimals, weather, noise and an empty answer', () => {
|
||||
const cases = loadGoldCases(GOLD_DIR);
|
||||
|
||||
expect(cases.some((c) => c.expected.some((e) => e.corrected))).toBe(true);
|
||||
expect(cases.some((c) => /\d,\d/.test(c.transcript))).toBe(true);
|
||||
expect(cases.some((c) => /Grad/.test(c.transcript))).toBe(true);
|
||||
expect(cases.some((c) => c.expected.length === 0)).toBe(true);
|
||||
expect(cases.some((c) => c.cannedExtra.length > 0)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseGoldCase', () => {
|
||||
it('accepts a valid case', () => {
|
||||
expect(parseGoldCase(VALID).ok).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['an unknown key', { ...VALID, extra: 1 }],
|
||||
['a bad id', { ...VALID, id: 'sample' }],
|
||||
[
|
||||
'a quote that is not in the transcript',
|
||||
{
|
||||
...VALID,
|
||||
expected: [{ field: 'broodFrames', value: 4, quote: 'Fünf Brutwaben' }],
|
||||
},
|
||||
],
|
||||
[
|
||||
'a value outside the field range',
|
||||
{
|
||||
...VALID,
|
||||
expected: [
|
||||
{ field: 'broodFrames', value: 400, quote: 'Vier Brutwaben' },
|
||||
],
|
||||
},
|
||||
],
|
||||
[
|
||||
'an unknown field',
|
||||
{
|
||||
...VALID,
|
||||
expected: [{ field: 'weather', value: 'x', quote: 'Vier Brutwaben' }],
|
||||
},
|
||||
],
|
||||
[
|
||||
'the same field twice',
|
||||
{
|
||||
...VALID,
|
||||
expected: [
|
||||
{ field: 'broodFrames', value: 4, quote: 'Vier Brutwaben' },
|
||||
{ field: 'broodFrames', value: 4, quote: 'Vier Brutwaben' },
|
||||
],
|
||||
},
|
||||
],
|
||||
['a negative dropped count', { ...VALID, expectedDropped: -1 }],
|
||||
])('rejects %s', (_label, raw) => {
|
||||
expect(parseGoldCase(raw).ok).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('offline eval over the gold set', () => {
|
||||
const cases = loadGoldCases(GOLD_DIR);
|
||||
|
||||
it('finds every expected suggestion and nothing else, and counts what validation drops', async () => {
|
||||
const results = await runEval(cases, offlineModelFor(MODEL), MODEL);
|
||||
|
||||
for (const [index, result] of results.entries()) {
|
||||
expect({
|
||||
id: result.id,
|
||||
outcome: result.outcome,
|
||||
unsaid: result.unsaid,
|
||||
dropped: result.dropped,
|
||||
}).toEqual({
|
||||
id: cases[index].id,
|
||||
outcome: 'ok',
|
||||
unsaid: [],
|
||||
dropped: cases[index].expectedDropped,
|
||||
});
|
||||
expect(
|
||||
Object.values(result.perField).every((each) => each.fp + each.fn === 0),
|
||||
).toBe(true);
|
||||
}
|
||||
const summary = summarize(results);
|
||||
expect(summary).toMatchObject({
|
||||
cases: 30,
|
||||
failedCalls: 0,
|
||||
precision: 1,
|
||||
recall: 1,
|
||||
unsaid: 0,
|
||||
failures: [],
|
||||
});
|
||||
expect(summary.dropped).toBe(
|
||||
cases.reduce((sum, each) => sum + each.expectedDropped, 0),
|
||||
);
|
||||
expect(summary.costUsd).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('prints a report with the verdict', async () => {
|
||||
const results = await runEval(cases, offlineModelFor(MODEL), MODEL);
|
||||
|
||||
const report = formatReport(results, summarize(results), {
|
||||
mode: 'offline',
|
||||
model: MODEL,
|
||||
});
|
||||
|
||||
expect(report).toContain('RESULT: PASS');
|
||||
expect(report).toContain('simulated');
|
||||
expect(report).toContain(cases[0].id);
|
||||
});
|
||||
});
|
||||
|
||||
describe('scoring', () => {
|
||||
const gold: GoldCase = {
|
||||
...(
|
||||
parseGoldCase({
|
||||
...VALID,
|
||||
transcript: 'Vier Brutwaben, Königin gesehen, Stifte da.',
|
||||
expected: [
|
||||
{ field: 'broodFrames', value: 4, quote: 'Vier Brutwaben' },
|
||||
{ field: 'queenSeen', value: true, quote: 'Königin gesehen' },
|
||||
],
|
||||
}) as { ok: true; value: GoldCase }
|
||||
).value,
|
||||
};
|
||||
|
||||
async function score(findings: object[]) {
|
||||
const model = new FakeStructuredModel([
|
||||
FakeStructuredModel.ok({ findings, unclear: [] }, MODEL),
|
||||
]);
|
||||
return runCase(gold, model, MODEL);
|
||||
}
|
||||
|
||||
it('counts a wrong value as a miss and as an unsaid adoption', async () => {
|
||||
const result = await score([
|
||||
{
|
||||
field: 'broodFrames',
|
||||
value: 4,
|
||||
quote: 'Vier Brutwaben',
|
||||
corrected: false,
|
||||
},
|
||||
{
|
||||
field: 'queenSeen',
|
||||
value: false,
|
||||
quote: 'Königin gesehen',
|
||||
corrected: false,
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result.perField.broodFrames).toEqual({ tp: 1, fp: 0, fn: 0 });
|
||||
expect(result.perField.queenSeen).toEqual({ tp: 0, fp: 1, fn: 1 });
|
||||
expect(result.unsaid).toEqual(['queenSeen']);
|
||||
});
|
||||
|
||||
it('counts a suggestion nobody expected as an unsaid adoption', async () => {
|
||||
const result = await score([
|
||||
{
|
||||
field: 'broodFrames',
|
||||
value: 4,
|
||||
quote: 'Vier Brutwaben',
|
||||
corrected: false,
|
||||
},
|
||||
{
|
||||
field: 'queenSeen',
|
||||
value: true,
|
||||
quote: 'Königin gesehen',
|
||||
corrected: false,
|
||||
},
|
||||
{
|
||||
field: 'eggsPresent',
|
||||
value: true,
|
||||
quote: 'Stifte da',
|
||||
corrected: false,
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result.perField.eggsPresent).toEqual({ tp: 0, fp: 1, fn: 0 });
|
||||
expect(result.unsaid).toEqual(['eggsPresent']);
|
||||
});
|
||||
|
||||
it('scores a refused, failed or malformed answer as all missed', async () => {
|
||||
const refused = await runCase(
|
||||
gold,
|
||||
new FakeStructuredModel([
|
||||
{
|
||||
kind: 'refused',
|
||||
usage: { input: 10, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
model: MODEL,
|
||||
},
|
||||
]),
|
||||
MODEL,
|
||||
);
|
||||
const failed = await runCase(
|
||||
gold,
|
||||
new FakeStructuredModel([{ kind: 'error', code: 'AUTH' }]),
|
||||
MODEL,
|
||||
);
|
||||
const malformed = await runCase(
|
||||
gold,
|
||||
new FakeStructuredModel([FakeStructuredModel.ok('no json', MODEL)]),
|
||||
MODEL,
|
||||
);
|
||||
|
||||
expect(refused.outcome).toBe('refused');
|
||||
expect(failed.outcome).toBe('error');
|
||||
expect(malformed.outcome).toBe('invalid');
|
||||
for (const result of [refused, failed, malformed]) {
|
||||
expect(result.perField.broodFrames).toEqual({ tp: 0, fp: 0, fn: 1 });
|
||||
}
|
||||
expect(summarize([refused]).recall).toBe(0);
|
||||
});
|
||||
|
||||
it('fails the verdict below the thresholds', async () => {
|
||||
const missed = await score([]);
|
||||
const summary = summarize([missed]);
|
||||
|
||||
expect(summary.recall).toBeLessThan(EVAL_THRESHOLDS.recall);
|
||||
expect(summary.failures).toHaveLength(1);
|
||||
|
||||
const wrong = summarize([
|
||||
await score([
|
||||
{
|
||||
field: 'broodFrames',
|
||||
value: 3,
|
||||
quote: 'Vier Brutwaben',
|
||||
corrected: false,
|
||||
},
|
||||
]),
|
||||
]);
|
||||
expect(wrong.precision).toBeLessThan(EVAL_THRESHOLDS.precision);
|
||||
});
|
||||
});
|
||||
|
||||
describe('estimateCostUsd', () => {
|
||||
const usage = {
|
||||
input: 1_000_000,
|
||||
output: 1_000_000,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
};
|
||||
|
||||
it('prices tokens per model and never guesses an unknown one', () => {
|
||||
expect(estimateCostUsd('claude-sonnet-5-5', usage)).toBe(12);
|
||||
expect(estimateCostUsd('claude-haiku-5-5', usage)).toBeCloseTo(0.6);
|
||||
expect(estimateCostUsd('some-other-model', usage)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('eval:voice stays out of CI', () => {
|
||||
it('no workflow runs eval:voice', () => {
|
||||
const workflows = join(
|
||||
__dirname,
|
||||
'..',
|
||||
'..',
|
||||
'..',
|
||||
'..',
|
||||
'.gitea',
|
||||
'workflows',
|
||||
);
|
||||
const files = readdirSync(workflows).filter((name) =>
|
||||
/\.ya?ml$/.test(name),
|
||||
);
|
||||
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
for (const file of files) {
|
||||
expect({
|
||||
file,
|
||||
callsEval: /eval[:-]voice/.test(
|
||||
readFileSync(join(workflows, file), 'utf8'),
|
||||
),
|
||||
}).toEqual({ file, callsEval: false });
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user