From 6edae3efec8b8c70df6b77e530651da2667b3ba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nick=20Rosenl=C3=B6cher?= Date: Fri, 16 Oct 2026 16:55:56 +0200 Subject: [PATCH] test: Backend-e2e in CI stabil (eigene Jahre, Sperren, ein Server je App) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - daily-evaluation nutzt die Jahre 2011/2012 statt 2001, damit der Job nicht die Daten von autopilot und reminders liest und verändert. - throttler: app.listen(0) einmal je App statt eines Servers je Anfrage (ECONNRESET bei 40 parallelen Anfragen). - outbox-claims: 120 s Zeitlimit für 2000 sequentielle Anweisungen. - migration-harness: eine `migrate deploy` zur Zeit über eine eigene Advisory-Sperre (P1002 bei parallelen Migrations-Suites). - authority-notices und authority-notices-herd laufen über exclusive-suite nacheinander; Herd-Wiederholung räumt eigene Entwürfe. Refs #32 Co-Authored-By: Claude Opus 5.5 (1M context) --- .../test/authority-notices-herd.e2e-spec.ts | 17 +- .../test/authority-notices.e2e-spec.ts | 6 + .../test/daily-evaluation.e2e-spec.ts | 202 +++++++++--------- .../test/outbox-claims.e2e-spec.ts | 52 +++-- ApiCore-Backend/test/throttler.e2e-spec.ts | 3 + ApiCore-Backend/test/utils/exclusive-suite.ts | 31 +++ .../test/utils/migration-harness.ts | 46 ++-- 7 files changed, 217 insertions(+), 140 deletions(-) create mode 100644 ApiCore-Backend/test/utils/exclusive-suite.ts diff --git a/ApiCore-Backend/test/authority-notices-herd.e2e-spec.ts b/ApiCore-Backend/test/authority-notices-herd.e2e-spec.ts index 807bb53..bf7f919 100644 --- a/ApiCore-Backend/test/authority-notices-herd.e2e-spec.ts +++ b/ApiCore-Backend/test/authority-notices-herd.e2e-spec.ts @@ -9,6 +9,7 @@ import { DomainEventRegistry } from '../src/domain-events/domain-event-registry' import { PrismaService } from '../src/prisma/prisma.service'; import { login } from './utils/api'; import { createTestApp } from './utils/create-test-app'; +import { acquireExclusive, EXCLUSIVE_KEYS } from './utils/exclusive-suite'; import { drainDomainEvents } from './utils/domain-events'; interface NoticeBody { @@ -55,7 +56,11 @@ describe('Authority notices: herd change notification (e2e)', () => { const nextKey = () => `herd-e2e-${suffix}-${(keyCounter += 1).toString().padStart(4, '0')}`; + let releaseExclusive: () => Promise; + beforeAll(async () => { + // Suites that clear or create the global open notice drafts take turns. + releaseExclusive = await acquireExclusive(EXCLUSIVE_KEYS.authorityNotices); app = await createTestApp(); prisma = app.get(PrismaService); const passwordHash = await bcrypt.hash(password, 12); @@ -107,6 +112,11 @@ describe('Authority notices: herd change notification (e2e)', () => { await prisma.notification.deleteMany({ where: { userId: { in: [...managerIds, staffId] } }, }); + // A retried attempt must start without the notices (and the open draft) + // of the failed one. + await prisma.authorityNotice.deleteMany({ + where: { id: { in: noticeIds.splice(0) } }, + }); } beforeEach(resetScenarioState); @@ -168,6 +178,7 @@ describe('Authority notices: herd change notification (e2e)', () => { }); await prisma.user.deleteMany({ where: { id: { in: userIds } } }); await app.close(); + await releaseExclusive(); }); /** A notice sent today is the baseline: the current herd. */ @@ -319,7 +330,7 @@ describe('Authority notices: herd change notification (e2e)', () => { scenario('an apiary move (LOCATION_CHANGED) counts', async () => { await sendBaseline(); - await patchApiary({ location: `Neuer Standort ${suffix}` }); + await patchApiary({ location: `Neuer Standort ${nextKey()}` }); expect(await unreadOf(managerIds)).toHaveLength(managerIds.length); }); @@ -327,7 +338,7 @@ describe('Authority notices: herd change notification (e2e)', () => { scenario('a rename does not count', async () => { await sendBaseline(); - await patchApiary({ name: `Herd Stand umbenannt ${suffix}` }); + await patchApiary({ name: `Herd Stand umbenannt ${nextKey()}` }); expect(await unreadOf(managerIds)).toHaveLength(0); }); @@ -344,7 +355,7 @@ describe('Authority notices: herd change notification (e2e)', () => { await addColony(); expect(await unreadOf(managerIds)).toHaveLength(0); - await patchApiary({ location: `Ort wieder neu ${suffix}` }); + await patchApiary({ location: `Ort wieder neu ${nextKey()}` }); expect(await unreadOf(managerIds)).toHaveLength(managerIds.length); }, ); diff --git a/ApiCore-Backend/test/authority-notices.e2e-spec.ts b/ApiCore-Backend/test/authority-notices.e2e-spec.ts index dd5afab..c578c92 100644 --- a/ApiCore-Backend/test/authority-notices.e2e-spec.ts +++ b/ApiCore-Backend/test/authority-notices.e2e-spec.ts @@ -6,6 +6,7 @@ import { loadHerdApiaryRows } from '../src/compliance/herd-snapshot'; import { PrismaService } from '../src/prisma/prisma.service'; import { login } from './utils/api'; import { createTestApp } from './utils/create-test-app'; +import { acquireExclusive, EXCLUSIVE_KEYS } from './utils/exclusive-suite'; interface NoticeBody { id: string; @@ -38,7 +39,11 @@ describe('Authority notices (e2e)', () => { const nextKey = () => `notices-e2e-${suffix}-${(keyCounter += 1).toString().padStart(4, '0')}`; + let releaseExclusive: () => Promise; + beforeAll(async () => { + // Suites that clear or create the global open notice drafts take turns. + releaseExclusive = await acquireExclusive(EXCLUSIVE_KEYS.authorityNotices); app = await createTestApp(); prisma = app.get(PrismaService); const passwordHash = await bcrypt.hash(password, 12); @@ -99,6 +104,7 @@ describe('Authority notices (e2e)', () => { await prisma.user.deleteMany({ where: { roleId: noRightsRoleId } }); await prisma.role.deleteMany({ where: { id: noRightsRoleId } }); await app.close(); + await releaseExclusive(); }); async function livingCount(): Promise { diff --git a/ApiCore-Backend/test/daily-evaluation.e2e-spec.ts b/ApiCore-Backend/test/daily-evaluation.e2e-spec.ts index 1d08cba..0f35ed1 100644 --- a/ApiCore-Backend/test/daily-evaluation.e2e-spec.ts +++ b/ApiCore-Backend/test/daily-evaluation.e2e-spec.ts @@ -25,8 +25,10 @@ const NOTHING_NEW = { /** * The job evaluates the whole database. The e2e files share it and run in - * parallel, so the evaluation runs on days in 2001: no other file has - * records there, and the events of this file stay its own. Called through + * parallel, so the evaluation runs on days in 2011 and 2012: no other file has + * records there (autopilot and reminders own 2001; this file used to share it, + * so its "nothing new" counts picked up their colonies and its phenology step + * wrote season rows for their apiaries), and the events of this file stay its own. Called through * TimeEventsStep.run(day) instead of runCli, which would also write JobRun * rows that test/jobs.e2e-spec.ts counts. */ @@ -66,7 +68,7 @@ describe('daily-evaluation (e2e)', () => { phenologyStep = app.get(PhenologyStep); expiryStep = app.get(SuggestionsExpiryStep); // The real AutopilotStep, but on the colonies of this file only: other - // files share the database and must not be reconciled on a 2001 day. + // files share the database and must not be reconciled on a 2011 day. autopilotStep = new AutopilotStep( { colony: { @@ -107,18 +109,18 @@ describe('daily-evaluation (e2e)', () => { }); ids[`queen:${name}`] = created.id; }; - await queen('aged', { hatchYear: 1999, status: 'ACTIVE' }); - await queen('young', { hatchYear: 2000, status: 'ACTIVE' }); - await queen('reserve', { hatchYear: 1999, status: 'RESERVE' }); + await queen('aged', { hatchYear: 2009, status: 'ACTIVE' }); + await queen('young', { hatchYear: 2010, status: 'ACTIVE' }); + await queen('reserve', { hatchYear: 2009, status: 'RESERVE' }); - // Living, created 2001-04-20 and never measured: varroa due 2001-05-18. + // Living, created 2011-04-20 and never measured: varroa due 2011-05-18. const living = await prisma.colony.create({ data: { code: `DE-E2E-${suffix}-living`, apiaryId: apiary.id, colonyType: 'PRODUCTION_COLONY', currentQueenId: ids['queen:aged'], - createdAt: d('2001-04-20'), + createdAt: d('2011-04-20'), }, }); ids.living = living.id; @@ -128,15 +130,15 @@ describe('daily-evaluation (e2e)', () => { apiaryId: apiary.id, colonyType: 'PRODUCTION_COLONY', status: 'DECEASED', - createdAt: d('2001-04-20'), + createdAt: d('2011-04-20'), }, }); ids.dead = dead.id; - // Inspected 2001-04-28, in the swarm season (every 7 days): the next - // inspection is due 2001-05-05 and overdue on 2001-05-06. + // Inspected 2011-04-28, in the swarm season (every 7 days): the next + // inspection is due 2011-05-05 and overdue on 2011-05-06. for (const colonyId of [living.id, dead.id]) { await prisma.inspection.create({ - data: { colonyId, userId: user.id, date: d('2001-04-28') }, + data: { colonyId, userId: user.id, date: d('2011-04-28') }, }); } @@ -157,18 +159,18 @@ describe('daily-evaluation (e2e)', () => { }); ids[`treatment:${name}`] = created.id; }; - // 2001-05-10 + 7 = 2001-05-17, in the window of 2001-05-20. + // 2011-05-10 + 7 = 2011-05-17, in the window of 2011-05-20. await treatment('ending', { - date: d('2001-05-08'), - endDate: d('2001-05-10'), + date: d('2011-05-08'), + endDate: d('2011-05-10'), withdrawalPeriodDays: 7, }); - // 2001-05-01 + 7 = 2001-05-08, before the window. + // 2011-05-01 + 7 = 2011-05-08, before the window. await treatment('ended-earlier', { - date: d('2001-05-01'), + date: d('2011-05-01'), withdrawalPeriodDays: 7, }); - await treatment('no-withdrawal', { date: d('2001-05-15') }); + await treatment('no-withdrawal', { date: d('2011-05-15') }); const task = async ( name: string, @@ -179,8 +181,8 @@ describe('daily-evaluation (e2e)', () => { }); ids[`task:${name}`] = created.id; }; - await task('open', { dueDate: d('2001-05-18'), status: 'OPEN' }); - await task('done', { dueDate: d('2001-05-18'), status: 'DONE' }); + await task('open', { dueDate: d('2011-05-18'), status: 'OPEN' }); + await task('done', { dueDate: d('2011-05-18'), status: 'DONE' }); }); afterAll(async () => { @@ -222,10 +224,10 @@ describe('daily-evaluation (e2e)', () => { }); it('running the job twice yields exactly one WithdrawalEnded per treatment and day', async () => { - const first = await step.run('2001-05-20'); - const second = await step.run('2001-05-20'); + const first = await step.run('2011-05-20'); + const second = await step.run('2011-05-20'); // The next day the occurrence is still in the window - and still known. - const nextDay = await step.run('2001-05-21'); + const nextDay = await step.run('2011-05-21'); expect(first.WithdrawalEnded).toBeGreaterThanOrEqual(1); expect(second).toEqual(NOTHING_NEW); @@ -234,8 +236,8 @@ describe('daily-evaluation (e2e)', () => { { type: 'WithdrawalEnded', aggregateType: 'Treatment', - payload: { colonyId: ids.living, endedOn: '2001-05-17' }, - dedupeKey: `WithdrawalEnded:${ids['treatment:ending']}:2001-05-17`, + payload: { colonyId: ids.living, endedOn: '2011-05-17' }, + dedupeKey: `WithdrawalEnded:${ids['treatment:ending']}:2011-05-17`, actorUserId: null, }, ]); @@ -278,11 +280,11 @@ describe('daily-evaluation (e2e)', () => { afterEach(async () => { await prisma.notification.deleteMany({ - where: { relatedType: 'ComplianceItem', relatedId: 'tsk-report:2001' }, + where: { relatedType: 'ComplianceItem', relatedId: 'tsk-report:2011' }, }); - await prisma.authorityNotice.deleteMany({ where: { year: 2001 } }); + await prisma.authorityNotice.deleteMany({ where: { year: 2011 } }); await prisma.domainEvent.deleteMany({ - where: { aggregateId: 'tsk:2001' }, + where: { aggregateId: 'tsk:2011' }, }); await prisma.complianceSettings.update({ where: { id: 'singleton' }, @@ -291,25 +293,25 @@ describe('daily-evaluation (e2e)', () => { }); async function expectOneDraftAndNotices(): Promise { - expect(await eventsOf('tsk:2001')).toEqual([ + expect(await eventsOf('tsk:2011')).toEqual([ { type: 'TskReferenceDayReached', aggregateType: 'Compliance', - payload: { year: 2001, referenceDay: '2001-03-03' }, - dedupeKey: 'TskReferenceDayReached:tsk:2001:2001-03-03', + payload: { year: 2011, referenceDay: '2011-03-03' }, + dedupeKey: 'TskReferenceDayReached:tsk:2011:2011-03-03', actorUserId: null, }, ]); const drafts = await prisma.authorityNotice.findMany({ - where: { kind: 'TSK_REPORT', year: 2001 }, + where: { kind: 'TSK_REPORT', year: 2011 }, }); expect(drafts).toHaveLength(1); expect(drafts[0]).toMatchObject({ status: 'DRAFT', - openKey: 'TSK_REPORT:2001', + openKey: 'TSK_REPORT:2011', }); const notifications = await prisma.notification.findMany({ - where: { relatedType: 'ComplianceItem', relatedId: 'tsk-report:2001' }, + where: { relatedType: 'ComplianceItem', relatedId: 'tsk-report:2011' }, select: { userId: true, priority: true, category: true, message: true }, }); // One copy per recipient, written once - and every permitted user that @@ -331,17 +333,17 @@ describe('daily-evaluation (e2e)', () => { category: 'COMPLIANCE', }); expect(row.message).toMatch( - /^TSK-Stichtag 03\.03\.2001: \d+ Völker gezählt\. Bitte prüfen und melden\.$/, + /^TSK-Stichtag 03\.03\.2011: \d+ Völker gezählt\. Bitte prüfen und melden\.$/, ); } } it('daily evaluation twice on the reference day: one event, one draft', async () => { - const first = await step.run('2001-03-03'); - const second = await step.run('2001-03-03'); - await drainDomainEvents(app, ['tsk:2001']); - await step.run('2001-03-04'); - await drainDomainEvents(app, ['tsk:2001']); + const first = await step.run('2011-03-03'); + const second = await step.run('2011-03-03'); + await drainDomainEvents(app, ['tsk:2011']); + await step.run('2011-03-04'); + await drainDomainEvents(app, ['tsk:2011']); expect(first.TskReferenceDayReached).toBe(1); expect(second.TskReferenceDayReached).toBe(0); @@ -349,9 +351,9 @@ describe('daily-evaluation (e2e)', () => { }); it('catches up a missed reference day on the 5th exactly once', async () => { - const catchUp = await step.run('2001-03-05'); - const again = await step.run('2001-03-05'); - await drainDomainEvents(app, ['tsk:2001']); + const catchUp = await step.run('2011-03-05'); + const again = await step.run('2011-03-05'); + await drainDomainEvents(app, ['tsk:2011']); expect(catchUp.TskReferenceDayReached).toBe(1); expect(again.TskReferenceDayReached).toBe(0); @@ -360,14 +362,14 @@ describe('daily-evaluation (e2e)', () => { }); it('records due tasks and varroa checks only for open tasks and living colonies', async () => { - await step.run('2001-05-20'); + await step.run('2011-05-20'); expect(await eventsOf(ids['task:open'])).toEqual([ { type: 'TaskDue', aggregateType: 'Task', - payload: { dueOn: '2001-05-18' }, - dedupeKey: `TaskDue:${ids['task:open']}:2001-05-18`, + payload: { dueOn: '2011-05-18' }, + dedupeKey: `TaskDue:${ids['task:open']}:2011-05-18`, actorUserId: null, }, ]); @@ -377,12 +379,12 @@ describe('daily-evaluation (e2e)', () => { type: 'VarroaMeasurementDue', aggregateType: 'Colony', payload: { - dueOn: '2001-05-18', + dueOn: '2011-05-18', lastMeasuredOn: null, kind: 'INTERVAL', anchor: 'none', }, - dedupeKey: `VarroaMeasurementDue:${ids.living}:2001-05-18`, + dedupeKey: `VarroaMeasurementDue:${ids.living}:2011-05-18`, actorUserId: null, }, ]); @@ -390,9 +392,9 @@ describe('daily-evaluation (e2e)', () => { }); it('InspectionOverdue follows the segment interval', async () => { - await step.run('2001-05-12'); + await step.run('2011-05-12'); - // SCHWARMZEIT since 2001-04-15: 2001-04-28 + 7 = 2001-05-05, not + 21. + // SCHWARMZEIT since 2011-04-15: 2011-04-28 + 7 = 2011-05-05, not + 21. expect( (await eventsOf(ids.living)).filter( (event) => event.type === 'InspectionOverdue', @@ -402,11 +404,11 @@ describe('daily-evaluation (e2e)', () => { type: 'InspectionOverdue', aggregateType: 'Colony', payload: { - dueOn: '2001-05-05', - lastInspectedOn: '2001-04-28', + dueOn: '2011-05-05', + lastInspectedOn: '2011-04-28', firstOfYear: false, }, - dedupeKey: `InspectionOverdue:${ids.living}:2001-05-06`, + dedupeKey: `InspectionOverdue:${ids.living}:2011-05-06`, actorUserId: null, }, ]); @@ -414,50 +416,50 @@ describe('daily-evaluation (e2e)', () => { }); it('records no inspection in WINTERRUHE and the first one the day after FRUEHJAHR begins', async () => { - await step.run('2002-03-10'); + await step.run('2012-03-10'); expect( (await eventsOf(ids.living)).filter((event) => - event.dedupeKey?.endsWith(':2002-03-21'), + event.dedupeKey?.endsWith(':2012-03-21'), ), ).toEqual([]); - // FRUEHJAHR begins 2002-03-20; the colony was last inspected in 2001. - await step.run('2002-03-21'); + // FRUEHJAHR begins 2012-03-20; the colony was last inspected in 2011. + await step.run('2012-03-21'); expect( (await eventsOf(ids.living)).filter((event) => - event.dedupeKey?.endsWith(':2002-03-21'), + event.dedupeKey?.endsWith(':2012-03-21'), ), ).toEqual([ { type: 'InspectionOverdue', aggregateType: 'Colony', payload: { - dueOn: '2002-03-20', - lastInspectedOn: '2001-04-28', + dueOn: '2012-03-20', + lastInspectedOn: '2011-04-28', firstOfYear: true, }, - dedupeKey: `InspectionOverdue:${ids.living}:2002-03-21`, + dedupeKey: `InspectionOverdue:${ids.living}:2012-03-21`, actorUserId: null, }, ]); }); it('reports the current queen of a living colony once she is aged, on any job day of the year', async () => { - await step.run('2001-09-09'); - await step.run('2001-09-10'); + await step.run('2011-09-09'); + await step.run('2011-09-10'); - // One per queen and year: the 2002 run of the test above added its own. + // One per queen and year: the 2012 run of the test above added its own. const events = await eventsOf(ids['queen:aged']); expect(events.map((event) => event.dedupeKey)).toEqual([ - `QueenAged:${ids['queen:aged']}:2001-01-01`, - `QueenAged:${ids['queen:aged']}:2002-01-01`, + `QueenAged:${ids['queen:aged']}:2011-01-01`, + `QueenAged:${ids['queen:aged']}:2012-01-01`, ]); expect(events.slice(0, 1)).toEqual([ { type: 'QueenAged', aggregateType: 'Queen', - payload: { hatchYear: 1999, colonyId: ids.living }, - dedupeKey: `QueenAged:${ids['queen:aged']}:2001-01-01`, + payload: { hatchYear: 2009, colonyId: ids.living }, + dedupeKey: `QueenAged:${ids['queen:aged']}:2011-01-01`, actorUserId: null, }, ]); @@ -466,7 +468,7 @@ describe('daily-evaluation (e2e)', () => { }); describe('phenology step', () => { - const DAY = '2001-04-30'; + const DAY = '2011-04-30'; let weatherApiaryId = ''; beforeAll(async () => { @@ -481,8 +483,8 @@ describe('daily-evaluation (e2e)', () => { // 10 degrees every day from January 1st to April 29th: GTS 965. const days: Prisma.ApiaryWeatherDayCreateManyInput[] = []; for ( - let t = d('2001-01-01').getTime(); - t <= d('2001-04-29').getTime(); + let t = d('2011-01-01').getTime(); + t <= d('2011-04-29').getTime(); t += 86_400_000 ) { days.push({ @@ -497,7 +499,7 @@ describe('daily-evaluation (e2e)', () => { data: { apiaryId: apiary.id, phase: 'HOLUNDER', - date: d('2001-04-29'), + date: d('2011-04-29'), userId: ids.user, }, }); @@ -510,8 +512,8 @@ describe('daily-evaluation (e2e)', () => { await prisma.phenologyReport.deleteMany({ where: { apiaryId: weatherApiaryId }, }); - // Rows of 2001 would feed the season of the other tests of this file. - await prisma.apiaryPhenology.deleteMany({ where: { year: 2001 } }); + // Rows of 2011 would feed the season of the other tests of this file. + await prisma.apiaryPhenology.deleteMany({ where: { year: 2011 } }); await prisma.apiary.deleteMany({ where: { id: weatherApiaryId } }); }); @@ -530,7 +532,7 @@ describe('daily-evaluation (e2e)', () => { expect(Number(result.phenologyApiaries)).toBeGreaterThanOrEqual(2); const rows = await prisma.apiaryPhenology.findMany({ - where: { apiaryId: { in: [ids.apiary, weatherApiaryId] }, year: 2001 }, + where: { apiaryId: { in: [ids.apiary, weatherApiaryId] }, year: 2011 }, orderBy: { apiaryId: 'asc' }, }); expect(rows).toHaveLength(2); @@ -545,7 +547,7 @@ describe('daily-evaluation (e2e)', () => { // A report for HOLUNDER beats GTS and calendar. expect(byApiary.get(weatherApiaryId)).toMatchObject({ gts: 965, - gtsThroughDay: d('2001-04-29'), + gtsThroughDay: d('2011-04-29'), estimatedPhase: 'APFEL', reportedPhase: 'HOLUNDER', effectivePhase: 'HOLUNDER', @@ -579,25 +581,25 @@ describe('daily-evaluation (e2e)', () => { }); describe('autopilot and suggestions-expiry steps', () => { - /** Living colony, inspected 2001-05-03 and measured 2001-05-10. */ + /** Living colony, inspected 2011-05-03 and measured 2011-05-10. */ async function lateColony(name: string): Promise { const colony = await prisma.colony.create({ data: { code: `DE-E2E-${suffix}-ap-${name}`, apiaryId: ids.apiary, colonyType: 'PRODUCTION_COLONY', - createdAt: d('2001-04-20'), + createdAt: d('2011-04-20'), }, }); autopilotColonyIds.push(colony.id); await prisma.inspection.create({ - data: { colonyId: colony.id, userId: ids.user, date: d('2001-05-03') }, + data: { colonyId: colony.id, userId: ids.user, date: d('2011-05-03') }, }); await prisma.varroaMeasurement.create({ data: { colonyId: colony.id, userId: ids.user, - date: d('2001-05-10'), + date: d('2011-05-10'), method: 'NATURAL_MITE_FALL', }, }); @@ -613,10 +615,10 @@ describe('daily-evaluation (e2e)', () => { it('running autopilot and expiry twice yields the same suggestion counts', async () => { const colonyId = await lateColony('twice'); - const first = await autopilotStep.run('2001-05-12'); - const firstExpiry = await expiryStep.run('2001-05-12'); - const second = await autopilotStep.run('2001-05-12'); - const secondExpiry = await expiryStep.run('2001-05-12'); + const first = await autopilotStep.run('2011-05-12'); + const firstExpiry = await expiryStep.run('2011-05-12'); + const second = await autopilotStep.run('2011-05-12'); + const secondExpiry = await expiryStep.run('2011-05-12'); expect(first).toMatchObject({ autopilotColonies: 1, @@ -635,13 +637,13 @@ describe('daily-evaluation (e2e)', () => { it('an OPEN suggestion past expiresAt becomes EXPIRED', async () => { const colonyId = await lateColony('expires'); - await autopilotStep.run('2001-05-12'); + await autopilotStep.run('2011-05-12'); - // expiresAt = 2001-06-11 - expect(await expiryStep.run('2001-06-11')).toEqual({ + // expiresAt = 2011-06-11 + expect(await expiryStep.run('2011-06-11')).toEqual({ suggestionsExpired: 0, }); - expect(await expiryStep.run('2001-06-12')).toEqual({ + expect(await expiryStep.run('2011-06-12')).toEqual({ suggestionsExpired: expect.any(Number) as number, }); @@ -651,10 +653,10 @@ describe('daily-evaluation (e2e)', () => { it('an expired INSPECT suggestion of the same anchor is proposed again on the next sync (dedupeKey was released)', async () => { const colonyId = await lateColony('again'); - await autopilotStep.run('2001-05-12'); - await expiryStep.run('2001-06-12'); + await autopilotStep.run('2011-05-12'); + await expiryStep.run('2011-06-12'); - const result = await autopilotStep.run('2001-06-12'); + const result = await autopilotStep.run('2011-06-12'); expect(result.suggestionsOpened).toBeGreaterThanOrEqual(1); expect((await inspectSuggestions(colonyId)).map((s) => s.status)).toEqual( @@ -666,13 +668,13 @@ describe('daily-evaluation (e2e)', () => { const colonyId = await lateColony('overdue20'); await prisma.inspection.updateMany({ where: { colonyId }, - data: { date: d('2001-04-10') }, + data: { date: d('2011-04-10') }, }); - await step.run('2001-05-12'); - await autopilotStep.run('2001-05-12'); + await step.run('2011-05-12'); + await autopilotStep.run('2011-05-12'); - // Due 2001-04-17: the event of 04-18 is outside the 7-day catch-up. + // Due 2011-04-17: the event of 04-18 is outside the 7-day catch-up. expect( await prisma.domainEvent.count({ where: { aggregateId: colonyId, type: 'InspectionOverdue' }, @@ -680,19 +682,19 @@ describe('daily-evaluation (e2e)', () => { ).toBe(0); const [suggestion] = await inspectSuggestions(colonyId); expect(suggestion).toMatchObject({ status: 'OPEN', priority: 'HIGH' }); - expect(suggestion.dueDate?.toISOString().slice(0, 10)).toBe('2001-04-17'); + expect(suggestion.dueDate?.toISOString().slice(0, 10)).toBe('2011-04-17'); }); it('a RESOLVED autopilot suggestion whose condition returns is proposed again', async () => { const colonyId = await lateColony('returns'); - await autopilotStep.run('2001-05-12'); + await autopilotStep.run('2011-05-12'); const fresh = await prisma.inspection.create({ - data: { colonyId, userId: ids.user, date: d('2001-05-12') }, + data: { colonyId, userId: ids.user, date: d('2011-05-12') }, }); - const resolved = await autopilotStep.run('2001-05-12'); + const resolved = await autopilotStep.run('2011-05-12'); await prisma.inspection.delete({ where: { id: fresh.id } }); - const returned = await autopilotStep.run('2001-05-12'); + const returned = await autopilotStep.run('2011-05-12'); expect(resolved.suggestionsResolved).toBeGreaterThanOrEqual(1); expect(returned.suggestionsOpened).toBeGreaterThanOrEqual(1); diff --git a/ApiCore-Backend/test/outbox-claims.e2e-spec.ts b/ApiCore-Backend/test/outbox-claims.e2e-spec.ts index f131bbd..8de46a3 100644 --- a/ApiCore-Backend/test/outbox-claims.e2e-spec.ts +++ b/ApiCore-Backend/test/outbox-claims.e2e-spec.ts @@ -5,6 +5,10 @@ import { PrismaModule } from '../src/prisma/prisma.module'; import { PrismaService } from '../src/prisma/prisma.service'; const ROUNDS = 1000; +// 2 x ROUNDS sequential statements: seconds locally, but on a loaded CI +// database the default 20 s is too tight. The rounds themselves stay (they are +// what catches the rounding race). +const ROUNDS_TIMEOUT_MS = 120_000; /** * "availableAt" is TIMESTAMP(3): a value written from SQL now() (stale @@ -30,28 +34,32 @@ describe('claimBatch and millisecond rounding (e2e)', () => { await moduleRef.close(); }); - it('claims an entry whose availableAt was just set to now()', async () => { - const event = await prisma.domainEvent.create({ - data: { - type: 'TaskDue', - aggregateType: 'Task', - aggregateId: correlationId, - payload: {}, - correlationId, - }, - }); - let missed = 0; - for (let round = 0; round < ROUNDS; round++) { - await prisma.$executeRaw`UPDATE "DomainEvent" SET "status" = 'PENDING', "claimedAt" = NULL, "availableAt" = now() WHERE "id" = ${event.id}`; - const claimed = await claimBatch(prisma, 'DomainEvent', { - limit: 1, - filter: Prisma.sql`"correlationId" = ${correlationId}`, - orderBy: 'occurredAt', + it( + 'claims an entry whose availableAt was just set to now()', + async () => { + const event = await prisma.domainEvent.create({ + data: { + type: 'TaskDue', + aggregateType: 'Task', + aggregateId: correlationId, + payload: {}, + correlationId, + }, }); - if (claimed.length !== 1) { - missed++; + let missed = 0; + for (let round = 0; round < ROUNDS; round++) { + await prisma.$executeRaw`UPDATE "DomainEvent" SET "status" = 'PENDING', "claimedAt" = NULL, "availableAt" = now() WHERE "id" = ${event.id}`; + const claimed = await claimBatch(prisma, 'DomainEvent', { + limit: 1, + filter: Prisma.sql`"correlationId" = ${correlationId}`, + orderBy: 'occurredAt', + }); + if (claimed.length !== 1) { + missed++; + } } - } - expect(missed).toBe(0); - }); + expect(missed).toBe(0); + }, + ROUNDS_TIMEOUT_MS, + ); }); diff --git a/ApiCore-Backend/test/throttler.e2e-spec.ts b/ApiCore-Backend/test/throttler.e2e-spec.ts index 6e17016..c1b9a1d 100644 --- a/ApiCore-Backend/test/throttler.e2e-spec.ts +++ b/ApiCore-Backend/test/throttler.e2e-spec.ts @@ -22,6 +22,9 @@ describe('Database-backed throttler (e2e)', () => { ): Promise> => { namespaces.push(throttlerNamespace); const app = await createTestApp({ throttlerNamespace }); + // Listen once: otherwise supertest opens a throwaway server per request + // and 40 parallel ones reset connections (ECONNRESET) on a loaded CI box. + await app.listen(0); apps.push(app); prisma = app.get(PrismaService); return app; diff --git a/ApiCore-Backend/test/utils/exclusive-suite.ts b/ApiCore-Backend/test/utils/exclusive-suite.ts new file mode 100644 index 0000000..91de46f --- /dev/null +++ b/ApiCore-Backend/test/utils/exclusive-suite.ts @@ -0,0 +1,31 @@ +import { Client } from 'pg'; + +/** Arbitrary keys, one per shared resource a suite needs to own alone. */ +export const EXCLUSIVE_KEYS = { + /** Open drafts are unique per kind: only one suite may create/clear them. */ + authorityNotices: 72707371, +} as const; + +/** + * Holds a Postgres session advisory lock for the lifetime of a suite, so + * suites that clear or count a table whose rows are global (e.g. the open + * draft per notice kind) never run at the same time, even in parallel jest + * workers. Blocks until the previous owner released it; the lock disappears + * with the connection if a worker dies. Call release() in afterAll. + */ +export async function acquireExclusive( + key: number, +): Promise<() => Promise> { + const url = process.env.DATABASE_URL; + if (!url) { + throw new Error('DATABASE_URL is not set'); + } + const connection = new URL(url); + connection.searchParams.delete('schema'); + const client = new Client({ connectionString: connection.toString() }); + await client.connect(); + await client.query('SELECT pg_advisory_lock($1)', [key]); + return async () => { + await client.end(); + }; +} diff --git a/ApiCore-Backend/test/utils/migration-harness.ts b/ApiCore-Backend/test/utils/migration-harness.ts index bf3f1e6..2790f45 100644 --- a/ApiCore-Backend/test/utils/migration-harness.ts +++ b/ApiCore-Backend/test/utils/migration-harness.ts @@ -19,6 +19,15 @@ const PRISMA_CLI = path.join( 'index.js', ); const DEPLOY_TIMEOUT_MS = 100_000; +/** + * `prisma migrate deploy` takes pg_advisory_lock(72707369) and gives up after + * 10 s. Advisory locks are per DATABASE, not per schema, so scratch schemas + * of parallel migration suites (honey-ledger, t4, t6) collide on it (P1002 in + * CI). The harness therefore runs one deploy at a time across jest workers + * with its own, blocking advisory lock; the session lock disappears with the + * connection should a worker die. + */ +const HARNESS_DEPLOY_LOCK_KEY = 72707370; const SCHEMA_NAME_PATTERN = /^mig_[0-9a-f]{12}$/; export interface ScratchDatabase { @@ -119,21 +128,28 @@ export async function createScratchDatabase( copied.add(name); } } - await execFileAsync( - process.execPath, - [ - PRISMA_CLI, - 'migrate', - 'deploy', - '--config', - path.join(workDir, 'prisma.config.ts'), - ], - { - cwd: BACKEND_ROOT, - env: { ...process.env, MIGRATION_HARNESS_URL: scratchUrl }, - timeout: DEPLOY_TIMEOUT_MS, - }, - ); + await admin.query('SELECT pg_advisory_lock($1)', [HARNESS_DEPLOY_LOCK_KEY]); + try { + await execFileAsync( + process.execPath, + [ + PRISMA_CLI, + 'migrate', + 'deploy', + '--config', + path.join(workDir, 'prisma.config.ts'), + ], + { + cwd: BACKEND_ROOT, + env: { ...process.env, MIGRATION_HARNESS_URL: scratchUrl }, + timeout: DEPLOY_TIMEOUT_MS, + }, + ); + } finally { + await admin.query('SELECT pg_advisory_unlock($1)', [ + HARNESS_DEPLOY_LOCK_KEY, + ]); + } }; let client: Client | null = null;