feat: platform backbone and field capture complete (checkpoint)

Platform backbone (plan 2026-10-13-plattform-rueckgrat), all tasks reviewed:
- webhooks with delivery table, retries and stable delivery id
- calendar handlers; a finished task removes its Outlook event
- tasks, automations, inspections, varroa, treatments, feedings,
  harvests, colonies, wintering, stock and bottling record domain events
  in the same transaction; legacy post-commit calls removed
- row locks through one helper (lockRowForUpdate, lockRowsForShare,
  lockRowsForUpdate)
- job framework with JobRun, cli exit codes, cleanup, task templates,
  daily evaluation, hourly app-backup and the rekey job
- system API and page "Verwaltung > System" with retry
- @nestjs/schedule removed, guard tests against in-process schedulers

Field capture T3 (plan 2026-10-13-t3-erfassung-am-stand), all tasks reviewed:
- inspection photo offline without metadata
- bulk capture for treatments, feedings and varroa (backend + frontend)
- offline stand mode with service worker, offline pages and packs
- offline session, cold start and logout on shared devices

Docs: ARCHITECTURE.md assumptions and sections updated, TASKS.md.
Gates: backend lint, typecheck, 1770 unit, 420 e2e, build, no schema
drift; frontend lint, typecheck, 913 tests, build. Not rolled out.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
Nick SzaunigandClaude Opus 5.5 committed 2026-10-14 11:36:38 +02:00
1 parent c01a38835a
commit cdd30cf3ee
288 files changed
+39165 -2853

No files matched your search

@@ -0,0 +1,32 @@
import { ApiPropertyOptional } from '@nestjs/swagger';
import { Type } from 'class-transformer';
import { IsIn, IsInt, IsOptional, Max, Min } from 'class-validator';
import { JOB_NAMES, type JobName } from '../../jobs/job-names';
export const SYSTEM_LIST_DEFAULT_LIMIT = 50;
export const SYSTEM_LIST_MAX_LIMIT = 100;
export class SystemListQueryDto {
@ApiPropertyOptional({
minimum: 1,
maximum: SYSTEM_LIST_MAX_LIMIT,
default: SYSTEM_LIST_DEFAULT_LIMIT,
description: 'Anzahl der neuesten Einträge',
})
@IsOptional()
@Type(() => Number)
@IsInt()
@Min(1)
@Max(SYSTEM_LIST_MAX_LIMIT)
limit?: number;
}
export class JobRunListQueryDto extends SystemListQueryDto {
@ApiPropertyOptional({
enum: JOB_NAMES,
description: 'Nur Läufe dieses Jobs',
})
@IsOptional()
@IsIn(JOB_NAMES)
job?: JobName;
}
@@ -0,0 +1,287 @@
import { Logger } from '@nestjs/common';
import { JobRun, Prisma } from '../../generated/prisma';
import { JOB_NAMES } from '../jobs/job-names';
import { NotificationsService } from '../notifications/notifications.service';
import { PrismaService } from '../prisma/prisma.service';
import { JOB_ABORTED_ERROR, JobRunService } from './job-run.service';
type RawCall = [readonly string[], ...unknown[]];
const STARTED_AT = new Date('2026-10-13T04:00:00.000Z');
function runRow(overrides: Partial<JobRun> = {}): JobRun {
return {
id: 'run-1',
job: 'daily-evaluation',
status: 'RUNNING',
startedAt: STARTED_AT,
finishedAt: null,
result: null,
error: null,
...overrides,
};
}
describe('JobRunService', () => {
let jobRun: {
create: jest.Mock;
findMany: jest.Mock;
findFirst: jest.Mock;
};
let tx: {
$executeRaw: jest.Mock<Promise<number>, RawCall>;
$queryRaw: jest.Mock<Promise<unknown>, RawCall>;
};
let prisma: {
jobRun: typeof jobRun;
$transaction: jest.Mock;
$queryRaw: jest.Mock<Promise<unknown>, RawCall>;
};
let notifications: { recordForPermission: jest.Mock };
let service: JobRunService;
let errorSpy: jest.SpyInstance;
const sqlText = (call: RawCall): string => call[0].join('?');
beforeEach(() => {
jobRun = {
create: jest.fn(({ data }: { data: { job: string } }) =>
Promise.resolve(runRow({ job: data.job })),
),
findMany: jest.fn().mockResolvedValue([]),
findFirst: jest.fn().mockResolvedValue(null),
};
tx = {
$executeRaw: jest.fn<Promise<number>, RawCall>().mockResolvedValue(1),
$queryRaw: jest
.fn<Promise<unknown>, RawCall>()
.mockResolvedValue([{ notified: false }]),
};
prisma = {
jobRun,
$transaction: jest.fn((work: (client: typeof tx) => Promise<unknown>) =>
work(tx),
),
// Finishing a run: UPDATE … RETURNING *, the status is the first value.
$queryRaw: jest.fn<Promise<unknown>, RawCall>((_sql, status) =>
Promise.resolve([runRow({ status: status as JobRun['status'] })]),
),
};
notifications = { recordForPermission: jest.fn().mockResolvedValue(2) };
service = new JobRunService(
prisma as unknown as PrismaService,
notifications as unknown as NotificationsService,
);
errorSpy = jest
.spyOn(Logger.prototype, 'error')
.mockImplementation(() => undefined);
jest.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined);
});
afterEach(() => jest.restoreAllMocks());
describe('run', () => {
it('records RUNNING first, then SUCCEEDED with the result', async () => {
const work = jest.fn(() => {
expect(jobRun.create).toHaveBeenCalledWith({
data: { job: 'cleanup' },
});
return Promise.resolve({ deleted: 3, dryRun: false });
});
const run = await service.run('cleanup', work);
expect(work).toHaveBeenCalledTimes(1);
const [finish] = prisma.$queryRaw.mock.calls;
expect(finish.slice(1)).toEqual([
'SUCCEEDED',
'{"deleted":3,"dryRun":false}',
null,
'run-1',
]);
expect(run.status).toBe('SUCCEEDED');
expect(notifications.recordForPermission).not.toHaveBeenCalled();
});
it('records FAILED with the loggable message, never the Prisma message', async () => {
const run = await service.run('cleanup', () =>
Promise.reject(
new Prisma.PrismaClientKnownRequestError('title "Max Mustermann"', {
code: 'P2002',
clientVersion: '7.9.1',
meta: { modelName: 'Task' },
}),
),
);
expect(run.status).toBe('FAILED');
const [finish] = prisma.$queryRaw.mock.calls;
expect(finish.slice(1)).toEqual([
'FAILED',
null,
'PrismaClientKnownRequestError P2002 (Task)',
'run-1',
]);
expect(JSON.stringify(errorSpy.mock.calls)).not.toContain('Mustermann');
});
// finishedAt of every run - finished, failed or aborted - comes from
// the database clock, like the comparisons of cleanup.
it.each([
['succeeds', () => Promise.resolve({ deleted: 1 })],
['fails', () => Promise.reject(new Error('boom'))],
])(
'sets finishedAt to now() when the run %s',
async (_case, work: () => Promise<Record<string, number>>) => {
await service.run('cleanup', work);
const [finish] = prisma.$queryRaw.mock.calls;
expect(sqlText(finish)).toContain('"finishedAt" = now()');
expect(sqlText(finish)).toContain('RETURNING *');
expect(finish.slice(1).some((value) => value instanceof Date)).toBe(
false,
);
},
);
it('notifies holders of the job permission when a run fails', async () => {
await service.run('daily-evaluation', () =>
Promise.reject(new Error('boom')),
);
expect(notifications.recordForPermission).toHaveBeenCalledWith(
'automation:update',
{
priority: 'WARNUNG',
message: 'Job „Tägliche Auswertung“ fehlgeschlagen',
relatedType: 'JobRun',
relatedId: 'run-1',
},
tx,
);
});
// The hourly app-backup check would otherwise report one outage every
// hour. The check and the write share a transaction behind a lock per
// job, so two failing runs at once cannot both pass the check.
it('notifies at most once per job and UTC day, decided by the database clock', async () => {
tx.$queryRaw.mockResolvedValue([{ notified: true }]);
await service.run('app-backup', () => Promise.reject(new Error('boom')));
expect(notifications.recordForPermission).not.toHaveBeenCalled();
const lock = tx.$executeRaw.mock.calls[0];
expect(sqlText(lock)).toContain('pg_advisory_xact_lock');
expect(lock.slice(1)).toEqual(['job-failure-notification:app-backup']);
const check = tx.$queryRaw.mock.calls[0];
expect(sqlText(check)).toContain(`date_trunc('day', now())`);
expect(check.slice(1)).toEqual(['JobRun', 'app-backup']);
expect(tx.$executeRaw.mock.invocationCallOrder[0]).toBeLessThan(
tx.$queryRaw.mock.invocationCallOrder[0],
);
});
it('still returns the FAILED run when the notification cannot be written', async () => {
prisma.$transaction.mockRejectedValue(new Error('deadlock detected'));
const run = await service.run('cleanup', () =>
Promise.reject(new Error('boom')),
);
expect(run.status).toBe('FAILED');
expect(JSON.stringify(errorSpy.mock.calls)).toContain(
'deadlock detected',
);
});
});
describe('recent', () => {
it('returns the newest runs, optionally of one job', async () => {
await service.recent(20, 'cleanup');
await service.recent(5);
expect(jobRun.findMany).toHaveBeenNthCalledWith(1, {
where: { job: 'cleanup' },
orderBy: [{ startedAt: 'desc' }, { id: 'desc' }],
take: 20,
});
expect(jobRun.findMany).toHaveBeenNthCalledWith(2, {
where: {},
orderBy: [{ startedAt: 'desc' }, { id: 'desc' }],
take: 5,
});
});
it.each([0, -1, 1.5])('refuses the limit %p', async (limit) => {
await expect(service.recent(limit)).rejects.toThrow(RangeError);
});
});
describe('latestPerJob', () => {
it('returns the newest run of every job, null for a job that never ran', async () => {
const cleanupRun = runRow({ job: 'cleanup', status: 'SUCCEEDED' });
jobRun.findFirst.mockImplementation(
({ where }: { where: { job: string } }) =>
Promise.resolve(where.job === 'cleanup' ? cleanupRun : null),
);
const latest = await service.latestPerJob();
expect(Object.keys(latest)).toEqual([...JOB_NAMES]);
expect(latest.cleanup).toBe(cleanupRun);
expect(latest.rekey).toBeNull();
expect(jobRun.findFirst).toHaveBeenCalledWith({
where: { job: 'cleanup' },
orderBy: [{ startedAt: 'desc' }, { id: 'desc' }],
});
});
});
describe('abortStale', () => {
it('marks runs RUNNING longer than the window as FAILED, by the database clock', async () => {
prisma.$queryRaw.mockResolvedValue([]);
await expect(service.abortStale(6)).resolves.toBe(0);
const [call] = prisma.$queryRaw.mock.calls;
const sql = sqlText(call);
expect(sql).toContain(`"status" = 'RUNNING'`);
expect(sql).toContain('"startedAt" < now()');
expect(call.slice(1)).toEqual([JOB_ABORTED_ERROR, 6]);
expect(call.slice(1).some((value) => value instanceof Date)).toBe(false);
});
it('sends the failure notification once per job of the aborted runs', async () => {
prisma.$queryRaw.mockResolvedValue([
{ id: 'a', job: 'app-backup' },
{ id: 'b', job: 'app-backup' },
{ id: 'c', job: 'task-templates' },
]);
await expect(service.abortStale(6)).resolves.toBe(3);
expect(notifications.recordForPermission).toHaveBeenCalledTimes(2);
expect(notifications.recordForPermission).toHaveBeenCalledWith(
'backup:configure',
expect.objectContaining({
message: 'Job „App-Backup“ fehlgeschlagen',
relatedId: 'a',
}),
tx,
);
expect(notifications.recordForPermission).toHaveBeenCalledWith(
'automation:update',
expect.objectContaining({ relatedId: 'c' }),
tx,
);
});
it('aborts but does not notify for a job name that no longer exists', async () => {
prisma.$queryRaw.mockResolvedValue([{ id: 'x', job: 'retired-job' }]);
await expect(service.abortStale(6)).resolves.toBe(1);
expect(notifications.recordForPermission).not.toHaveBeenCalled();
});
});
});
@@ -0,0 +1,187 @@
import { Injectable, Logger } from '@nestjs/common';
import { JobRun } from '../../generated/prisma';
import { loggableError } from '../common/loggable-error';
import { JobResult } from '../jobs/job';
import {
isJobName,
JOB_FAILURE_PERMISSION,
JOB_LABELS_DE,
JOB_NAMES,
JobName,
} from '../jobs/job-names';
import { NotificationsService } from '../notifications/notifications.service';
import { PrismaService } from '../prisma/prisma.service';
/** JobRun.error of a run whose pod ended before it wrote a result. */
export const JOB_ABORTED_ERROR = 'aborted (pod ended without result)';
/** Notification.relatedType of a job failure; relatedId is the JobRun id. */
const JOB_RUN_RELATED_TYPE = 'JobRun';
const NEWEST_FIRST = [{ startedAt: 'desc' as const }, { id: 'desc' as const }];
type RunOutcome =
| { status: 'SUCCEEDED'; result: JobResult }
| { status: 'FAILED'; error: string };
/**
* Bookkeeping of the CronJob runs: every run is one JobRun row (RUNNING,
* then SUCCEEDED or FAILED), and a failed run notifies the holders of the
* job's permission (JOB_FAILURE_PERMISSION).
*/
@Injectable()
export class JobRunService {
private readonly logger = new Logger(JobRunService.name);
constructor(
private readonly prisma: PrismaService,
private readonly notifications: NotificationsService,
) {}
/**
* Runs `work` as one run of `job`. Resolves with the finished JobRun - a
* failure of `work` is recorded, not rethrown. Throws only if the run
* itself cannot be recorded.
*/
async run(job: JobName, work: () => Promise<JobResult>): Promise<JobRun> {
const started = await this.prisma.jobRun.create({ data: { job } });
let result: JobResult;
try {
result = await work();
} catch (error: unknown) {
return this.fail(job, started.id, error);
}
this.logger.log(`Job "${job}" succeeded: ${JSON.stringify(result)}`);
return this.finish(started.id, { status: 'SUCCEEDED', result });
}
/** The newest runs, of one job or of all. */
async recent(limit: number, job?: JobName): Promise<JobRun[]> {
if (!Number.isInteger(limit) || limit < 1) {
throw new RangeError(`Limit must be a positive integer, got ${limit}`);
}
return this.prisma.jobRun.findMany({
where: job ? { job } : {},
orderBy: NEWEST_FIRST,
take: limit,
});
}
/** The newest run of every job, null for a job that never ran. */
async latestPerJob(): Promise<Record<JobName, JobRun | null>> {
const latest = await Promise.all(
JOB_NAMES.map((job) =>
this.prisma.jobRun.findFirst({ where: { job }, orderBy: NEWEST_FIRST }),
),
);
return Object.fromEntries(
JOB_NAMES.map((job, index) => [job, latest[index]]),
) as Record<JobName, JobRun | null>;
}
/**
* Marks runs that are RUNNING for longer than `staleHours` as FAILED:
* their pod ended without a result (activeDeadlineSeconds, OOM, node
* loss). finishedAt is when this was noticed. Each affected job gets the
* usual failure notification - otherwise a job that is killed every time
* would fail without anyone hearing about it. Compared against now(), the
* database clock. Returns how many runs were aborted.
*/
async abortStale(staleHours: number): Promise<number> {
const aborted = await this.prisma.$queryRaw<{ id: string; job: string }[]>`
UPDATE "JobRun"
SET "status" = 'FAILED', "finishedAt" = now(), "error" = ${JOB_ABORTED_ERROR}
WHERE "status" = 'RUNNING'
AND "startedAt" < now() - (${staleHours} * interval '1 hour')
RETURNING "id", "job"`;
const firstRunPerJob = new Map<JobName, string>();
for (const { id, job } of aborted) {
if (isJobName(job) && !firstRunPerJob.has(job)) {
firstRunPerJob.set(job, id);
}
}
for (const [job, runId] of firstRunPerJob) {
await this.notifyFailure(job, runId);
}
return aborted.length;
}
private async fail(
job: JobName,
runId: string,
error: unknown,
): Promise<JobRun> {
const { message, stack } = loggableError(error);
this.logger.error(`Job "${job}" failed: ${message}`, stack);
const failed = await this.finish(runId, {
status: 'FAILED',
error: message,
});
await this.notifyFailure(job, runId);
return failed;
}
/**
* Ends a run. finishedAt is now(), the database clock - the same clock
* abortStale and every cleanup comparison use.
*/
private async finish(runId: string, outcome: RunOutcome): Promise<JobRun> {
const result =
outcome.status === 'SUCCEEDED' ? JSON.stringify(outcome.result) : null;
const error = outcome.status === 'FAILED' ? outcome.error : null;
const [finished] = await this.prisma.$queryRaw<JobRun[]>`
UPDATE "JobRun"
SET "status" = CAST(${outcome.status} AS "JobRunStatus"),
"finishedAt" = now(),
"result" = CAST(${result} AS jsonb),
"error" = ${error}
WHERE "id" = ${runId}
RETURNING *`;
if (!finished) {
throw new Error(`JobRun ${runId} no longer exists`);
}
return finished;
}
/**
* At most one notification per job and UTC day (database clock): the
* hourly app-backup check would otherwise report the same outage every
* hour. The advisory lock per job serialises check and write, so two
* runs failing at once cannot both pass the check. A notification that
* cannot be written is logged; the run stays FAILED either way.
*/
private async notifyFailure(job: JobName, runId: string): Promise<void> {
try {
await this.prisma.$transaction(async (tx) => {
await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtextextended(${`job-failure-notification:${job}`}, 0))`;
const [{ notified }] = await tx.$queryRaw<{ notified: boolean }[]>`
SELECT EXISTS (
SELECT 1 FROM "Notification" n
JOIN "JobRun" r ON r."id" = n."relatedId"
WHERE n."relatedType" = ${JOB_RUN_RELATED_TYPE}
AND r."job" = ${job}
AND n."createdAt" >= date_trunc('day', now())
) AS "notified"`;
if (notified) {
return;
}
await this.notifications.recordForPermission(
JOB_FAILURE_PERMISSION[job],
{
priority: 'WARNUNG',
message: `Job „${JOB_LABELS_DE[job]}“ fehlgeschlagen`,
relatedType: JOB_RUN_RELATED_TYPE,
relatedId: runId,
},
tx,
);
});
} catch (error: unknown) {
const { message, stack } = loggableError(error);
this.logger.error(
`Failure notification for job "${job}" could not be written: ${message}`,
stack,
);
}
}
}
@@ -0,0 +1,207 @@
import { ConflictException, NotFoundException } from '@nestjs/common';
import { AuditLogService } from '../audit/audit-log.service';
import { PrismaService } from '../prisma/prisma.service';
import { JobRunService } from './job-run.service';
import { SystemQueueService } from './system-queue.service';
type RawCall = [readonly string[], ...unknown[]];
describe('SystemQueueService', () => {
let tx: {
$queryRaw: jest.Mock<Promise<unknown>, RawCall>;
$executeRaw: jest.Mock<Promise<number>, RawCall>;
};
let prisma: {
$queryRaw: jest.Mock<Promise<unknown>, RawCall>;
$transaction: jest.Mock;
domainEvent: { findMany: jest.Mock; findUnique: jest.Mock };
webhookDelivery: { findMany: jest.Mock; findUnique: jest.Mock };
};
let jobRuns: { latestPerJob: jest.Mock };
let audit: { record: jest.Mock };
let service: SystemQueueService;
const sqlText = (call: RawCall): string => call[0].join('?');
beforeEach(() => {
tx = {
$queryRaw: jest.fn<Promise<unknown>, RawCall>().mockResolvedValue([]),
$executeRaw: jest.fn<Promise<number>, RawCall>().mockResolvedValue(1),
};
prisma = {
$queryRaw: jest.fn<Promise<unknown>, RawCall>(),
$transaction: jest.fn((fn: (t: typeof tx) => Promise<unknown>) => fn(tx)),
domainEvent: { findMany: jest.fn(), findUnique: jest.fn() },
webhookDelivery: { findMany: jest.fn(), findUnique: jest.fn() },
};
jobRuns = {
latestPerJob: jest.fn().mockResolvedValue({
'daily-evaluation': null,
'task-templates': null,
cleanup: null,
'app-backup': null,
rekey: null,
}),
};
audit = { record: jest.fn().mockResolvedValue(undefined) };
service = new SystemQueueService(
prisma as unknown as PrismaService,
jobRuns as unknown as JobRunService,
audit as unknown as AuditLogService,
);
});
it('reports the oldest pending age from the database clock', async () => {
prisma.$queryRaw
.mockResolvedValueOnce([
{ pending: 3, processing: 1, failed: 2, oldestPendingAgeSeconds: 42.5 },
])
.mockResolvedValueOnce([{ pending: 4, failed: 1 }]);
const overview = await service.overview();
expect(overview.domainEvents).toEqual({
pending: 3,
processing: 1,
failed: 2,
oldestPendingAgeSeconds: 42.5,
});
expect(overview.webhookDeliveries).toEqual({ pending: 4, failed: 1 });
expect(overview.jobs).toHaveLength(5);
const sql = sqlText(prisma.$queryRaw.mock.calls[0]);
expect(sql).toContain('EXTRACT(EPOCH FROM now() - MIN("occurredAt")');
});
it('reports null age when nothing is pending', async () => {
prisma.$queryRaw
.mockResolvedValueOnce([
{ pending: 0, processing: 0, failed: 0, oldestPendingAgeSeconds: null },
])
.mockResolvedValueOnce([{ pending: 0, failed: 0 }]);
const overview = await service.overview();
expect(overview.domainEvents.oldestPendingAgeSeconds).toBeNull();
});
it('lists failed events without payload, with their failed handlers', async () => {
prisma.domainEvent.findMany.mockResolvedValue([
{
id: 'e1',
type: 'x',
aggregateType: 'A',
aggregateId: 'a1',
occurredAt: new Date(),
attempts: 8,
lastError: 'boom',
deliveries: [{ handler: 'h1', lastError: 'bad' }],
},
]);
const [view] = await service.failedDomainEvents(10);
expect(view.failedHandlers).toEqual([{ handler: 'h1', lastError: 'bad' }]);
const args = (
prisma.domainEvent.findMany.mock.calls as unknown[][]
)[0][0] as {
select: Record<string, unknown>;
take: number;
};
expect(args.select).not.toHaveProperty('payload');
expect(args.take).toBe(10);
});
it('retry resets a FAILED event and its failed deliveries, writes an audit entry and wakes the worker', async () => {
tx.$queryRaw.mockResolvedValueOnce([{ attempts: 8 }]);
await service.retryDomainEvent('e1', 'user-1');
const statements = [
...tx.$queryRaw.mock.calls,
...tx.$executeRaw.mock.calls,
].map(sqlText);
expect(statements[0]).toContain('"availableAt" = now()');
expect(statements[0]).toContain('"attempts" = 0');
expect(statements.some((s) => s.includes('"DomainEventDelivery"'))).toBe(
true,
);
expect(audit.record).toHaveBeenCalledWith(
expect.objectContaining({
entityType: 'DomainEvent',
entityId: 'e1',
action: 'UPDATE',
userId: 'user-1',
}),
tx,
);
expect(
tx.$executeRaw.mock.calls.some((c) =>
c.slice(1).includes('domain_event'),
),
).toBe(true);
});
it('retry of a DONE event is 409', async () => {
tx.$queryRaw.mockResolvedValueOnce([]);
prisma.domainEvent.findUnique.mockResolvedValue({ id: 'e1' });
await expect(service.retryDomainEvent('e1', 'u')).rejects.toBeInstanceOf(
ConflictException,
);
expect(audit.record).not.toHaveBeenCalled();
});
it('retry of an unknown event is 404', async () => {
tx.$queryRaw.mockResolvedValueOnce([]);
prisma.domainEvent.findUnique.mockResolvedValue(null);
await expect(service.retryDomainEvent('nope', 'u')).rejects.toBeInstanceOf(
NotFoundException,
);
});
it('shows only the host of a webhook target', async () => {
prisma.webhookDelivery.findMany.mockResolvedValue([
{
id: 'd1',
subscriptionId: 's1',
event: 'harvest.created',
attempts: 8,
lastStatusCode: 500,
lastError: 'HTTP 500',
createdAt: new Date(),
subscription: {
url: 'https://user:[email protected]:8443/hidden/path?key=abc',
},
},
]);
const [view] = await service.failedWebhookDeliveries(5);
expect(view.targetHost).toBe('hooks.example.com:8443');
expect(JSON.stringify(view)).not.toMatch(/hidden|key=|pw/);
expect(view).not.toHaveProperty('body');
expect(view).not.toHaveProperty('subscription');
});
it('retries a FAILED webhook delivery and wakes the delivery worker', async () => {
tx.$queryRaw.mockResolvedValueOnce([{ attempts: 8 }]);
await service.retryWebhookDelivery('d1', 'user-1');
expect(sqlText(tx.$queryRaw.mock.calls[0])).toContain(
'"availableAt" = now()',
);
expect(
tx.$executeRaw.mock.calls.some((c) =>
c.slice(1).includes('webhook_delivery'),
),
).toBe(true);
expect(audit.record).toHaveBeenCalledWith(
expect.objectContaining({ entityType: 'WebhookDelivery' }),
tx,
);
});
it('retry of a non-FAILED delivery is 409', async () => {
tx.$queryRaw.mockResolvedValueOnce([]);
prisma.webhookDelivery.findUnique.mockResolvedValue({ id: 'd1' });
await expect(
service.retryWebhookDelivery('d1', 'u'),
).rejects.toBeInstanceOf(ConflictException);
});
});
@@ -0,0 +1,291 @@
import {
ConflictException,
Injectable,
NotFoundException,
} from '@nestjs/common';
import { JobRun } from '../../generated/prisma';
import { AuditLogService } from '../audit/audit-log.service';
import { notifyWorker, WAKE_CHANNELS } from '../common/worker-wakeup';
import { JobResult } from '../jobs/job';
import { isJobName, JOB_NAMES, JobName } from '../jobs/job-names';
import { PrismaService } from '../prisma/prisma.service';
import { JobRunService } from './job-run.service';
export interface JobRunView {
id: string;
job: JobName;
status: JobRun['status'];
startedAt: Date;
finishedAt: Date | null;
error: string | null;
result: JobResult | null;
}
export interface SystemOverview {
jobs: { job: JobName; lastRun: JobRunView | null }[];
domainEvents: {
pending: number;
processing: number;
failed: number;
oldestPendingAgeSeconds: number | null;
};
webhookDeliveries: { pending: number; failed: number };
}
export interface FailedDomainEventView {
id: string;
type: string;
aggregateType: string;
aggregateId: string;
occurredAt: Date;
attempts: number;
lastError: string | null;
failedHandlers: { handler: string; lastError: string | null }[];
}
export interface FailedWebhookDeliveryView {
id: string;
subscriptionId: string;
targetHost: string;
event: string;
attempts: number;
lastStatusCode: number | null;
lastError: string | null;
createdAt: Date;
}
/** Shown when a stored webhook URL cannot be parsed. */
const UNKNOWN_HOST = 'unknown';
interface DomainEventCounts {
pending: number;
processing: number;
failed: number;
oldestPendingAgeSeconds: number | null;
}
interface WebhookCounts {
pending: number;
failed: number;
}
/** Host (and port) of a webhook URL - never path, query or credentials. */
function hostOf(rawUrl: string): string {
try {
return new URL(rawUrl).host || UNKNOWN_HOST;
} catch {
return UNKNOWN_HOST;
}
}
/** The row was not retryable: 404 if it does not exist, else 409. */
function retryRejection(
existing: { id: string } | null,
label: string,
): NotFoundException | ConflictException {
return existing
? new ConflictException(`${label} is not FAILED`)
: new NotFoundException(`${label} not found`);
}
/** JobRun for the API: `job` narrowed, `result` as the stored JSON object. */
export function toJobRunView(run: JobRun): JobRunView {
return {
id: run.id,
job: run.job as JobName,
status: run.status,
startedAt: run.startedAt,
finishedAt: run.finishedAt,
error: run.error,
result: (run.result ?? null) as JobResult | null,
};
}
/**
* Read model and retry actions of the System page: the CronJob runs, the
* state of both outboxes and the re-queueing of FAILED entries. Views never
* carry payloads, webhook bodies, secrets or full URLs; `lastError` is
* shown as stored (written via loggableError).
*/
@Injectable()
export class SystemQueueService {
constructor(
private readonly prisma: PrismaService,
private readonly jobRuns: JobRunService,
private readonly audit: AuditLogService,
) {}
async overview(): Promise<SystemOverview> {
const [latest, [events], [webhooks]] = await Promise.all([
this.jobRuns.latestPerJob(),
// The age is computed by the database: the API pod's clock may differ.
this.prisma.$queryRaw<DomainEventCounts[]>`
SELECT
count(*) FILTER (WHERE "status" = 'PENDING')::int AS "pending",
count(*) FILTER (WHERE "status" = 'PROCESSING')::int AS "processing",
count(*) FILTER (WHERE "status" = 'FAILED')::int AS "failed",
EXTRACT(EPOCH FROM now() - MIN("occurredAt") FILTER (WHERE "status" = 'PENDING'))::float8 AS "oldestPendingAgeSeconds"
FROM "DomainEvent"`,
this.prisma.$queryRaw<WebhookCounts[]>`
SELECT
count(*) FILTER (WHERE "status" = 'PENDING')::int AS "pending",
count(*) FILTER (WHERE "status" = 'FAILED')::int AS "failed"
FROM "WebhookDelivery"`,
]);
return {
jobs: JOB_NAMES.map((job) => ({
job,
lastRun: latest[job] ? toJobRunView(latest[job]) : null,
})),
domainEvents: {
pending: events.pending,
processing: events.processing,
failed: events.failed,
oldestPendingAgeSeconds: events.oldestPendingAgeSeconds,
},
webhookDeliveries: { pending: webhooks.pending, failed: webhooks.failed },
};
}
/** The newest runs, of one job or of all (limit validated by the DTO). */
async jobRunViews(limit: number, job?: JobName): Promise<JobRunView[]> {
const runs = await this.jobRuns.recent(limit, job);
return runs.filter((run) => isJobName(run.job)).map(toJobRunView);
}
async failedDomainEvents(limit: number): Promise<FailedDomainEventView[]> {
const events = await this.prisma.domainEvent.findMany({
where: { status: 'FAILED' },
orderBy: [{ occurredAt: 'desc' }, { id: 'desc' }],
take: limit,
// Never the payload: it holds the full record the event was about.
select: {
id: true,
type: true,
aggregateType: true,
aggregateId: true,
occurredAt: true,
attempts: true,
lastError: true,
deliveries: {
where: { status: 'FAILED' },
orderBy: { handler: 'asc' },
select: { handler: true, lastError: true },
},
},
});
return events.map(({ deliveries, ...event }) => ({
...event,
failedHandlers: deliveries,
}));
}
/**
* Re-queues a FAILED event and its FAILED handler deliveries (DONE
* deliveries stay DONE and are skipped, so no effect repeats). Resets
* attempts and availableAt in SQL (database clock), audits the change in
* the same transaction and wakes the worker. 404 unknown, 409 not FAILED.
*/
async retryDomainEvent(id: string, actingUserId: string): Promise<void> {
await this.prisma.$transaction(async (tx) => {
const [before] = await tx.$queryRaw<{ attempts: number }[]>`
WITH old AS (
SELECT "id", "attempts" FROM "DomainEvent"
WHERE "id" = ${id} AND "status" = 'FAILED'
FOR UPDATE
)
UPDATE "DomainEvent" e
SET "status" = 'PENDING', "attempts" = 0, "availableAt" = now(),
"claimedAt" = NULL, "processedAt" = NULL, "lastError" = NULL
FROM old
WHERE e."id" = old."id"
RETURNING old."attempts" AS "attempts"`;
if (!before) {
const existing = await this.prisma.domainEvent.findUnique({
where: { id },
select: { id: true },
});
throw retryRejection(existing, 'Domain event');
}
await tx.$executeRaw`
UPDATE "DomainEventDelivery"
SET "status" = 'PENDING', "attempts" = 0, "lastError" = NULL,
"updatedAt" = now()
WHERE "eventId" = ${id} AND "status" = 'FAILED'`;
await this.audit.record(
{
entityType: 'DomainEvent',
entityId: id,
action: 'UPDATE',
userId: actingUserId,
beforeData: { status: 'FAILED', attempts: before.attempts },
afterData: { status: 'PENDING', attempts: 0 },
},
tx,
);
await notifyWorker(tx, WAKE_CHANNELS.domainEvents);
});
}
async failedWebhookDeliveries(
limit: number,
): Promise<FailedWebhookDeliveryView[]> {
const deliveries = await this.prisma.webhookDelivery.findMany({
where: { status: 'FAILED' },
orderBy: [{ createdAt: 'desc' }, { id: 'desc' }],
take: limit,
// Never the body; of the subscription only the URL, reduced to its host.
select: {
id: true,
subscriptionId: true,
event: true,
attempts: true,
lastStatusCode: true,
lastError: true,
createdAt: true,
subscription: { select: { url: true } },
},
});
return deliveries.map(({ subscription, ...delivery }) => ({
...delivery,
targetHost: hostOf(subscription.url),
}));
}
/** Re-queues a FAILED delivery; same rules as retryDomainEvent. */
async retryWebhookDelivery(id: string, actingUserId: string): Promise<void> {
await this.prisma.$transaction(async (tx) => {
const [before] = await tx.$queryRaw<{ attempts: number }[]>`
WITH old AS (
SELECT "id", "attempts" FROM "WebhookDelivery"
WHERE "id" = ${id} AND "status" = 'FAILED'
FOR UPDATE
)
UPDATE "WebhookDelivery" d
SET "status" = 'PENDING', "attempts" = 0, "availableAt" = now(),
"claimedAt" = NULL, "lastStatusCode" = NULL, "lastError" = NULL
FROM old
WHERE d."id" = old."id"
RETURNING old."attempts" AS "attempts"`;
if (!before) {
const existing = await this.prisma.webhookDelivery.findUnique({
where: { id },
select: { id: true },
});
throw retryRejection(existing, 'Webhook delivery');
}
await this.audit.record(
{
entityType: 'WebhookDelivery',
entityId: id,
action: 'UPDATE',
userId: actingUserId,
beforeData: { status: 'FAILED', attempts: before.attempts },
afterData: { status: 'PENDING', attempts: 0 },
},
tx,
);
await notifyWorker(tx, WAKE_CHANNELS.webhookDeliveries);
});
}
}
@@ -0,0 +1,87 @@
import {
Controller,
Get,
HttpCode,
HttpStatus,
Param,
Post,
Query,
} from '@nestjs/common';
import { ApiBearerAuth, ApiTags } from '@nestjs/swagger';
import { CurrentUser } from '../auth/decorators/current-user.decorator';
import { RequirePermission } from '../auth/decorators/require-permission.decorator';
import type { AuthenticatedUser } from '../auth/types/authenticated-user';
import {
JobRunListQueryDto,
SYSTEM_LIST_DEFAULT_LIMIT,
SystemListQueryDto,
} from './dto/system-list-query.dto';
import {
FailedDomainEventView,
FailedWebhookDeliveryView,
JobRunView,
SystemOverview,
SystemQueueService,
} from './system-queue.service';
@ApiTags('system')
@ApiBearerAuth()
@Controller('system')
export class SystemController {
constructor(private readonly queue: SystemQueueService) {}
@RequirePermission('system:read')
@Get('overview')
overview(): Promise<SystemOverview> {
return this.queue.overview();
}
@RequirePermission('system:read')
@Get('job-runs')
jobRuns(@Query() query: JobRunListQueryDto): Promise<JobRunView[]> {
return this.queue.jobRunViews(
query.limit ?? SYSTEM_LIST_DEFAULT_LIMIT,
query.job,
);
}
@RequirePermission('system:read')
@Get('domain-events/failed')
failedDomainEvents(
@Query() query: SystemListQueryDto,
): Promise<FailedDomainEventView[]> {
return this.queue.failedDomainEvents(
query.limit ?? SYSTEM_LIST_DEFAULT_LIMIT,
);
}
@RequirePermission('system:manage')
@Post('domain-events/:id/retry')
@HttpCode(HttpStatus.NO_CONTENT)
retryDomainEvent(
@Param('id') id: string,
@CurrentUser() actor: AuthenticatedUser,
): Promise<void> {
return this.queue.retryDomainEvent(id, actor.id);
}
@RequirePermission('system:read')
@Get('webhook-deliveries/failed')
failedWebhookDeliveries(
@Query() query: SystemListQueryDto,
): Promise<FailedWebhookDeliveryView[]> {
return this.queue.failedWebhookDeliveries(
query.limit ?? SYSTEM_LIST_DEFAULT_LIMIT,
);
}
@RequirePermission('system:manage')
@Post('webhook-deliveries/:id/retry')
@HttpCode(HttpStatus.NO_CONTENT)
retryWebhookDelivery(
@Param('id') id: string,
@CurrentUser() actor: AuthenticatedUser,
): Promise<void> {
return this.queue.retryWebhookDelivery(id, actor.id);
}
}
@@ -0,0 +1,17 @@
import { Module } from '@nestjs/common';
import { NotificationsModule } from '../notifications/notifications.module';
import { JobRunService } from './job-run.service';
import { SystemController } from './system.controller';
import { SystemQueueService } from './system-queue.service';
/**
* Operations of the platform itself: the CronJob runs (JobRun). Imported by
* the API (System page) and by JobsModule (dist/cli.js).
*/
@Module({
imports: [NotificationsModule],
controllers: [SystemController],
providers: [JobRunService, SystemQueueService],
exports: [JobRunService],
})
export class SystemModule {}