feat: platform backbone and field capture complete (checkpoint)
Platform backbone (plan 2026-10-13-plattform-rueckgrat), all tasks reviewed: - webhooks with delivery table, retries and stable delivery id - calendar handlers; a finished task removes its Outlook event - tasks, automations, inspections, varroa, treatments, feedings, harvests, colonies, wintering, stock and bottling record domain events in the same transaction; legacy post-commit calls removed - row locks through one helper (lockRowForUpdate, lockRowsForShare, lockRowsForUpdate) - job framework with JobRun, cli exit codes, cleanup, task templates, daily evaluation, hourly app-backup and the rekey job - system API and page "Verwaltung > System" with retry - @nestjs/schedule removed, guard tests against in-process schedulers Field capture T3 (plan 2026-10-13-t3-erfassung-am-stand), all tasks reviewed: - inspection photo offline without metadata - bulk capture for treatments, feedings and varroa (backend + frontend) - offline stand mode with service worker, offline pages and packs - offline session, cold start and logout on shared devices Docs: ARCHITECTURE.md assumptions and sections updated, TASKS.md. Gates: backend lint, typecheck, 1770 unit, 420 e2e, build, no schema drift; frontend lint, typecheck, 913 tests, build. Not rolled out. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
Nick SzaunigandClaude Opus 5.5 committed
1 parent
c01a38835a
commit
cdd30cf3ee
288 files changed
+39165
-2853
No files matched your search
@@ -0,0 +1,21 @@
|
||||
import { ApiPropertyOptional } from '@nestjs/swagger';
|
||||
import { Type } from 'class-transformer';
|
||||
import { IsInt, IsOptional, Max, Min } from 'class-validator';
|
||||
|
||||
export const WEBHOOK_DELIVERIES_DEFAULT_LIMIT = 50;
|
||||
export const WEBHOOK_DELIVERIES_MAX_LIMIT = 100;
|
||||
|
||||
export class ListWebhookDeliveriesDto {
|
||||
@ApiPropertyOptional({
|
||||
minimum: 1,
|
||||
maximum: WEBHOOK_DELIVERIES_MAX_LIMIT,
|
||||
default: WEBHOOK_DELIVERIES_DEFAULT_LIMIT,
|
||||
description: 'Anzahl der neuesten Zustellungen',
|
||||
})
|
||||
@IsOptional()
|
||||
@Type(() => Number)
|
||||
@IsInt()
|
||||
@Min(1)
|
||||
@Max(WEBHOOK_DELIVERIES_MAX_LIMIT)
|
||||
limit?: number;
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
import * as net from 'node:net';
|
||||
import {
|
||||
postWebhook,
|
||||
WEBHOOK_DEADLINE_MS,
|
||||
WEBHOOK_TIMEOUT_MS,
|
||||
} from './webhook-delivery';
|
||||
|
||||
// Real deliveries refuse loopback; these tests need a local receiver. The
|
||||
// guard itself is covered by webhook-url-guard.spec.ts and webhooks.e2e.
|
||||
jest.mock('./webhook-url-guard', () => ({
|
||||
...jest.requireActual<object>('./webhook-url-guard'),
|
||||
isBlockedIp: () => false,
|
||||
}));
|
||||
|
||||
const DEADLINE_MESSAGE = 'Webhook receiver did not finish answering in time';
|
||||
/** Well below the 5 s idle timeout: only the deadline can end the exchange. */
|
||||
const TRICKLE_EVERY_MS = 50;
|
||||
const TEST_DEADLINE_MS = 300;
|
||||
|
||||
interface Receiver {
|
||||
url: string;
|
||||
/** Resolves once the receiver's side of the connection is closed. */
|
||||
closed: Promise<void>;
|
||||
stop: () => Promise<void>;
|
||||
}
|
||||
|
||||
/** A raw TCP receiver: `answer` writes whatever bytes it wants. */
|
||||
async function startReceiver(
|
||||
answer: (socket: net.Socket) => void,
|
||||
): Promise<Receiver> {
|
||||
let markClosed!: () => void;
|
||||
const closed = new Promise<void>((resolve) => {
|
||||
markClosed = resolve;
|
||||
});
|
||||
const sockets = new Set<net.Socket>();
|
||||
const server = net.createServer((socket) => {
|
||||
sockets.add(socket);
|
||||
socket.on('error', () => undefined);
|
||||
socket.on('close', () => {
|
||||
sockets.delete(socket);
|
||||
markClosed();
|
||||
});
|
||||
socket.once('data', () => answer(socket));
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
|
||||
const { port } = server.address() as net.AddressInfo;
|
||||
return {
|
||||
url: `http://127.0.0.1:${port}/hook`,
|
||||
closed,
|
||||
stop: async () => {
|
||||
sockets.forEach((socket) => socket.destroy());
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Writes `chunk` every TRICKLE_EVERY_MS until the connection closes. */
|
||||
function trickle(socket: net.Socket, chunk: string): void {
|
||||
const timer = setInterval(() => socket.write(chunk), TRICKLE_EVERY_MS);
|
||||
socket.on('close', () => clearInterval(timer));
|
||||
}
|
||||
|
||||
describe('postWebhook', () => {
|
||||
let receiver: Receiver | undefined;
|
||||
|
||||
afterEach(async () => {
|
||||
jest.restoreAllMocks();
|
||||
await receiver?.stop();
|
||||
receiver = undefined;
|
||||
});
|
||||
|
||||
it('pins the idle timeout and the deadline of a whole exchange', () => {
|
||||
expect(WEBHOOK_TIMEOUT_MS).toBe(5_000);
|
||||
expect(WEBHOOK_DEADLINE_MS).toBe(10_000);
|
||||
});
|
||||
|
||||
it('aborts the request and closes the socket when the headers are not complete by the deadline', async () => {
|
||||
receiver = await startReceiver((socket) => {
|
||||
socket.write('HTTP/1.1 200 OK\r\n');
|
||||
trickle(socket, 'X-Slow: 1\r\n');
|
||||
});
|
||||
const startedAt = performance.now();
|
||||
|
||||
await expect(
|
||||
postWebhook(receiver.url, {}, '{}', TEST_DEADLINE_MS),
|
||||
).rejects.toThrow(DEADLINE_MESSAGE);
|
||||
await receiver.closed;
|
||||
|
||||
expect(performance.now() - startedAt).toBeLessThan(2_000);
|
||||
}, 3_000);
|
||||
|
||||
it('cuts an answer body that is still trickling at the deadline', async () => {
|
||||
receiver = await startReceiver((socket) => {
|
||||
socket.write('HTTP/1.1 200 OK\r\nContent-Length: 100000\r\n\r\n');
|
||||
trickle(socket, 'x');
|
||||
});
|
||||
const startedAt = performance.now();
|
||||
|
||||
// The status counts as soon as it arrives; the body is never read.
|
||||
await expect(
|
||||
postWebhook(receiver.url, {}, '{}', TEST_DEADLINE_MS),
|
||||
).resolves.toEqual({ status: 200 });
|
||||
await receiver.closed;
|
||||
|
||||
expect(performance.now() - startedAt).toBeLessThan(2_000);
|
||||
}, 3_000);
|
||||
|
||||
it('answers a prompt receiver and clears its deadline, 10 s by default', async () => {
|
||||
const setTimeoutSpy = jest.spyOn(global, 'setTimeout');
|
||||
const clearTimeoutSpy = jest.spyOn(global, 'clearTimeout');
|
||||
receiver = await startReceiver((socket) => {
|
||||
socket.end('HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\n');
|
||||
});
|
||||
|
||||
await expect(postWebhook(receiver.url, {}, '{}')).resolves.toEqual({
|
||||
status: 204,
|
||||
});
|
||||
await receiver.closed;
|
||||
|
||||
const deadlineCall = setTimeoutSpy.mock.calls.findIndex(
|
||||
([, ms]) => ms === WEBHOOK_DEADLINE_MS,
|
||||
);
|
||||
expect(deadlineCall).toBeGreaterThanOrEqual(0);
|
||||
const deadlineTimer = setTimeoutSpy.mock.results[deadlineCall]
|
||||
.value as NodeJS.Timeout;
|
||||
// Cleared once the request closes - a finished exchange is never cut.
|
||||
const deadlineCleared = () =>
|
||||
clearTimeoutSpy.mock.calls.some(([timer]) => timer === deadlineTimer);
|
||||
for (let i = 0; i < 50 && !deadlineCleared(); i++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
expect(deadlineCleared()).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -5,21 +5,30 @@ import * as net from 'node:net';
|
||||
|
||||
/** A receiver gets this long to answer; a hanging one must not pile up. */
|
||||
export const WEBHOOK_TIMEOUT_MS = 5_000;
|
||||
/**
|
||||
* Upper bound for a whole exchange, answer included. WEBHOOK_TIMEOUT_MS is
|
||||
* an idle timeout: a receiver that trickles its answer byte by byte would
|
||||
* otherwise keep the connection - and a worker slot - open indefinitely.
|
||||
*/
|
||||
export const WEBHOOK_DEADLINE_MS = 10_000;
|
||||
|
||||
export interface WebhookResponse {
|
||||
status: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* POSTs a webhook without following redirects, with a hard timeout, and
|
||||
* through guardedLookup, so the connection can only ever go to a public
|
||||
* address - whatever the host name resolves to by then. The response body
|
||||
* is discarded unread.
|
||||
* POSTs a webhook without following redirects, with an idle timeout and a
|
||||
* deadline, and through guardedLookup, so the connection can only ever go
|
||||
* to a public address - whatever the host name resolves to by then. The
|
||||
* status counts as soon as it arrives; the response body is discarded
|
||||
* unread. At the deadline the request is destroyed, and with it the socket
|
||||
* and an answer still in transfer - before the status, the promise rejects.
|
||||
*/
|
||||
export function postWebhook(
|
||||
rawUrl: string,
|
||||
headers: Record<string, string>,
|
||||
body: string,
|
||||
deadlineMs = WEBHOOK_DEADLINE_MS,
|
||||
): Promise<WebhookResponse> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const url = new URL(rawUrl);
|
||||
@@ -45,6 +54,15 @@ export function postWebhook(
|
||||
resolve({ status: response.statusCode ?? 0 });
|
||||
},
|
||||
);
|
||||
const deadline = setTimeout(
|
||||
() =>
|
||||
request.destroy(
|
||||
new Error('Webhook receiver did not finish answering in time'),
|
||||
),
|
||||
deadlineMs,
|
||||
);
|
||||
// 'close' ends every exchange - answered, failed or destroyed.
|
||||
request.on('close', () => clearTimeout(deadline));
|
||||
request.on('timeout', () =>
|
||||
request.destroy(new Error('Webhook receiver did not answer in time')),
|
||||
);
|
||||
|
||||
@@ -0,0 +1,561 @@
|
||||
import * as crypto from 'node:crypto';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { Prisma } from '../../generated/prisma';
|
||||
import { MAX_ATTEMPTS } from '../common/retry-policy';
|
||||
import { DRAIN_MAX_ROUNDS } from '../domain-events/domain-event-dispatcher';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { postWebhook, WebhookResponse } from './webhook-delivery';
|
||||
import {
|
||||
WEBHOOK_DELIVERY_BATCH_SIZE,
|
||||
WEBHOOK_DELIVERY_CONCURRENCY,
|
||||
WebhookDeliveryWorker,
|
||||
} from './webhook-delivery.worker';
|
||||
import { sealWebhookSecret } from './webhook-secret';
|
||||
|
||||
jest.mock('./webhook-delivery', () => ({
|
||||
...jest.requireActual<object>('./webhook-delivery'),
|
||||
postWebhook: jest.fn(),
|
||||
}));
|
||||
|
||||
const postWebhookMock = postWebhook as jest.MockedFunction<typeof postWebhook>;
|
||||
|
||||
const SECRET = 'topsecret-signing-key';
|
||||
const CLAIMED_AT = new Date('2026-10-13T08:00:01.234Z');
|
||||
const BODY = JSON.stringify({
|
||||
event: 'colony.created',
|
||||
data: { id: 'c1', notes: 'private note' },
|
||||
});
|
||||
|
||||
/** A claimed WebhookDelivery row, as RETURNING * yields it. */
|
||||
interface ClaimedRow {
|
||||
id: string;
|
||||
subscriptionId: string;
|
||||
event: string;
|
||||
body: string;
|
||||
status: 'PROCESSING';
|
||||
attempts: number;
|
||||
claimedAt: Date;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
interface SubscriptionRow {
|
||||
url: string;
|
||||
secret: string;
|
||||
active: boolean;
|
||||
}
|
||||
|
||||
interface StatusCodeUpdate {
|
||||
where: { id: string; status: string; claimedAt: Date };
|
||||
data: { lastStatusCode: number | null };
|
||||
}
|
||||
|
||||
function row(overrides: Partial<ClaimedRow> = {}): ClaimedRow {
|
||||
return {
|
||||
id: 'd1',
|
||||
subscriptionId: 'wh-1',
|
||||
event: 'colony.created',
|
||||
body: BODY,
|
||||
status: 'PROCESSING',
|
||||
attempts: 0,
|
||||
claimedAt: CLAIMED_AT,
|
||||
createdAt: new Date('2026-10-13T08:00:00.000Z'),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function text(query: Prisma.Sql): string {
|
||||
return query.sql.replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function deferred<T>(): {
|
||||
promise: Promise<T>;
|
||||
resolve: (value: T) => void;
|
||||
} {
|
||||
let resolve!: (value: T) => void;
|
||||
const promise = new Promise<T>((r) => {
|
||||
resolve = r;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
const flush = () => new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
/**
|
||||
* Stands in for Postgres: a queue of claim results, the subscriptions, and
|
||||
* every statement the worker sent.
|
||||
*/
|
||||
class FakeDb {
|
||||
readonly claims: ClaimedRow[][] = [];
|
||||
/** Rows the stale release returns, one list per call. */
|
||||
readonly releases: { id: string; status: 'PENDING' | 'FAILED' }[][] = [];
|
||||
readonly subscriptions = new Map<string, SubscriptionRow>();
|
||||
|
||||
readonly tx = {
|
||||
webhookDelivery: {
|
||||
updateMany: jest.fn().mockResolvedValue({ count: 1 }),
|
||||
},
|
||||
$executeRaw: jest.fn().mockResolvedValue(1),
|
||||
};
|
||||
|
||||
readonly prisma = {
|
||||
$queryRaw: jest.fn((query: Prisma.Sql) =>
|
||||
Promise.resolve(
|
||||
(query.sql.includes('SKIP LOCKED')
|
||||
? this.claims.shift()
|
||||
: this.releases.shift()) ?? [],
|
||||
),
|
||||
),
|
||||
$transaction: jest.fn((fn: (tx: unknown) => Promise<unknown>) =>
|
||||
fn(this.tx),
|
||||
),
|
||||
webhookSubscription: {
|
||||
findUnique: jest.fn(({ where }: { where: { id: string } }) =>
|
||||
Promise.resolve(this.subscriptions.get(where.id) ?? null),
|
||||
),
|
||||
},
|
||||
};
|
||||
|
||||
claimQuery(call = 0): Prisma.Sql {
|
||||
return (this.prisma.$queryRaw.mock.calls as unknown as [Prisma.Sql][])[
|
||||
call
|
||||
][0];
|
||||
}
|
||||
|
||||
/** Settle statements (outbox-claims), in the order they were sent. */
|
||||
settles(): Prisma.Sql[] {
|
||||
return (this.tx.$executeRaw.mock.calls as [Prisma.Sql][]).map(
|
||||
([query]) => query,
|
||||
);
|
||||
}
|
||||
|
||||
statusCodeUpdates(): StatusCodeUpdate[] {
|
||||
return (
|
||||
this.tx.webhookDelivery.updateMany.mock.calls as [StatusCodeUpdate][]
|
||||
).map(([args]) => args);
|
||||
}
|
||||
}
|
||||
|
||||
describe('WebhookDeliveryWorker', () => {
|
||||
let db: FakeDb;
|
||||
let worker: WebhookDeliveryWorker;
|
||||
let logWarn: jest.SpyInstance;
|
||||
let logError: jest.SpyInstance;
|
||||
|
||||
beforeEach(() => {
|
||||
process.env.JWT_ACCESS_SECRET = 'unit-test-jwt-access-secret-0123456789';
|
||||
db = new FakeDb();
|
||||
db.subscriptions.set('wh-1', {
|
||||
url: 'https://receiver.example/hook',
|
||||
secret: sealWebhookSecret(SECRET),
|
||||
active: true,
|
||||
});
|
||||
postWebhookMock.mockReset();
|
||||
postWebhookMock.mockResolvedValue({ status: 200 });
|
||||
worker = new WebhookDeliveryWorker(db.prisma as unknown as PrismaService);
|
||||
logWarn = jest
|
||||
.spyOn(Logger.prototype, 'warn')
|
||||
.mockImplementation(() => undefined);
|
||||
logError = jest
|
||||
.spyOn(Logger.prototype, 'error')
|
||||
.mockImplementation(() => undefined);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
function logLines(): string {
|
||||
const calls = [
|
||||
...(logWarn.mock.calls as unknown[][]),
|
||||
...(logError.mock.calls as unknown[][]),
|
||||
];
|
||||
return calls
|
||||
.flat()
|
||||
.map((part) => String(part))
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
it('pins batch size and concurrency', () => {
|
||||
expect(WEBHOOK_DELIVERY_BATCH_SIZE).toBe(50);
|
||||
expect(WEBHOOK_DELIVERY_CONCURRENCY).toBe(5);
|
||||
});
|
||||
|
||||
it('claims at most 50 due deliveries, oldest first, skipping locked rows, by the database clock', async () => {
|
||||
await worker.runOnce();
|
||||
|
||||
const claim = db.claimQuery();
|
||||
expect(text(claim)).toContain(
|
||||
`UPDATE "WebhookDelivery" SET "status" = 'PROCESSING', "claimedAt" = now()`,
|
||||
);
|
||||
expect(text(claim)).toContain(
|
||||
`WHERE "status" = 'PENDING' AND "availableAt" <= now()`,
|
||||
);
|
||||
expect(text(claim)).toContain(
|
||||
'ORDER BY "createdAt" LIMIT ? FOR UPDATE SKIP LOCKED',
|
||||
);
|
||||
expect(claim.values).toEqual([50]);
|
||||
});
|
||||
|
||||
it('restricts the claim to the given subscriptions', async () => {
|
||||
await worker.runOnce({ subscriptionIds: ['wh-1', 'wh-2'] });
|
||||
|
||||
const claim = db.claimQuery();
|
||||
expect(text(claim)).toContain(
|
||||
`"availableAt" <= now() AND "subscriptionId" = ANY(?) ORDER BY`,
|
||||
);
|
||||
expect(claim.values).toEqual([['wh-1', 'wh-2'], 50]);
|
||||
});
|
||||
|
||||
it('signs "<t>.<delivery id>.<body>" and sends the delivery id as X-ApiCore-Delivery', async () => {
|
||||
db.claims.push([row()]);
|
||||
|
||||
await expect(worker.runOnce()).resolves.toBe(1);
|
||||
|
||||
expect(postWebhookMock).toHaveBeenCalledTimes(1);
|
||||
const [url, headers, body] = postWebhookMock.mock.calls[0];
|
||||
expect(url).toBe('https://receiver.example/hook');
|
||||
// The body stored at enqueue time, byte for byte.
|
||||
expect(body).toBe(BODY);
|
||||
expect(headers).toEqual({
|
||||
'Content-Type': 'application/json',
|
||||
'User-Agent': 'ApiCore-Webhooks/1',
|
||||
'X-ApiCore-Event': 'colony.created',
|
||||
'X-ApiCore-Delivery': 'd1',
|
||||
'X-ApiCore-Timestamp': expect.stringMatching(/^\d+$/) as unknown,
|
||||
'X-ApiCore-Signature': expect.stringMatching(
|
||||
/^sha256=[0-9a-f]{64}$/,
|
||||
) as unknown,
|
||||
});
|
||||
const expected = crypto
|
||||
.createHmac('sha256', SECRET)
|
||||
.update(`${headers['X-ApiCore-Timestamp']}.d1.${BODY}`)
|
||||
.digest('hex');
|
||||
expect(headers['X-ApiCore-Signature']).toBe(`sha256=${expected}`);
|
||||
});
|
||||
|
||||
it.each([200, 201, 204, 299])(
|
||||
'counts %i as delivered and stores the status code',
|
||||
async (status) => {
|
||||
postWebhookMock.mockResolvedValueOnce({ status });
|
||||
db.claims.push([row()]);
|
||||
|
||||
await worker.runOnce();
|
||||
|
||||
expect(db.statusCodeUpdates()).toEqual([
|
||||
{
|
||||
where: { id: 'd1', status: 'PROCESSING', claimedAt: CLAIMED_AT },
|
||||
data: { lastStatusCode: status },
|
||||
},
|
||||
]);
|
||||
const [settle] = db.settles();
|
||||
expect(text(settle)).toContain('"deliveredAt" = now()');
|
||||
expect(text(settle)).toContain(
|
||||
`WHERE "id" = ? AND "status" = 'PROCESSING' AND "claimedAt" = ?`,
|
||||
);
|
||||
expect(settle.values).toEqual(['DELIVERED', 'd1', CLAIMED_AT]);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([301, 302, 400, 404, 429, 500, 503])(
|
||||
'counts only 2xx as delivered: %i is a failed attempt and its status code is stored',
|
||||
async (status) => {
|
||||
postWebhookMock.mockResolvedValueOnce({ status });
|
||||
db.claims.push([row()]);
|
||||
|
||||
await worker.runOnce();
|
||||
|
||||
expect(db.statusCodeUpdates()[0].data).toEqual({
|
||||
lastStatusCode: status,
|
||||
});
|
||||
const [settle] = db.settles();
|
||||
expect(text(settle)).not.toContain('"deliveredAt"');
|
||||
expect(settle.values).toContain('PENDING');
|
||||
expect(settle.values).toContain(`Receiver answered HTTP ${status}`);
|
||||
},
|
||||
);
|
||||
|
||||
it('schedules a retry 30 s ahead after a 500 and keeps the same id', async () => {
|
||||
postWebhookMock
|
||||
.mockResolvedValueOnce({ status: 500 })
|
||||
.mockResolvedValueOnce({ status: 200 });
|
||||
db.claims.push([row({ attempts: 0 })]);
|
||||
|
||||
await worker.runOnce();
|
||||
|
||||
const [retry] = db.settles();
|
||||
expect(text(retry)).toContain(`"attempts" = "attempts" + 1`);
|
||||
expect(text(retry)).toContain(
|
||||
`"availableAt" = now() + (? * interval '1 millisecond')`,
|
||||
);
|
||||
expect(text(retry)).toContain(
|
||||
`WHERE "id" = ? AND "status" = 'PROCESSING' AND "claimedAt" = ?`,
|
||||
);
|
||||
expect(retry.values).toEqual([
|
||||
'PENDING',
|
||||
'Receiver answered HTTP 500',
|
||||
30_000,
|
||||
'd1',
|
||||
CLAIMED_AT,
|
||||
]);
|
||||
|
||||
// The retry is the same row, claimed again: same id, same body.
|
||||
const reclaimedAt = new Date('2026-10-13T08:00:31.500Z');
|
||||
db.claims.push([row({ attempts: 1, claimedAt: reclaimedAt })]);
|
||||
await worker.runOnce();
|
||||
|
||||
const [first, second] = postWebhookMock.mock.calls;
|
||||
expect(second[1]['X-ApiCore-Delivery']).toBe(
|
||||
first[1]['X-ApiCore-Delivery'],
|
||||
);
|
||||
expect(second[2]).toBe(first[2]);
|
||||
expect(db.settles()[1].values).toEqual(['DELIVERED', 'd1', reclaimedAt]);
|
||||
});
|
||||
|
||||
it('marks FAILED after the 8th attempt', async () => {
|
||||
postWebhookMock.mockResolvedValueOnce({ status: 500 });
|
||||
db.claims.push([row({ attempts: MAX_ATTEMPTS - 1 })]);
|
||||
|
||||
await worker.runOnce();
|
||||
|
||||
const [settle] = db.settles();
|
||||
expect(settle.values).toEqual([
|
||||
'FAILED',
|
||||
'Receiver answered HTTP 500',
|
||||
'd1',
|
||||
CLAIMED_AT,
|
||||
]);
|
||||
expect(text(settle)).not.toContain('"availableAt"');
|
||||
expect(logError).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Webhook delivery d1 (colony.created) FAILED'),
|
||||
);
|
||||
});
|
||||
|
||||
it('fails a delivery for an inactive subscription without sending ("subscription inactive")', async () => {
|
||||
db.subscriptions.set('wh-1', {
|
||||
...db.subscriptions.get('wh-1')!,
|
||||
active: false,
|
||||
});
|
||||
db.claims.push([row()]);
|
||||
|
||||
await worker.runOnce();
|
||||
|
||||
expect(postWebhookMock).not.toHaveBeenCalled();
|
||||
expect(db.statusCodeUpdates()[0].data).toEqual({ lastStatusCode: null });
|
||||
const [settle] = db.settles();
|
||||
expect(settle.values).toEqual([
|
||||
'PENDING',
|
||||
'subscription inactive',
|
||||
30_000,
|
||||
'd1',
|
||||
CLAIMED_AT,
|
||||
]);
|
||||
});
|
||||
|
||||
it('stores the loggable error of a failed connection without a status code', async () => {
|
||||
postWebhookMock.mockRejectedValueOnce(
|
||||
new Error('Webhook host receiver.example resolves to a blocked address'),
|
||||
);
|
||||
db.claims.push([row()]);
|
||||
|
||||
await worker.runOnce();
|
||||
|
||||
expect(db.statusCodeUpdates()[0].data).toEqual({ lastStatusCode: null });
|
||||
expect(db.settles()[0].values).toContain(
|
||||
'Webhook host receiver.example resolves to a blocked address',
|
||||
);
|
||||
});
|
||||
|
||||
it('never stores the response body or the secret in lastError', async () => {
|
||||
postWebhookMock
|
||||
.mockResolvedValueOnce({ status: 500 })
|
||||
.mockRejectedValueOnce(new Error('socket hang up'));
|
||||
db.claims.push([row({ id: 'd1' }), row({ id: 'd2' })]);
|
||||
|
||||
await worker.runOnce();
|
||||
|
||||
const lastErrorOf = (id: string): string =>
|
||||
String(
|
||||
db.settles().find((settle) => settle.values.includes(id))?.values[1],
|
||||
);
|
||||
// A fixed text for an answer: postWebhook discards the body unread.
|
||||
expect(lastErrorOf('d1')).toBe('Receiver answered HTTP 500');
|
||||
expect(lastErrorOf('d2')).toBe('socket hang up');
|
||||
const signatures = postWebhookMock.mock.calls.map(([, headers]) =>
|
||||
headers['X-ApiCore-Signature'].slice('sha256='.length),
|
||||
);
|
||||
for (const stored of [lastErrorOf('d1'), lastErrorOf('d2'), logLines()]) {
|
||||
expect(stored).not.toContain(SECRET);
|
||||
expect(stored).not.toContain('private note');
|
||||
signatures.forEach((signature) =>
|
||||
expect(stored).not.toContain(signature),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it('logs a Prisma error by name and code only', async () => {
|
||||
db.prisma.webhookSubscription.findUnique.mockRejectedValueOnce(
|
||||
new Prisma.PrismaClientKnownRequestError(
|
||||
'Invalid query: url = "https://receiver.example/hook?token=abc"',
|
||||
{ code: 'P2024', clientVersion: '7.9.1' },
|
||||
),
|
||||
);
|
||||
db.claims.push([row()]);
|
||||
|
||||
await worker.runOnce();
|
||||
|
||||
expect(logLines()).toContain('P2024');
|
||||
expect(logLines()).not.toContain('token=abc');
|
||||
});
|
||||
|
||||
it('sends at most 5 at once', async () => {
|
||||
const answers = Array.from({ length: 12 }, () =>
|
||||
deferred<WebhookResponse>(),
|
||||
);
|
||||
let inFlight = 0;
|
||||
let maxInFlight = 0;
|
||||
answers.forEach((answer) =>
|
||||
postWebhookMock.mockImplementationOnce(() => {
|
||||
inFlight += 1;
|
||||
maxInFlight = Math.max(maxInFlight, inFlight);
|
||||
return answer.promise.finally(() => {
|
||||
inFlight -= 1;
|
||||
});
|
||||
}),
|
||||
);
|
||||
db.claims.push(
|
||||
Array.from({ length: 12 }, (_, index) => row({ id: `d${index}` })),
|
||||
);
|
||||
|
||||
const run = worker.runOnce();
|
||||
await flush();
|
||||
expect(postWebhookMock).toHaveBeenCalledTimes(5);
|
||||
|
||||
answers[0].resolve({ status: 200 });
|
||||
await flush();
|
||||
expect(postWebhookMock).toHaveBeenCalledTimes(6);
|
||||
|
||||
answers.forEach((answer) => answer.resolve({ status: 200 }));
|
||||
await expect(run).resolves.toBe(12);
|
||||
expect(postWebhookMock).toHaveBeenCalledTimes(12);
|
||||
expect(maxInFlight).toBe(5);
|
||||
});
|
||||
|
||||
// postWebhook ends the exchange - request and socket - at its deadline
|
||||
// (webhook-delivery.spec.ts); the worker keeps the default and counts it.
|
||||
it('counts a missed deadline as a failed attempt without a status code', async () => {
|
||||
postWebhookMock.mockRejectedValueOnce(
|
||||
new Error('Webhook receiver did not finish answering in time'),
|
||||
);
|
||||
db.claims.push([row()]);
|
||||
|
||||
await expect(worker.runOnce()).resolves.toBe(1);
|
||||
|
||||
expect(postWebhookMock.mock.calls[0]).toEqual([
|
||||
'https://receiver.example/hook',
|
||||
expect.any(Object),
|
||||
BODY,
|
||||
]);
|
||||
expect(db.statusCodeUpdates()[0].data).toEqual({ lastStatusCode: null });
|
||||
expect(db.settles()[0].values).toContain(
|
||||
'Webhook receiver did not finish answering in time',
|
||||
);
|
||||
});
|
||||
|
||||
it('settles the rest of the batch when one delivery cannot be processed', async () => {
|
||||
db.prisma.webhookSubscription.findUnique.mockRejectedValueOnce(
|
||||
new Error('connection reset'),
|
||||
);
|
||||
db.claims.push([row({ id: 'd1' }), row({ id: 'd2' })]);
|
||||
|
||||
await expect(worker.runOnce()).resolves.toBe(2);
|
||||
|
||||
expect(db.settles().map((settle) => settle.values)).toEqual([
|
||||
['DELIVERED', 'd2', CLAIMED_AT],
|
||||
]);
|
||||
expect(logError).toHaveBeenCalledWith(
|
||||
expect.stringContaining(
|
||||
'Webhook delivery d1 (colony.created) not settled',
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('changes nothing when the claim was lost before settling', async () => {
|
||||
db.tx.webhookDelivery.updateMany.mockResolvedValueOnce({ count: 0 });
|
||||
db.claims.push([row()]);
|
||||
|
||||
await worker.runOnce();
|
||||
|
||||
expect(db.settles()).toEqual([]);
|
||||
expect(logWarn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('claim lost before settling'),
|
||||
);
|
||||
});
|
||||
|
||||
describe('drain', () => {
|
||||
it('runs batches until one claims nothing', async () => {
|
||||
db.claims.push([row({ id: 'd1' })], [row({ id: 'd2' })]);
|
||||
|
||||
await expect(worker.drain()).resolves.toBe(2);
|
||||
|
||||
expect(db.prisma.$queryRaw).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it(`stops after ${DRAIN_MAX_ROUNDS} rounds by default and after maxRounds when given`, async () => {
|
||||
for (let i = 0; i < DRAIN_MAX_ROUNDS + 5; i++) {
|
||||
db.claims.push([row({ id: `d${i}` })]);
|
||||
}
|
||||
|
||||
await expect(worker.drain(undefined, 2)).resolves.toBe(2);
|
||||
await expect(worker.drain()).resolves.toBe(DRAIN_MAX_ROUNDS);
|
||||
});
|
||||
|
||||
it('passes the subscription filter to every claim', async () => {
|
||||
db.claims.push([row()]);
|
||||
|
||||
await worker.drain({ subscriptionIds: ['wh-1'] });
|
||||
|
||||
expect(db.claimQuery(0).values).toEqual([['wh-1'], 50]);
|
||||
expect(db.claimQuery(1).values).toEqual([['wh-1'], 50]);
|
||||
});
|
||||
|
||||
it('claims no further batch once the signal aborts, but finishes the claimed one', async () => {
|
||||
const stop = new AbortController();
|
||||
postWebhookMock.mockImplementationOnce(() => {
|
||||
stop.abort();
|
||||
return Promise.resolve({ status: 200 });
|
||||
});
|
||||
db.claims.push([row({ id: 'd1' })], [row({ id: 'd2' })]);
|
||||
|
||||
await expect(worker.drain(undefined, 20, stop.signal)).resolves.toBe(1);
|
||||
|
||||
expect(db.prisma.$queryRaw).toHaveBeenCalledTimes(1);
|
||||
expect(db.settles()[0].values).toEqual(['DELIVERED', 'd1', CLAIMED_AT]);
|
||||
});
|
||||
|
||||
it('claims nothing when the signal is already aborted', async () => {
|
||||
const stop = new AbortController();
|
||||
stop.abort();
|
||||
|
||||
await expect(worker.drain(undefined, 20, stop.signal)).resolves.toBe(0);
|
||||
|
||||
expect(db.prisma.$queryRaw).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
it('releases stale claims of the delivery table and logs the ones that FAILED', async () => {
|
||||
db.releases.push([
|
||||
{ id: 'd1', status: 'PENDING' },
|
||||
{ id: 'd2', status: 'FAILED' },
|
||||
]);
|
||||
|
||||
await expect(worker.releaseStaleClaims()).resolves.toBe(2);
|
||||
|
||||
const release = db.claimQuery();
|
||||
expect(text(release)).toContain('UPDATE "WebhookDelivery"');
|
||||
expect(text(release)).toContain(
|
||||
`"claimedAt" < now() - (? * interval '1 millisecond')`,
|
||||
);
|
||||
expect(logError).toHaveBeenCalledWith(expect.stringContaining('d2'));
|
||||
expect(logLines()).not.toContain('d1');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,285 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import * as crypto from 'node:crypto';
|
||||
import { Prisma } from '../../generated/prisma';
|
||||
import { loggableError } from '../common/loggable-error';
|
||||
import {
|
||||
claimBatch,
|
||||
releaseStaleClaimRows,
|
||||
settleFailure,
|
||||
settleSuccess,
|
||||
} from '../common/outbox/outbox-claims';
|
||||
import { DRAIN_MAX_ROUNDS } from '../domain-events/domain-event-dispatcher';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { postWebhook } from './webhook-delivery';
|
||||
import { openWebhookSecret } from './webhook-secret';
|
||||
|
||||
export const WEBHOOK_DELIVERY_BATCH_SIZE = 50;
|
||||
export const WEBHOOK_DELIVERY_CONCURRENCY = 5;
|
||||
|
||||
const SUBSCRIPTION_INACTIVE = 'subscription inactive';
|
||||
|
||||
export interface WebhookDeliveryFilter {
|
||||
/** Only deliveries of these subscriptions (e2e tests: no foreign rows). */
|
||||
subscriptionIds?: readonly string[];
|
||||
}
|
||||
|
||||
/** A claimed WebhookDelivery row, as RETURNING * yields it. */
|
||||
interface ClaimedWebhookDelivery {
|
||||
id: string;
|
||||
subscriptionId: string;
|
||||
event: string;
|
||||
body: string;
|
||||
attempts: number;
|
||||
claimedAt: Date;
|
||||
}
|
||||
|
||||
interface DeliveryTarget {
|
||||
url: string;
|
||||
secret: string;
|
||||
}
|
||||
|
||||
/** What one attempt came to; `error` is set exactly when not delivered. */
|
||||
type AttemptOutcome =
|
||||
| { delivered: true; statusCode: number }
|
||||
| { delivered: false; statusCode: number | null; error: string };
|
||||
|
||||
/**
|
||||
* Sends the queued webhook deliveries (WebhooksService.enqueueDeliveries):
|
||||
* claims up to 50 due rows, posts at most 5 at once and settles each one -
|
||||
* DELIVERED on a 2xx answer, otherwise a failed attempt under the shared
|
||||
* retry rule (retry-policy.ts), FAILED after the 8th. The delivery id is
|
||||
* the row id, so a receiver sees the same X-ApiCore-Delivery and the same
|
||||
* body on every retry and can drop duplicates. Run by the worker's
|
||||
* webhook-deliveries loop.
|
||||
*/
|
||||
@Injectable()
|
||||
export class WebhookDeliveryWorker {
|
||||
private readonly logger = new Logger(WebhookDeliveryWorker.name);
|
||||
|
||||
constructor(private readonly prisma: PrismaService) {}
|
||||
|
||||
/** One batch: claims up to 50 deliveries and returns how many it claimed. */
|
||||
async runOnce(filter?: WebhookDeliveryFilter): Promise<number> {
|
||||
const deliveries = await claimBatch<ClaimedWebhookDelivery>(
|
||||
this.prisma,
|
||||
'WebhookDelivery',
|
||||
{
|
||||
limit: WEBHOOK_DELIVERY_BATCH_SIZE,
|
||||
filter: claimFilter(filter),
|
||||
orderBy: 'createdAt',
|
||||
},
|
||||
);
|
||||
await forEachConcurrently(
|
||||
deliveries,
|
||||
WEBHOOK_DELIVERY_CONCURRENCY,
|
||||
(delivery) => this.deliverSafely(delivery),
|
||||
);
|
||||
return deliveries.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs batches until one claims nothing, at most `maxRounds`. Returns the
|
||||
* claimed total. Once `signal` aborts (worker shutdown), no further batch
|
||||
* is claimed; the batch already claimed is finished, since its deliveries
|
||||
* would otherwise wait for the stale-claim release.
|
||||
*/
|
||||
async drain(
|
||||
filter?: WebhookDeliveryFilter,
|
||||
maxRounds = DRAIN_MAX_ROUNDS,
|
||||
signal?: AbortSignal,
|
||||
): Promise<number> {
|
||||
let total = 0;
|
||||
for (let round = 0; round < maxRounds && !signal?.aborted; round++) {
|
||||
const claimed = await this.runOnce(filter);
|
||||
if (claimed === 0) {
|
||||
break;
|
||||
}
|
||||
total += claimed;
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
/** Puts deliveries whose worker died back in line (or to FAILED). */
|
||||
async releaseStaleClaims(): Promise<number> {
|
||||
const released = await releaseStaleClaimRows(
|
||||
this.prisma,
|
||||
'WebhookDelivery',
|
||||
);
|
||||
const failed = released
|
||||
.filter((claim) => claim.status === 'FAILED')
|
||||
.map((claim) => claim.id);
|
||||
if (failed.length > 0) {
|
||||
this.logger.error(
|
||||
`Webhook deliveries FAILED, an expired claim used up their attempts: ${failed.join(', ')}`,
|
||||
);
|
||||
}
|
||||
return released.length;
|
||||
}
|
||||
|
||||
/** One delivery must not stop the batch; an unsettled one goes stale. */
|
||||
private async deliverSafely(delivery: ClaimedWebhookDelivery): Promise<void> {
|
||||
try {
|
||||
await this.deliver(delivery);
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Webhook delivery ${delivery.id} (${delivery.event}) not settled: ${loggableError(error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async deliver(delivery: ClaimedWebhookDelivery): Promise<void> {
|
||||
const subscription = await this.prisma.webhookSubscription.findUnique({
|
||||
where: { id: delivery.subscriptionId },
|
||||
select: { url: true, secret: true, active: true },
|
||||
});
|
||||
// A paused subscription sends nothing; the attempt counts, so its
|
||||
// deliveries go out if it is resumed within the retry window.
|
||||
const outcome: AttemptOutcome = subscription?.active
|
||||
? await this.attempt(delivery, subscription)
|
||||
: { delivered: false, statusCode: null, error: SUBSCRIPTION_INACTIVE };
|
||||
await this.settle(delivery, outcome);
|
||||
}
|
||||
|
||||
/**
|
||||
* Signs and posts the stored body. Signed together with a timestamp and
|
||||
* the delivery id, so a receiver can reject a captured request replayed
|
||||
* later ("t.id.body", the same idea as Stripe's signatures). Only a 2xx
|
||||
* answer counts as delivered; the answer's body is never read.
|
||||
*/
|
||||
private async attempt(
|
||||
delivery: ClaimedWebhookDelivery,
|
||||
target: DeliveryTarget,
|
||||
): Promise<AttemptOutcome> {
|
||||
try {
|
||||
const timestamp = Math.floor(Date.now() / 1000).toString();
|
||||
const signature = crypto
|
||||
.createHmac('sha256', openWebhookSecret(target.secret))
|
||||
.update(`${timestamp}.${delivery.id}.${delivery.body}`)
|
||||
.digest('hex');
|
||||
// No redirects, connect-time address check, 5 s idle timeout and a
|
||||
// 10 s deadline that also closes the connection - a trickling
|
||||
// receiver can hold neither a slot nor a socket. See postWebhook().
|
||||
const { status } = await postWebhook(
|
||||
target.url,
|
||||
{
|
||||
'Content-Type': 'application/json',
|
||||
'User-Agent': 'ApiCore-Webhooks/1',
|
||||
'X-ApiCore-Event': delivery.event,
|
||||
'X-ApiCore-Delivery': delivery.id,
|
||||
'X-ApiCore-Timestamp': timestamp,
|
||||
'X-ApiCore-Signature': `sha256=${signature}`,
|
||||
},
|
||||
delivery.body,
|
||||
);
|
||||
return status >= 200 && status < 300
|
||||
? { delivered: true, statusCode: status }
|
||||
: {
|
||||
delivered: false,
|
||||
statusCode: status,
|
||||
error: `Receiver answered HTTP ${status}`,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
delivered: false,
|
||||
statusCode: null,
|
||||
error: loggableError(error).message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores the status code and settles the claim in one transaction. Both
|
||||
* are a compare-and-set on id + claimedAt: a worker whose claim was
|
||||
* released in the meantime changes nothing.
|
||||
*/
|
||||
private async settle(
|
||||
delivery: ClaimedWebhookDelivery,
|
||||
outcome: AttemptOutcome,
|
||||
): Promise<void> {
|
||||
const result = await this.prisma.$transaction(async (tx) => {
|
||||
const { count } = await tx.webhookDelivery.updateMany({
|
||||
where: {
|
||||
id: delivery.id,
|
||||
status: 'PROCESSING',
|
||||
claimedAt: delivery.claimedAt,
|
||||
},
|
||||
data: { lastStatusCode: outcome.statusCode },
|
||||
});
|
||||
if (count === 0) {
|
||||
return 'lost' as const;
|
||||
}
|
||||
if (outcome.delivered) {
|
||||
await settleSuccess(
|
||||
tx,
|
||||
'WebhookDelivery',
|
||||
delivery.id,
|
||||
delivery.claimedAt,
|
||||
'DELIVERED',
|
||||
);
|
||||
return 'delivered' as const;
|
||||
}
|
||||
return settleFailure(
|
||||
tx,
|
||||
'WebhookDelivery',
|
||||
delivery.id,
|
||||
delivery.claimedAt,
|
||||
delivery.attempts,
|
||||
outcome.error,
|
||||
);
|
||||
});
|
||||
this.logOutcome(delivery, outcome, result);
|
||||
}
|
||||
|
||||
private logOutcome(
|
||||
delivery: ClaimedWebhookDelivery,
|
||||
outcome: AttemptOutcome,
|
||||
result: 'lost' | 'delivered' | 'retry' | 'failed',
|
||||
): void {
|
||||
const subject = `Webhook delivery ${delivery.id} (${delivery.event}) to subscription ${delivery.subscriptionId}`;
|
||||
const error = outcome.delivered ? '' : outcome.error;
|
||||
const attempt = delivery.attempts + 1;
|
||||
if (result === 'lost') {
|
||||
this.logger.warn(
|
||||
`${subject}: claim lost before settling, left to its current owner`,
|
||||
);
|
||||
} else if (result === 'failed') {
|
||||
this.logger.error(
|
||||
`Webhook delivery ${delivery.id} (${delivery.event}) FAILED after ${attempt} attempts to subscription ${delivery.subscriptionId}: ${error}`,
|
||||
);
|
||||
} else if (result === 'retry') {
|
||||
this.logger.warn(
|
||||
`${subject}: attempt ${attempt} failed, retrying: ${error}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function claimFilter(filter?: WebhookDeliveryFilter): Prisma.Sql | undefined {
|
||||
if (filter?.subscriptionIds === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
return Prisma.sql`"subscriptionId" = ANY(${[...filter.subscriptionIds]})`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs `work` for every item, at most `limit` at a time, and resolves once
|
||||
* all are done. `work` must not reject - every delivery settles its own
|
||||
* errors (deliverSafely).
|
||||
*/
|
||||
async function forEachConcurrently<T>(
|
||||
items: readonly T[],
|
||||
limit: number,
|
||||
work: (item: T) => Promise<void>,
|
||||
): Promise<void> {
|
||||
let next = 0;
|
||||
const lane = async (): Promise<void> => {
|
||||
while (next < items.length) {
|
||||
const item = items[next];
|
||||
next += 1;
|
||||
await work(item);
|
||||
}
|
||||
};
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(limit, items.length) }, lane),
|
||||
);
|
||||
}
|
||||
@@ -1,6 +1,9 @@
|
||||
export const WEBHOOK_EVENTS = [
|
||||
'colony.created',
|
||||
'inspection.created',
|
||||
'varroa.measured',
|
||||
'treatment.created',
|
||||
'feeding.created',
|
||||
'queen.created',
|
||||
'queen.replaced',
|
||||
'harvest.created',
|
||||
|
||||
@@ -2,6 +2,7 @@ import { Logger } from '@nestjs/common';
|
||||
import { encryptMfaSecret } from '../auth/mfa/mfa-crypto.util';
|
||||
import { KEY_ID_PATTERN } from '../common/crypto/key-ring';
|
||||
import {
|
||||
isPlaintextWebhookSecret,
|
||||
openWebhookSecret,
|
||||
sealWebhookSecret,
|
||||
webhookSecretNeedsRekey,
|
||||
@@ -46,6 +47,14 @@ describe('webhook-secret', () => {
|
||||
expect(webhookSecretNeedsRekey(LEGACY_PLAIN)).toBe(true);
|
||||
});
|
||||
|
||||
it('tells plaintext secrets from sealed ones, so rekey can report them apart', () => {
|
||||
expect(isPlaintextWebhookSecret(LEGACY_PLAIN)).toBe(true);
|
||||
expect(isPlaintextWebhookSecret(LEGACY_SEALED)).toBe(false);
|
||||
expect(isPlaintextWebhookSecret(sealWebhookSecret('topsecret'))).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('opens a secret sealed with WEBHOOK_ENCRYPTION_KEY_PREVIOUS and flags it for rekey', () => {
|
||||
const stored = sealWebhookSecret('topsecret');
|
||||
process.env.WEBHOOK_ENCRYPTION_KEY_PREVIOUS =
|
||||
|
||||
@@ -34,3 +34,8 @@ export function openWebhookSecret(stored: string): string {
|
||||
export function webhookSecretNeedsRekey(stored: string): boolean {
|
||||
return !isSealedWithCurrentKey(loadKeyRing('webhook'), stored);
|
||||
}
|
||||
|
||||
/** True for a secret from before encryption, stored as it is (neither `k…:` nor `enc:v1:`). */
|
||||
export function isPlaintextWebhookSecret(stored: string): boolean {
|
||||
return !KEY_ID_PATTERN.test(stored) && !stored.startsWith(LEGACY_PREFIX);
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
import { Prisma } from '../../generated/prisma';
|
||||
import type {
|
||||
AnyDomainEventEnvelope,
|
||||
DomainEventEnvelope,
|
||||
DomainEventType,
|
||||
} from '../domain-events/domain-event-types';
|
||||
import { domainEventHandlerMetadata } from '../domain-events/on-domain-event.decorator';
|
||||
import {
|
||||
DOMAIN_EVENT_WEBHOOKS,
|
||||
WebhookEventHandlers,
|
||||
} from './webhook.handlers';
|
||||
import type { WebhooksService } from './webhooks.service';
|
||||
|
||||
type RecordModel =
|
||||
| 'inspection'
|
||||
| 'varroaMeasurement'
|
||||
| 'treatment'
|
||||
| 'feeding'
|
||||
| 'harvest'
|
||||
| 'honeyBatch'
|
||||
| 'task';
|
||||
|
||||
const RECORD_MODELS: readonly RecordModel[] = [
|
||||
'inspection',
|
||||
'varroaMeasurement',
|
||||
'treatment',
|
||||
'feeding',
|
||||
'harvest',
|
||||
'honeyBatch',
|
||||
'task',
|
||||
];
|
||||
|
||||
function envelope<T extends DomainEventType>(
|
||||
type: T,
|
||||
aggregateId: string,
|
||||
payload: DomainEventEnvelope<T>['payload'],
|
||||
): AnyDomainEventEnvelope {
|
||||
return {
|
||||
id: 'e1',
|
||||
type,
|
||||
aggregateType: 'Inspection',
|
||||
aggregateId,
|
||||
payload,
|
||||
actorUserId: 'u1',
|
||||
correlationId: 'corr-1',
|
||||
occurredAt: new Date('2026-10-13T08:00:00.000Z'),
|
||||
attempts: 0,
|
||||
} as AnyDomainEventEnvelope;
|
||||
}
|
||||
|
||||
describe('WebhookEventHandlers', () => {
|
||||
let tx: Record<RecordModel, { findUnique: jest.Mock }>;
|
||||
let webhooks: { enqueueDeliveries: jest.Mock };
|
||||
let handlers: WebhookEventHandlers;
|
||||
|
||||
const forward = (event: AnyDomainEventEnvelope) =>
|
||||
handlers.forward(event, tx as unknown as Prisma.TransactionClient);
|
||||
|
||||
beforeEach(() => {
|
||||
tx = Object.fromEntries(
|
||||
RECORD_MODELS.map((model) => [
|
||||
model,
|
||||
{ findUnique: jest.fn().mockResolvedValue(null) },
|
||||
]),
|
||||
) as Record<RecordModel, { findUnique: jest.Mock }>;
|
||||
webhooks = { enqueueDeliveries: jest.fn().mockResolvedValue(1) };
|
||||
handlers = new WebhookEventHandlers(webhooks as unknown as WebhooksService);
|
||||
});
|
||||
|
||||
it('maps exactly these domain events to webhook events', () => {
|
||||
expect(DOMAIN_EVENT_WEBHOOKS).toEqual({
|
||||
InspectionRecorded: 'inspection.created',
|
||||
VarroaMeasured: 'varroa.measured',
|
||||
TreatmentRecorded: 'treatment.created',
|
||||
FeedingRecorded: 'feeding.created',
|
||||
HarvestRecorded: 'harvest.created',
|
||||
BatchBottled: 'batch.updated',
|
||||
BottlingCorrected: 'batch.updated',
|
||||
StockChanged: 'stock.changed',
|
||||
TaskCreated: 'task.created',
|
||||
TaskCompleted: 'task.completed',
|
||||
});
|
||||
});
|
||||
|
||||
it('registers forward as "webhooks.forward" for every mapped type', () => {
|
||||
// The decorator stores its metadata on the method itself.
|
||||
const method = Reflect.get(WebhookEventHandlers.prototype, 'forward');
|
||||
expect(domainEventHandlerMetadata(method as object)).toEqual([
|
||||
{
|
||||
types: Object.keys(DOMAIN_EVENT_WEBHOOKS),
|
||||
handlerId: 'webhooks.forward',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('enqueues inspection.created with the inspection as data', async () => {
|
||||
const inspection = { id: 'i1', colonyId: 'c1', notes: 'calm' };
|
||||
tx.inspection.findUnique.mockResolvedValueOnce(inspection);
|
||||
|
||||
await forward(envelope('InspectionRecorded', 'i1', { colonyId: 'c1' }));
|
||||
|
||||
expect(tx.inspection.findUnique).toHaveBeenCalledWith({
|
||||
where: { id: 'i1' },
|
||||
});
|
||||
// Through the delivery transaction: the rows commit with the delivery.
|
||||
expect(webhooks.enqueueDeliveries).toHaveBeenCalledWith(
|
||||
tx,
|
||||
'inspection.created',
|
||||
inspection,
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['VarroaMeasured', 'varroaMeasurement', 'varroa.measured'],
|
||||
['TreatmentRecorded', 'treatment', 'treatment.created'],
|
||||
['FeedingRecorded', 'feeding', 'feeding.created'],
|
||||
] as const)(
|
||||
'enqueues %s as %s record under %s',
|
||||
async (type, model, webhookEvent) => {
|
||||
const record = { id: 'r1', colonyId: 'c1' };
|
||||
tx[model].findUnique.mockResolvedValueOnce(record);
|
||||
|
||||
await forward(envelope(type, 'r1', { colonyId: 'c1' }));
|
||||
|
||||
expect(tx[model].findUnique).toHaveBeenCalledWith({
|
||||
where: { id: 'r1' },
|
||||
});
|
||||
expect(webhooks.enqueueDeliveries).toHaveBeenCalledWith(
|
||||
tx,
|
||||
webhookEvent,
|
||||
record,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
['TaskCreated', 'task.created'],
|
||||
['TaskCompleted', 'task.completed'],
|
||||
] as const)('enqueues %s as %s with the task', async (type, webhookEvent) => {
|
||||
const task = { id: 't1', title: 'Varroa zählen' };
|
||||
tx.task.findUnique.mockResolvedValueOnce(task);
|
||||
|
||||
await forward(
|
||||
envelope(type, 't1', {
|
||||
relatedType: 'COLONY',
|
||||
relatedId: 'c1',
|
||||
dueDate: null,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(tx.task.findUnique).toHaveBeenCalledWith({ where: { id: 't1' } });
|
||||
expect(webhooks.enqueueDeliveries).toHaveBeenCalledWith(
|
||||
tx,
|
||||
webhookEvent,
|
||||
task,
|
||||
);
|
||||
});
|
||||
|
||||
it('enqueues harvest.created with the harvest and its colonies', async () => {
|
||||
const harvest = {
|
||||
id: 'h1',
|
||||
colonies: [{ colonyId: 'c1', colony: { id: 'c1', code: 'V-1' } }],
|
||||
};
|
||||
tx.harvest.findUnique.mockResolvedValueOnce(harvest);
|
||||
|
||||
await forward(
|
||||
envelope('HarvestRecorded', 'h1', { apiaryId: 'a1', colonyIds: ['c1'] }),
|
||||
);
|
||||
|
||||
expect(tx.harvest.findUnique).toHaveBeenCalledWith({
|
||||
where: { id: 'h1' },
|
||||
include: { colonies: { include: { colony: true } } },
|
||||
});
|
||||
expect(webhooks.enqueueDeliveries).toHaveBeenCalledWith(
|
||||
tx,
|
||||
'harvest.created',
|
||||
harvest,
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['BatchBottled', { honeyBatchId: 'hb1' }],
|
||||
['BottlingCorrected', { honeyBatchId: 'hb1', removed: true }],
|
||||
] as const)(
|
||||
'enqueues batch.updated with the honey batch for %s',
|
||||
async (type, payload) => {
|
||||
const honeyBatch = { id: 'hb1', lotNumber: 'L-2026-001' };
|
||||
tx.honeyBatch.findUnique.mockResolvedValueOnce(honeyBatch);
|
||||
|
||||
// The aggregate is the bottling batch; the data is its honey batch,
|
||||
// which still exists after a bottling was removed.
|
||||
await forward(
|
||||
envelope(
|
||||
type,
|
||||
'bb1',
|
||||
payload as DomainEventEnvelope<'BottlingCorrected'>['payload'],
|
||||
),
|
||||
);
|
||||
|
||||
expect(tx.honeyBatch.findUnique).toHaveBeenCalledWith({
|
||||
where: { id: 'hb1' },
|
||||
});
|
||||
expect(webhooks.enqueueDeliveries).toHaveBeenCalledWith(
|
||||
tx,
|
||||
'batch.updated',
|
||||
honeyBatch,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it('enqueues stock.changed with the bottling batch and product ids, without raw material kinds', async () => {
|
||||
await forward(
|
||||
envelope('StockChanged', 'm1', {
|
||||
bottlingBatchIds: ['b1', 'b1', 'b2'],
|
||||
productIds: ['p1'],
|
||||
rawMaterialKinds: ['BEESWAX'],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(webhooks.enqueueDeliveries).toHaveBeenCalledWith(
|
||||
tx,
|
||||
'stock.changed',
|
||||
{ bottlingBatchIds: ['b1', 'b2'], productIds: ['p1'] },
|
||||
);
|
||||
RECORD_MODELS.forEach((model) =>
|
||||
expect(tx[model].findUnique).not.toHaveBeenCalled(),
|
||||
);
|
||||
});
|
||||
|
||||
it('enqueues nothing for StockChanged that only touches raw material', async () => {
|
||||
await forward(
|
||||
envelope('StockChanged', 'm1', {
|
||||
bottlingBatchIds: [],
|
||||
productIds: [],
|
||||
rawMaterialKinds: ['PROPOLIS'],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(webhooks.enqueueDeliveries).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('skips a record that no longer exists', async () => {
|
||||
tx.inspection.findUnique.mockResolvedValueOnce(null);
|
||||
|
||||
await forward(envelope('InspectionRecorded', 'gone', { colonyId: 'c1' }));
|
||||
|
||||
expect(webhooks.enqueueDeliveries).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('enqueues nothing for a type without a webhook', async () => {
|
||||
await forward(envelope('TaskDue', 't1', { dueOn: '2026-10-13' }));
|
||||
|
||||
expect(webhooks.enqueueDeliveries).not.toHaveBeenCalled();
|
||||
RECORD_MODELS.forEach((model) =>
|
||||
expect(tx[model].findUnique).not.toHaveBeenCalled(),
|
||||
);
|
||||
});
|
||||
|
||||
// The dispatcher rolls the delivery transaction back and retries.
|
||||
it('fails when the deliveries cannot be enqueued', async () => {
|
||||
tx.task.findUnique.mockResolvedValueOnce({ id: 't1' });
|
||||
webhooks.enqueueDeliveries.mockRejectedValueOnce(new Error('db down'));
|
||||
|
||||
await expect(
|
||||
forward(
|
||||
envelope('TaskCompleted', 't1', {
|
||||
relatedType: 'GENERAL',
|
||||
relatedId: null,
|
||||
}),
|
||||
),
|
||||
).rejects.toThrow('db down');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,117 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import type {
|
||||
AnyDomainEventEnvelope,
|
||||
DomainEventPayloads,
|
||||
DomainEventType,
|
||||
} from '../domain-events/domain-event-types';
|
||||
import { OnDomainEvent } from '../domain-events/on-domain-event.decorator';
|
||||
import { Prisma } from '../../generated/prisma';
|
||||
import { WebhookEvent } from './webhook-events';
|
||||
import { WebhooksService } from './webhooks.service';
|
||||
|
||||
/** Which domain event goes out as which webhook event. */
|
||||
export const DOMAIN_EVENT_WEBHOOKS: Readonly<
|
||||
Partial<Record<DomainEventType, WebhookEvent>>
|
||||
> = Object.freeze({
|
||||
InspectionRecorded: 'inspection.created',
|
||||
VarroaMeasured: 'varroa.measured',
|
||||
TreatmentRecorded: 'treatment.created',
|
||||
FeedingRecorded: 'feeding.created',
|
||||
HarvestRecorded: 'harvest.created',
|
||||
BatchBottled: 'batch.updated',
|
||||
BottlingCorrected: 'batch.updated',
|
||||
StockChanged: 'stock.changed',
|
||||
TaskCreated: 'task.created',
|
||||
TaskCompleted: 'task.completed',
|
||||
});
|
||||
|
||||
/**
|
||||
* Turns domain events into queued webhook deliveries. Pure DB writer: it
|
||||
* reads the record through the delivery transaction and queues the rows in
|
||||
* it (WebhooksService.enqueueDeliveries); the worker's webhook-deliveries
|
||||
* loop sends them. Errors propagate, so a failed delivery is retried.
|
||||
*/
|
||||
@Injectable()
|
||||
export class WebhookEventHandlers {
|
||||
constructor(private readonly webhooks: WebhooksService) {}
|
||||
|
||||
@OnDomainEvent(
|
||||
Object.keys(DOMAIN_EVENT_WEBHOOKS) as DomainEventType[],
|
||||
'webhooks.forward',
|
||||
)
|
||||
async forward(
|
||||
event: AnyDomainEventEnvelope,
|
||||
tx: Prisma.TransactionClient,
|
||||
): Promise<void> {
|
||||
const webhookEvent = DOMAIN_EVENT_WEBHOOKS[event.type];
|
||||
if (webhookEvent === undefined) {
|
||||
return;
|
||||
}
|
||||
const data = await webhookData(event, tx);
|
||||
// The record is gone (deleted before the event was dispatched), or the
|
||||
// change is of no interest to receivers: nothing to send.
|
||||
if (data === null) {
|
||||
return;
|
||||
}
|
||||
await this.webhooks.enqueueDeliveries(tx, webhookEvent, data);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The webhook's `data`, as the legacy path sent it: the record the event is
|
||||
* about. Null when there is nothing to send.
|
||||
*/
|
||||
async function webhookData(
|
||||
event: AnyDomainEventEnvelope,
|
||||
tx: Prisma.TransactionClient,
|
||||
): Promise<unknown> {
|
||||
const byAggregate = { where: { id: event.aggregateId } };
|
||||
switch (event.type) {
|
||||
case 'InspectionRecorded':
|
||||
return tx.inspection.findUnique(byAggregate);
|
||||
case 'VarroaMeasured':
|
||||
return tx.varroaMeasurement.findUnique(byAggregate);
|
||||
case 'TreatmentRecorded':
|
||||
return tx.treatment.findUnique(byAggregate);
|
||||
case 'FeedingRecorded':
|
||||
return tx.feeding.findUnique(byAggregate);
|
||||
case 'HarvestRecorded':
|
||||
return tx.harvest.findUnique({
|
||||
...byAggregate,
|
||||
include: { colonies: { include: { colony: true } } },
|
||||
});
|
||||
// The aggregate is the bottling batch; receivers get its honey batch,
|
||||
// which outlives a removed bottling.
|
||||
case 'BatchBottled':
|
||||
case 'BottlingCorrected':
|
||||
return tx.honeyBatch.findUnique({
|
||||
where: { id: event.payload.honeyBatchId },
|
||||
});
|
||||
case 'StockChanged':
|
||||
return stockChange(event.payload);
|
||||
case 'TaskCreated':
|
||||
case 'TaskCompleted':
|
||||
return tx.task.findUnique(byAggregate);
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Receivers (the shop) fetch the numbers themselves. Raw material is not
|
||||
* sold there, so a change that only touches raw material is not sent.
|
||||
*/
|
||||
function stockChange(
|
||||
payload: DomainEventPayloads['StockChanged'],
|
||||
): { bottlingBatchIds: string[]; productIds: string[] } | null {
|
||||
if (
|
||||
payload.bottlingBatchIds.length === 0 &&
|
||||
payload.productIds.length === 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
bottlingBatchIds: [...new Set(payload.bottlingBatchIds)],
|
||||
productIds: [...new Set(payload.productIds)],
|
||||
};
|
||||
}
|
||||
@@ -7,12 +7,17 @@ import {
|
||||
Param,
|
||||
Patch,
|
||||
Post,
|
||||
Query,
|
||||
} from '@nestjs/common';
|
||||
import { ApiBearerAuth, ApiTags } from '@nestjs/swagger';
|
||||
import { RequirePermission } from '../auth/decorators/require-permission.decorator';
|
||||
import { CurrentUser } from '../auth/decorators/current-user.decorator';
|
||||
import type { AuthenticatedUser } from '../auth/types/authenticated-user';
|
||||
import { CreateWebhookDto } from './dto/create-webhook.dto';
|
||||
import {
|
||||
ListWebhookDeliveriesDto,
|
||||
WEBHOOK_DELIVERIES_DEFAULT_LIMIT,
|
||||
} from './dto/list-webhook-deliveries.dto';
|
||||
import { UpdateWebhookDto } from './dto/update-webhook.dto';
|
||||
import { WebhooksService } from './webhooks.service';
|
||||
|
||||
@@ -28,6 +33,19 @@ export class WebhooksController {
|
||||
return this.webhooksService.findAll();
|
||||
}
|
||||
|
||||
/** The delivery log of one subscription, newest first, without bodies. */
|
||||
@RequirePermission('webhooks:read')
|
||||
@Get(':id/deliveries')
|
||||
findDeliveries(
|
||||
@Param('id') id: string,
|
||||
@Query() query: ListWebhookDeliveriesDto,
|
||||
) {
|
||||
return this.webhooksService.findDeliveries(
|
||||
id,
|
||||
query.limit ?? WEBHOOK_DELIVERIES_DEFAULT_LIMIT,
|
||||
);
|
||||
}
|
||||
|
||||
@RequirePermission('webhooks:create')
|
||||
@Post()
|
||||
create(
|
||||
|
||||
@@ -1,11 +1,18 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
import { WebhookDeliveryWorker } from './webhook-delivery.worker';
|
||||
import { WebhookEventHandlers } from './webhook.handlers';
|
||||
import { WebhooksController } from './webhooks.controller';
|
||||
import { WebhooksService } from './webhooks.service';
|
||||
|
||||
/**
|
||||
* Subscriptions, the delivery queue and the handler that fills it from
|
||||
* domain events. WebhookDeliveryWorker sends the queue; only the worker
|
||||
* runtime (and e2e tests) call it.
|
||||
*/
|
||||
@Global()
|
||||
@Module({
|
||||
controllers: [WebhooksController],
|
||||
providers: [WebhooksService],
|
||||
exports: [WebhooksService],
|
||||
providers: [WebhooksService, WebhookDeliveryWorker, WebhookEventHandlers],
|
||||
exports: [WebhooksService, WebhookDeliveryWorker],
|
||||
})
|
||||
export class WebhooksModule {}
|
||||
@@ -1,15 +1,15 @@
|
||||
import * as crypto from 'node:crypto';
|
||||
import { BadRequestException, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { WebhooksService } from './webhooks.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { AuditLogService } from '../audit/audit-log.service';
|
||||
import { postWebhook } from './webhook-delivery';
|
||||
import { openWebhookSecret, sealWebhookSecret } from './webhook-secret';
|
||||
import { DbClient } from '../prisma/db-client';
|
||||
import { openWebhookSecret } from './webhook-secret';
|
||||
|
||||
jest.mock('./webhook-delivery', () => ({ postWebhook: jest.fn() }));
|
||||
|
||||
const postWebhookMock = postWebhook as jest.MockedFunction<typeof postWebhook>;
|
||||
const flushDeliveries = () => new Promise((resolve) => setImmediate(resolve));
|
||||
interface DeliveryRowInput {
|
||||
subscriptionId: string;
|
||||
event: string;
|
||||
body: string;
|
||||
}
|
||||
|
||||
describe('WebhooksService', () => {
|
||||
let service: WebhooksService;
|
||||
@@ -21,13 +21,16 @@ describe('WebhooksService', () => {
|
||||
update: jest.Mock;
|
||||
delete: jest.Mock;
|
||||
};
|
||||
webhookDelivery: {
|
||||
createMany: jest.Mock;
|
||||
findMany: jest.Mock;
|
||||
};
|
||||
$executeRaw: jest.Mock;
|
||||
};
|
||||
let auditLog: { record: jest.Mock };
|
||||
|
||||
beforeEach(() => {
|
||||
process.env.JWT_ACCESS_SECRET = 'unit-test-jwt-access-secret-0123456789';
|
||||
postWebhookMock.mockReset();
|
||||
postWebhookMock.mockResolvedValue({ status: 200 });
|
||||
prisma = {
|
||||
webhookSubscription: {
|
||||
findMany: jest.fn().mockResolvedValue([]),
|
||||
@@ -36,6 +39,15 @@ describe('WebhooksService', () => {
|
||||
update: jest.fn(),
|
||||
delete: jest.fn(),
|
||||
},
|
||||
webhookDelivery: {
|
||||
createMany: jest
|
||||
.fn()
|
||||
.mockImplementation(({ data }: { data: DeliveryRowInput[] }) =>
|
||||
Promise.resolve({ count: data.length }),
|
||||
),
|
||||
findMany: jest.fn().mockResolvedValue([]),
|
||||
},
|
||||
$executeRaw: jest.fn().mockResolvedValue(1),
|
||||
};
|
||||
auditLog = { record: jest.fn().mockResolvedValue(undefined) };
|
||||
service = new WebhooksService(
|
||||
@@ -44,6 +56,10 @@ describe('WebhooksService', () => {
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
|
||||
const db = () => prisma as unknown as DbClient;
|
||||
|
||||
it('create generates a random secret and stores it', async () => {
|
||||
prisma.webhookSubscription.create.mockImplementation(
|
||||
({ data }: { data: Record<string, unknown> }) => ({
|
||||
@@ -85,108 +101,191 @@ describe('WebhooksService', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('dispatch only queries subscriptions matching the event and active flag', async () => {
|
||||
await service.dispatch('colony.created', { id: 'colony-1' });
|
||||
describe('enqueueDeliveries', () => {
|
||||
it('enqueueDeliveries writes one row per active subscription for the event and wakes the worker', async () => {
|
||||
prisma.webhookSubscription.findMany.mockResolvedValueOnce([
|
||||
{ id: 'wh-1' },
|
||||
{ id: 'wh-2' },
|
||||
]);
|
||||
const payload = { id: 'colony-1', code: 'V-1' };
|
||||
|
||||
expect(prisma.webhookSubscription.findMany).toHaveBeenCalledWith({
|
||||
where: { active: true, events: { has: 'colony.created' } },
|
||||
await expect(
|
||||
service.enqueueDeliveries(db(), 'colony.created', payload),
|
||||
).resolves.toBe(2);
|
||||
|
||||
expect(prisma.webhookSubscription.findMany).toHaveBeenCalledWith({
|
||||
where: { active: true, events: { has: 'colony.created' } },
|
||||
select: { id: true },
|
||||
});
|
||||
// The body is fixed here, so every retry sends the same bytes.
|
||||
const body = JSON.stringify({ event: 'colony.created', data: payload });
|
||||
expect(prisma.webhookDelivery.createMany).toHaveBeenCalledWith({
|
||||
data: [
|
||||
{ subscriptionId: 'wh-1', event: 'colony.created', body },
|
||||
{ subscriptionId: 'wh-2', event: 'colony.created', body },
|
||||
],
|
||||
});
|
||||
expect(prisma.$executeRaw).toHaveBeenCalledTimes(1);
|
||||
const [[strings, ...values]] = prisma.$executeRaw.mock.calls as [
|
||||
[TemplateStringsArray, ...unknown[]],
|
||||
];
|
||||
expect(strings.join('$')).toBe("SELECT pg_notify($, '')");
|
||||
expect(values).toEqual(['webhook_delivery']);
|
||||
});
|
||||
});
|
||||
|
||||
it('dispatch signs timestamp, delivery id and payload with HMAC-SHA256', async () => {
|
||||
const subscription = {
|
||||
id: 'wh-1',
|
||||
url: 'https://example.test/hook',
|
||||
secret: sealWebhookSecret('topsecret'),
|
||||
active: true,
|
||||
};
|
||||
prisma.webhookSubscription.findMany.mockResolvedValueOnce([subscription]);
|
||||
it('writes nothing and wakes nobody without a matching subscription', async () => {
|
||||
await expect(
|
||||
service.enqueueDeliveries(db(), 'task.created', { id: 't1' }),
|
||||
).resolves.toBe(0);
|
||||
|
||||
await service.dispatch('colony.created', { id: 'colony-1' });
|
||||
await flushDeliveries();
|
||||
expect(prisma.webhookDelivery.createMany).not.toHaveBeenCalled();
|
||||
expect(prisma.$executeRaw).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
expect(postWebhookMock).toHaveBeenCalledTimes(1);
|
||||
const [url, headers, body] = postWebhookMock.mock.calls[0];
|
||||
expect(url).toBe('https://example.test/hook');
|
||||
expect(headers['X-ApiCore-Event']).toBe('colony.created');
|
||||
expect(headers['X-ApiCore-Delivery']).toMatch(/^[0-9a-f-]{36}$/);
|
||||
expect(Number(headers['X-ApiCore-Timestamp'])).toBeGreaterThan(0);
|
||||
it('works on the client it is given, e.g. a handler transaction', async () => {
|
||||
const tx = {
|
||||
webhookSubscription: {
|
||||
findMany: jest.fn().mockResolvedValue([{ id: 'wh-1' }]),
|
||||
},
|
||||
webhookDelivery: {
|
||||
createMany: jest.fn().mockResolvedValue({ count: 1 }),
|
||||
},
|
||||
$executeRaw: jest.fn().mockResolvedValue(1),
|
||||
};
|
||||
|
||||
const expectedSignature = crypto
|
||||
.createHmac('sha256', 'topsecret')
|
||||
.update(
|
||||
`${headers['X-ApiCore-Timestamp']}.${headers['X-ApiCore-Delivery']}.${body}`,
|
||||
)
|
||||
.digest('hex');
|
||||
expect(headers['X-ApiCore-Signature']).toBe(`sha256=${expectedSignature}`);
|
||||
});
|
||||
|
||||
it('dispatch returns before a slow receiver has answered', async () => {
|
||||
prisma.webhookSubscription.findMany.mockResolvedValueOnce([
|
||||
{ id: 'wh-1', url: 'https://example.test/hook', secret: 's' },
|
||||
]);
|
||||
postWebhookMock.mockReturnValueOnce(new Promise(() => undefined));
|
||||
|
||||
await expect(
|
||||
service.dispatch('colony.created', { id: 'colony-1' }),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('dispatch does not throw when delivery fails', async () => {
|
||||
prisma.webhookSubscription.findMany.mockResolvedValueOnce([
|
||||
{
|
||||
id: 'wh-1',
|
||||
url: 'https://example.test/hook',
|
||||
secret: 's',
|
||||
active: true,
|
||||
},
|
||||
]);
|
||||
postWebhookMock.mockRejectedValueOnce(new Error('network down'));
|
||||
|
||||
await expect(
|
||||
service.dispatch('colony.created', { id: 'colony-1' }),
|
||||
).resolves.toBeUndefined();
|
||||
await flushDeliveries();
|
||||
});
|
||||
|
||||
// dispatch() runs after the event's record is committed: a failure must
|
||||
// not fail that request (the client would retry and store it twice).
|
||||
describe('dispatch after commit', () => {
|
||||
let errorSpy: jest.SpyInstance;
|
||||
|
||||
beforeEach(() => {
|
||||
errorSpy = jest
|
||||
.spyOn(Logger.prototype, 'error')
|
||||
.mockImplementation(() => undefined);
|
||||
prisma.webhookSubscription.findMany.mockRejectedValue(
|
||||
new Error('db down'),
|
||||
await service.enqueueDeliveries(
|
||||
tx as unknown as DbClient,
|
||||
'stock.changed',
|
||||
{ bottlingBatchIds: ['b1'], productIds: [] },
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
errorSpy.mockRestore();
|
||||
expect(tx.webhookDelivery.createMany).toHaveBeenCalledTimes(1);
|
||||
expect(tx.$executeRaw).toHaveBeenCalledTimes(1);
|
||||
expect(prisma.webhookDelivery.createMany).not.toHaveBeenCalled();
|
||||
expect(prisma.$executeRaw).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('dispatch (legacy path)', () => {
|
||||
it('enqueues on the root client after the commit and sends nothing itself', async () => {
|
||||
prisma.webhookSubscription.findMany.mockResolvedValueOnce([
|
||||
{ id: 'wh-1' },
|
||||
]);
|
||||
|
||||
it('never throws when the subscriptions cannot be loaded and logs the record', async () => {
|
||||
await expect(
|
||||
service.dispatch('colony.created', { id: 'colony-1' }),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(errorSpy).toHaveBeenCalledTimes(1);
|
||||
const [message] = errorSpy.mock.calls[0] as [string];
|
||||
expect(message).toContain('colony.created colony-1');
|
||||
expect(message).toContain('db down');
|
||||
expect(prisma.webhookSubscription.findMany).toHaveBeenCalledWith({
|
||||
where: { active: true, events: { has: 'colony.created' } },
|
||||
select: { id: true },
|
||||
});
|
||||
expect(prisma.webhookDelivery.createMany).toHaveBeenCalledWith({
|
||||
data: [
|
||||
{
|
||||
subscriptionId: 'wh-1',
|
||||
event: 'colony.created',
|
||||
body: '{"event":"colony.created","data":{"id":"colony-1"}}',
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(prisma.$executeRaw).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('also logs a payload without its own id', async () => {
|
||||
await service.dispatch('stock.changed', {
|
||||
bottlingBatchIds: ['b-1'],
|
||||
productIds: [],
|
||||
// dispatch() runs after the event's record is committed: a failure must
|
||||
// not fail that request (the client would retry and store it twice).
|
||||
describe('when enqueuing fails', () => {
|
||||
let errorSpy: jest.SpyInstance;
|
||||
|
||||
beforeEach(() => {
|
||||
errorSpy = jest
|
||||
.spyOn(Logger.prototype, 'error')
|
||||
.mockImplementation(() => undefined);
|
||||
});
|
||||
|
||||
const [message] = errorSpy.mock.calls[0] as [string];
|
||||
expect(message).toContain('stock.changed');
|
||||
expect(message).toContain('db down');
|
||||
it('dispatch never throws when enqueuing fails (legacy path)', async () => {
|
||||
prisma.webhookSubscription.findMany.mockResolvedValueOnce([
|
||||
{ id: 'wh-1' },
|
||||
]);
|
||||
prisma.webhookDelivery.createMany.mockRejectedValueOnce(
|
||||
new Error('db down'),
|
||||
);
|
||||
|
||||
await expect(
|
||||
service.dispatch('colony.created', { id: 'colony-1' }),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(errorSpy).toHaveBeenCalledTimes(1);
|
||||
const [message] = errorSpy.mock.calls[0] as [string];
|
||||
expect(message).toContain('colony.created colony-1');
|
||||
expect(message).toContain('db down');
|
||||
});
|
||||
|
||||
it('never throws when the subscriptions cannot be loaded and logs the record', async () => {
|
||||
prisma.webhookSubscription.findMany.mockRejectedValueOnce(
|
||||
new Error('db down'),
|
||||
);
|
||||
|
||||
await expect(
|
||||
service.dispatch('colony.created', { id: 'colony-1' }),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
const [message] = errorSpy.mock.calls[0] as [string];
|
||||
expect(message).toContain('colony.created colony-1');
|
||||
expect(message).toContain('db down');
|
||||
});
|
||||
|
||||
it('also logs a payload without its own id', async () => {
|
||||
prisma.webhookSubscription.findMany.mockRejectedValueOnce(
|
||||
new Error('db down'),
|
||||
);
|
||||
|
||||
await service.dispatch('stock.changed', {
|
||||
bottlingBatchIds: ['b-1'],
|
||||
productIds: [],
|
||||
});
|
||||
|
||||
const [message] = errorSpy.mock.calls[0] as [string];
|
||||
expect(message).toContain('stock.changed');
|
||||
expect(message).toContain('db down');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('findDeliveries', () => {
|
||||
it('lists the newest deliveries of a subscription without their body', async () => {
|
||||
prisma.webhookSubscription.findUnique.mockResolvedValueOnce({
|
||||
id: 'wh-1',
|
||||
});
|
||||
const rows = [{ id: 'd2' }, { id: 'd1' }];
|
||||
prisma.webhookDelivery.findMany.mockResolvedValueOnce(rows);
|
||||
|
||||
await expect(service.findDeliveries('wh-1', 20)).resolves.toBe(rows);
|
||||
|
||||
expect(prisma.webhookDelivery.findMany).toHaveBeenCalledWith({
|
||||
where: { subscriptionId: 'wh-1' },
|
||||
orderBy: [{ createdAt: 'desc' }, { id: 'desc' }],
|
||||
take: 20,
|
||||
select: {
|
||||
id: true,
|
||||
event: true,
|
||||
status: true,
|
||||
attempts: true,
|
||||
lastStatusCode: true,
|
||||
lastError: true,
|
||||
createdAt: true,
|
||||
deliveredAt: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('throws NotFoundException for an unknown subscription', async () => {
|
||||
prisma.webhookSubscription.findUnique.mockResolvedValueOnce(null);
|
||||
|
||||
await expect(
|
||||
service.findDeliveries('missing', 50),
|
||||
).rejects.toBeInstanceOf(NotFoundException);
|
||||
expect(prisma.webhookDelivery.findMany).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -2,17 +2,33 @@ import { Injectable, Logger, NotFoundException } from '@nestjs/common';
|
||||
import * as crypto from 'node:crypto';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { AuditLogService } from '../audit/audit-log.service';
|
||||
import { WebhookSubscription } from '../../generated/prisma';
|
||||
import {
|
||||
WebhookDeliveryStatus,
|
||||
WebhookSubscription,
|
||||
} from '../../generated/prisma';
|
||||
import { DbClient } from '../prisma/db-client';
|
||||
import { CreateWebhookDto } from './dto/create-webhook.dto';
|
||||
import { UpdateWebhookDto } from './dto/update-webhook.dto';
|
||||
import { WebhookEvent } from './webhook-events';
|
||||
import { assertPublicWebhookUrl } from './webhook-url-guard';
|
||||
import { postWebhook } from './webhook-delivery';
|
||||
import { openWebhookSecret, sealWebhookSecret } from './webhook-secret';
|
||||
import { runAfterCommit } from '../common/after-commit';
|
||||
import { notifyWorker, WAKE_CHANNELS } from '../common/worker-wakeup';
|
||||
|
||||
const SECRET_BYTES = 32;
|
||||
|
||||
/** One entry of a subscription's delivery log - without the body. */
|
||||
export interface WebhookDeliveryView {
|
||||
id: string;
|
||||
event: string;
|
||||
status: WebhookDeliveryStatus;
|
||||
attempts: number;
|
||||
lastStatusCode: number | null;
|
||||
lastError: string | null;
|
||||
createdAt: Date;
|
||||
deliveredAt: Date | null;
|
||||
}
|
||||
|
||||
/** The record an event is about, for the log - some payloads have none. */
|
||||
function payloadId(payload: unknown): string {
|
||||
if (
|
||||
@@ -146,12 +162,46 @@ export class WebhooksService {
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts delivery and returns without waiting for the receivers: a slow
|
||||
* or hanging endpoint must never hold up the request that caused the
|
||||
* event. Failures are logged by deliver().
|
||||
* Queues one WebhookDelivery per active subscription of `event` and wakes
|
||||
* the worker, which sends them (webhook-delivery.worker.ts) - nothing is
|
||||
* sent here. The body is fixed now, so every retry sends the same bytes
|
||||
* under the same delivery id. On a transaction client the rows commit (and
|
||||
* the wake-up fires) with the caller's transaction, e.g. a domain event
|
||||
* handler's. Returns how many deliveries were queued.
|
||||
*/
|
||||
async enqueueDeliveries(
|
||||
db: DbClient,
|
||||
event: WebhookEvent,
|
||||
payload: unknown,
|
||||
): Promise<number> {
|
||||
const subscriptions = await db.webhookSubscription.findMany({
|
||||
where: { active: true, events: { has: event } },
|
||||
select: { id: true },
|
||||
});
|
||||
if (subscriptions.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
const body = JSON.stringify({ event, data: payload });
|
||||
const { count } = await db.webhookDelivery.createMany({
|
||||
data: subscriptions.map((subscription) => ({
|
||||
subscriptionId: subscription.id,
|
||||
event,
|
||||
body,
|
||||
})),
|
||||
});
|
||||
if (count > 0) {
|
||||
await notifyWorker(db, WAKE_CHANNELS.webhookDeliveries);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Legacy path for producers that do not record a domain event yet: queues
|
||||
* the deliveries right after the record was committed. A slow or hanging
|
||||
* receiver can never hold up the request - the worker sends.
|
||||
*
|
||||
* Called after the event's record is committed, so it never throws - not
|
||||
* even when the subscriptions cannot be loaded (see runAfterCommit).
|
||||
* even when the deliveries cannot be queued (see runAfterCommit).
|
||||
*/
|
||||
async dispatch(event: WebhookEvent, payload: unknown): Promise<void> {
|
||||
await runAfterCommit(
|
||||
@@ -162,51 +212,38 @@ export class WebhooksService {
|
||||
entityId: payloadId(payload),
|
||||
},
|
||||
async () => {
|
||||
const subscriptions = await this.prisma.webhookSubscription.findMany({
|
||||
where: { active: true, events: { has: event } },
|
||||
});
|
||||
for (const subscription of subscriptions) {
|
||||
void this.deliver(subscription, event, payload);
|
||||
}
|
||||
await this.enqueueDeliveries(this.prisma, event, payload);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async deliver(
|
||||
subscription: WebhookSubscription,
|
||||
event: WebhookEvent,
|
||||
payload: unknown,
|
||||
): Promise<void> {
|
||||
const body = JSON.stringify({ event, data: payload });
|
||||
// Signed together with a timestamp and a unique delivery id, so a
|
||||
// receiver can reject a captured request replayed later
|
||||
// ("t.id.body", the same idea as Stripe's signatures).
|
||||
const deliveryId = crypto.randomUUID();
|
||||
const timestamp = Math.floor(Date.now() / 1000).toString();
|
||||
try {
|
||||
const signature = crypto
|
||||
.createHmac('sha256', openWebhookSecret(subscription.secret))
|
||||
.update(`${timestamp}.${deliveryId}.${body}`)
|
||||
.digest('hex');
|
||||
// No redirects, 5 s timeout, connect-time address check - see
|
||||
// postWebhook().
|
||||
await postWebhook(
|
||||
subscription.url,
|
||||
{
|
||||
'Content-Type': 'application/json',
|
||||
'User-Agent': 'ApiCore-Webhooks/1',
|
||||
'X-ApiCore-Event': event,
|
||||
'X-ApiCore-Delivery': deliveryId,
|
||||
'X-ApiCore-Timestamp': timestamp,
|
||||
'X-ApiCore-Signature': `sha256=${signature}`,
|
||||
},
|
||||
body,
|
||||
);
|
||||
} catch (error: unknown) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
this.logger.warn(
|
||||
`Webhook delivery to ${subscription.url} failed for event ${event}: ${message}`,
|
||||
);
|
||||
/** The newest deliveries of a subscription, for its delivery log. */
|
||||
async findDeliveries(
|
||||
subscriptionId: string,
|
||||
limit: number,
|
||||
): Promise<WebhookDeliveryView[]> {
|
||||
const subscription = await this.prisma.webhookSubscription.findUnique({
|
||||
where: { id: subscriptionId },
|
||||
select: { id: true },
|
||||
});
|
||||
if (!subscription) {
|
||||
throw new NotFoundException('Webhook subscription not found');
|
||||
}
|
||||
// Never the body: it holds the full record the event was about.
|
||||
return this.prisma.webhookDelivery.findMany({
|
||||
where: { subscriptionId },
|
||||
orderBy: [{ createdAt: 'desc' }, { id: 'desc' }],
|
||||
take: limit,
|
||||
select: {
|
||||
id: true,
|
||||
event: true,
|
||||
status: true,
|
||||
attempts: true,
|
||||
lastStatusCode: true,
|
||||
lastError: true,
|
||||
createdAt: true,
|
||||
deliveredAt: true,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user