feat: T4b/T4c wave 9 - phenology reports, autopilot A4-A10, honey ledger, Telegram delivery
- Phenology reports API with corrections and revokes; recompute per apiary via PhenologyRecomputeRequested, allowBackward for corrections; season changes are announced in the morning digest (changes since the previous digest, each once) - Season autopilot A4-A10 with per-apiary sync (Apiary before Colony locks) and WinterCheckRecorded - Honey batch ledger at runtime: initial amount, BOTTLING bookings, manual movements, one gram rounding, untracked bottlings never book - Telegram delivery: planning, Berlin delivery times incl. DST, quiet hours, permission re-check and stale skip at send time, rate limits, cleanup of old deliveries - ARCHITECTURE.md: assumptions 47-50 and updated sections 11.x Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
1 parent
1971b2d9c4
commit
16cc557462
74 files changed
+9634
-340
No files matched your search
@@ -0,0 +1,892 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import type { PermissionResolverService } from '../auth/permissions/permission-resolver.service';
|
||||
import * as claims from '../common/outbox/outbox-claims';
|
||||
import type { NotificationsService } from '../notifications/notifications.service';
|
||||
import type { PrismaService } from '../prisma/prisma.service';
|
||||
import * as rowLock from '../prisma/row-lock';
|
||||
import { TelegramApiError } from './telegram-api-error';
|
||||
import type { TelegramClient } from './telegram-client';
|
||||
import {
|
||||
TELEGRAM_BATCH_BUDGET_MS,
|
||||
TELEGRAM_CHAT_SPACING_MS,
|
||||
TELEGRAM_MAX_PER_SECOND,
|
||||
TELEGRAM_SEND_BATCH,
|
||||
TELEGRAM_SEND_POLL_MS,
|
||||
TELEGRAM_TOKEN_INVALID_PAUSE_MS,
|
||||
TelegramSender,
|
||||
} from './telegram-sender';
|
||||
import { TELEGRAM_TEMPLATES } from './telegram-texts';
|
||||
|
||||
jest.mock('../common/outbox/outbox-claims');
|
||||
jest.mock('../prisma/row-lock');
|
||||
|
||||
const claimBatch = jest.mocked(claims.claimBatch);
|
||||
const deferClaim = jest.mocked(claims.deferClaim);
|
||||
const settleSkipped = jest.mocked(claims.settleSkipped);
|
||||
const settleSuccess = jest.mocked(claims.settleSuccess);
|
||||
const settleFailure = jest.mocked(claims.settleFailure);
|
||||
const releaseStaleClaimRows = jest.mocked(claims.releaseStaleClaimRows);
|
||||
const lockRowForUpdate = jest.mocked(rowLock.lockRowForUpdate);
|
||||
|
||||
const CLAIMED_AT = new Date('2026-10-14T05:00:00.000Z');
|
||||
const CHAT = 4_711_000_001n;
|
||||
const ORIGIN = 'https://app.example.org';
|
||||
|
||||
interface Row {
|
||||
id: string;
|
||||
userId: string | null;
|
||||
chatId: bigint | null;
|
||||
notificationId: string | null;
|
||||
templateCode: string | null;
|
||||
attempts: number;
|
||||
claimedAt: Date;
|
||||
}
|
||||
|
||||
const delivery = (id: string, userId = `u-${id}`, attempts = 0): Row => ({
|
||||
id,
|
||||
userId,
|
||||
chatId: null,
|
||||
notificationId: `n-${id}`,
|
||||
templateCode: null,
|
||||
attempts,
|
||||
claimedAt: CLAIMED_AT,
|
||||
});
|
||||
|
||||
const template = (id: string, code: string, chatId = CHAT): Row => ({
|
||||
id,
|
||||
userId: null,
|
||||
chatId,
|
||||
notificationId: null,
|
||||
templateCode: code,
|
||||
attempts: 0,
|
||||
claimedAt: CLAIMED_AT,
|
||||
});
|
||||
|
||||
const context = (overrides: Record<string, unknown> = {}) => ({
|
||||
userActive: true,
|
||||
telegramEnabled: true,
|
||||
categories: ['TASK_DUE', 'DIGEST'],
|
||||
detail: true,
|
||||
chatId: CHAT,
|
||||
category: 'TASK_DUE',
|
||||
message: 'Heute fällig: Futter bestellen',
|
||||
link: '/tasks',
|
||||
requiredPermissions: ['tasks:read'],
|
||||
stale: false,
|
||||
taskOpen: true,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe('TelegramSender', () => {
|
||||
let prisma: { $queryRaw: jest.Mock; $transaction: jest.Mock };
|
||||
let tx: {
|
||||
telegramLink: { deleteMany: jest.Mock };
|
||||
notificationPreference: { updateMany: jest.Mock };
|
||||
};
|
||||
let client: {
|
||||
sendMessage: jest.Mock<Promise<void>, [bigint, string, AbortSignal?]>;
|
||||
};
|
||||
let permissions: {
|
||||
getFreshPermissions: jest.Mock;
|
||||
getPermissions: jest.Mock;
|
||||
};
|
||||
let notifications: { record: jest.Mock; recordForPermission: jest.Mock };
|
||||
let sender: TelegramSender;
|
||||
let signal: AbortSignal;
|
||||
let logError: jest.SpyInstance;
|
||||
|
||||
/** Queue the claim and the answers of the send-time check, in order. */
|
||||
function claim(rows: Row[], ...contexts: (object | null)[]): void {
|
||||
claimBatch.mockResolvedValueOnce(rows);
|
||||
for (const answer of contexts) {
|
||||
prisma.$queryRaw.mockResolvedValueOnce(answer === null ? [] : [answer]);
|
||||
}
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
jest.resetAllMocks();
|
||||
process.env.FRONTEND_ORIGIN = ORIGIN;
|
||||
prisma = {
|
||||
$queryRaw: jest.fn(),
|
||||
$transaction: jest.fn(),
|
||||
};
|
||||
tx = {
|
||||
telegramLink: { deleteMany: jest.fn().mockResolvedValue({ count: 1 }) },
|
||||
notificationPreference: {
|
||||
updateMany: jest.fn().mockResolvedValue({ count: 1 }),
|
||||
},
|
||||
};
|
||||
prisma.$transaction.mockImplementation((work: (t: unknown) => unknown) =>
|
||||
Promise.resolve(work(tx)),
|
||||
);
|
||||
client = {
|
||||
sendMessage: jest
|
||||
.fn<Promise<void>, [bigint, string, AbortSignal?]>()
|
||||
.mockResolvedValue(undefined),
|
||||
};
|
||||
permissions = {
|
||||
getFreshPermissions: jest.fn().mockResolvedValue(new Set(['tasks:read'])),
|
||||
getPermissions: jest.fn(),
|
||||
};
|
||||
notifications = {
|
||||
record: jest.fn().mockResolvedValue({ id: 'n' }),
|
||||
recordForPermission: jest.fn().mockResolvedValue([]),
|
||||
};
|
||||
claimBatch.mockResolvedValue([]);
|
||||
deferClaim.mockResolvedValue(true);
|
||||
settleSkipped.mockResolvedValue(true);
|
||||
settleSuccess.mockResolvedValue(true);
|
||||
settleFailure.mockResolvedValue('retry');
|
||||
releaseStaleClaimRows.mockResolvedValue([]);
|
||||
lockRowForUpdate.mockResolvedValue(true);
|
||||
signal = new AbortController().signal;
|
||||
logError = jest
|
||||
.spyOn(Logger.prototype, 'error')
|
||||
.mockImplementation(() => undefined);
|
||||
sender = new TelegramSender(
|
||||
prisma as unknown as PrismaService,
|
||||
client as unknown as TelegramClient,
|
||||
permissions as unknown as PermissionResolverService,
|
||||
notifications as unknown as NotificationsService,
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.restoreAllMocks();
|
||||
delete process.env.FRONTEND_ORIGIN;
|
||||
});
|
||||
|
||||
it('exposes the documented limits', () => {
|
||||
expect(TELEGRAM_SEND_POLL_MS).toBe(5_000);
|
||||
expect(TELEGRAM_SEND_BATCH).toBe(25);
|
||||
expect(TELEGRAM_MAX_PER_SECOND).toBe(25);
|
||||
expect(TELEGRAM_CHAT_SPACING_MS).toBe(1_100);
|
||||
expect(TELEGRAM_TOKEN_INVALID_PAUSE_MS).toBe(900_000);
|
||||
});
|
||||
|
||||
describe('claiming', () => {
|
||||
it('claims a batch of 25 oldest first and returns how many it got', async () => {
|
||||
claim([delivery('d1')], context());
|
||||
|
||||
const claimed = await sender.runOnce(signal);
|
||||
|
||||
expect(claimed).toBe(1);
|
||||
expect(claimBatch).toHaveBeenCalledWith(prisma, 'NotificationDelivery', {
|
||||
limit: 25,
|
||||
orderBy: 'createdAt',
|
||||
filter: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it('sends only the oldest delivery of a user and puts the rest back for 1.1 s', async () => {
|
||||
claim([delivery('d1', 'u1'), delivery('d2', 'u1')], context());
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(client.sendMessage).toHaveBeenCalledTimes(1);
|
||||
expect(settleSuccess).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
'd1',
|
||||
CLAIMED_AT,
|
||||
'DELIVERED',
|
||||
);
|
||||
expect(deferClaim).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
'd2',
|
||||
CLAIMED_AT,
|
||||
1_100,
|
||||
);
|
||||
expect(settleFailure).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('treats replies to one chat like one target', async () => {
|
||||
claim([
|
||||
template('t1', 'LINKED'),
|
||||
template('t2', 'HELP'),
|
||||
template('t3', 'STOPPED', 5n),
|
||||
]);
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(client.sendMessage.mock.calls.map((call) => call[0])).toEqual([
|
||||
CHAT,
|
||||
5n,
|
||||
]);
|
||||
expect(deferClaim).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
't2',
|
||||
CLAIMED_AT,
|
||||
1_100,
|
||||
);
|
||||
});
|
||||
|
||||
it('limits the claim to given users', async () => {
|
||||
claim([]);
|
||||
|
||||
await sender.runOnce(signal, { userIds: ['u1', 'u2'] });
|
||||
|
||||
const options = claimBatch.mock.calls[0][2];
|
||||
expect(options.filter).toBeDefined();
|
||||
});
|
||||
|
||||
it('claims nothing once the signal is aborted', async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
expect(await sender.runOnce(controller.signal)).toBe(0);
|
||||
expect(claimBatch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('puts claimed deliveries back without an attempt when the signal aborts mid-batch', async () => {
|
||||
const controller = new AbortController();
|
||||
claim([delivery('d1'), delivery('d2')], context());
|
||||
client.sendMessage.mockImplementation(() => {
|
||||
controller.abort();
|
||||
return Promise.resolve();
|
||||
});
|
||||
|
||||
await sender.runOnce(controller.signal);
|
||||
|
||||
expect(settleSuccess).toHaveBeenCalledTimes(1);
|
||||
expect(deferClaim).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
'd2',
|
||||
CLAIMED_AT,
|
||||
0,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('checks at send time', () => {
|
||||
it.each([
|
||||
['user inactive', { userActive: false }],
|
||||
['telegram disabled', { telegramEnabled: false }],
|
||||
['not linked', { chatId: null }],
|
||||
['category off', { categories: ['DIGEST'] }],
|
||||
['stale', { stale: true }],
|
||||
['task already done', { taskOpen: false }],
|
||||
])('skips with "%s"', async (reason, overrides) => {
|
||||
claim([delivery('d1')], context(overrides));
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(client.sendMessage).not.toHaveBeenCalled();
|
||||
expect(settleSkipped).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
'd1',
|
||||
CLAIMED_AT,
|
||||
reason,
|
||||
);
|
||||
});
|
||||
|
||||
it('checks permissions and the link at send time and skips with a reason', async () => {
|
||||
permissions.getFreshPermissions.mockResolvedValue(
|
||||
new Set(['colonies:read']),
|
||||
);
|
||||
claim(
|
||||
[delivery('d1', 'u1')],
|
||||
context({ requiredPermissions: ['tasks:read', 'colonies:read'] }),
|
||||
);
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(permissions.getFreshPermissions).toHaveBeenCalledWith('u1');
|
||||
expect(client.sendMessage).not.toHaveBeenCalled();
|
||||
expect(settleSkipped).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
'd1',
|
||||
CLAIMED_AT,
|
||||
'permission revoked',
|
||||
);
|
||||
});
|
||||
|
||||
it('the permission check reads fresh, not the 30 s cache', async () => {
|
||||
claim([delivery('d1')], context());
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(permissions.getFreshPermissions).toHaveBeenCalledTimes(1);
|
||||
expect(permissions.getPermissions).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('a stale DIGEST from yesterday is SKIPPED', async () => {
|
||||
claim(
|
||||
[delivery('d1')],
|
||||
context({ category: 'DIGEST', requiredPermissions: [], stale: true }),
|
||||
);
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(client.sendMessage).not.toHaveBeenCalled();
|
||||
expect(settleSkipped).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
'd1',
|
||||
CLAIMED_AT,
|
||||
'stale',
|
||||
);
|
||||
});
|
||||
|
||||
it('asks the database once for the staleness: Berlin day for TASK_DUE and DIGEST, 24 hours for the rest', async () => {
|
||||
claim([delivery('d1')], context());
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(prisma.$queryRaw).toHaveBeenCalledTimes(1);
|
||||
const [query] = prisma.$queryRaw.mock.calls[0] as [{ sql: string }];
|
||||
expect(query.sql).toContain("'TASK_DUE', 'DIGEST'");
|
||||
expect(query.sql).toContain("AT TIME ZONE 'Europe/Berlin'");
|
||||
expect(query.sql).toContain("interval '24 hours'");
|
||||
expect(query.sql).toContain('"Task"');
|
||||
});
|
||||
|
||||
it('skips when the notification is gone', async () => {
|
||||
claim([delivery('d1')], null);
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(settleSkipped).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
'd1',
|
||||
CLAIMED_AT,
|
||||
'notification gone',
|
||||
);
|
||||
});
|
||||
|
||||
it('answers a bot reply without any of these checks', async () => {
|
||||
claim([template('t1', 'LINK_INVALID')]);
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(prisma.$queryRaw).not.toHaveBeenCalled();
|
||||
expect(permissions.getFreshPermissions).not.toHaveBeenCalled();
|
||||
expect(client.sendMessage).toHaveBeenCalledWith(
|
||||
CHAT,
|
||||
TELEGRAM_TEMPLATES.LINK_INVALID,
|
||||
signal,
|
||||
);
|
||||
expect(settleSuccess).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
't1',
|
||||
CLAIMED_AT,
|
||||
'DELIVERED',
|
||||
);
|
||||
});
|
||||
|
||||
it('skips a reply with an unknown template code', async () => {
|
||||
claim([template('t1', 'NOPE')]);
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(client.sendMessage).not.toHaveBeenCalled();
|
||||
expect(settleSkipped).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
't1',
|
||||
CLAIMED_AT,
|
||||
'unknown template',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('text', () => {
|
||||
it('sends plain text of at most 600 characters with the app link', async () => {
|
||||
claim([delivery('d1')], context({ message: 'x'.repeat(2_000) }));
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
const [chatId, text] = client.sendMessage.mock.calls[0] as [
|
||||
bigint,
|
||||
string,
|
||||
AbortSignal,
|
||||
];
|
||||
expect(chatId).toBe(CHAT);
|
||||
expect([...text].length).toBeLessThanOrEqual(600);
|
||||
expect(text.endsWith(`\n\n${ORIGIN}/tasks`)).toBe(true);
|
||||
});
|
||||
|
||||
it('sends only a hint and the link when the user turned details off', async () => {
|
||||
claim([delivery('d1')], context({ detail: false }));
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(client.sendMessage.mock.calls[0][1]).toBe(
|
||||
`Neue Meldung in ApiCore.\n\n${ORIGIN}/tasks`,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('answers of Telegram', () => {
|
||||
it('200 settles as DELIVERED', async () => {
|
||||
claim([delivery('d1')], context());
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(settleSuccess).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
'd1',
|
||||
CLAIMED_AT,
|
||||
'DELIVERED',
|
||||
);
|
||||
});
|
||||
|
||||
it('429 defers by retry_after without counting an attempt', async () => {
|
||||
claim([delivery('d1', 'u1', 2)], context());
|
||||
client.sendMessage.mockRejectedValue(
|
||||
new TelegramApiError('sendMessage', 429, 'Too Many Requests', 17),
|
||||
);
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(deferClaim).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
'd1',
|
||||
CLAIMED_AT,
|
||||
17_000,
|
||||
);
|
||||
expect(settleFailure).not.toHaveBeenCalled();
|
||||
expect(settleSkipped).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('403 removes the link, switches Telegram off and notifies the user in the app', async () => {
|
||||
claim([delivery('d1', 'u1')], context());
|
||||
client.sendMessage.mockRejectedValue(
|
||||
new TelegramApiError(
|
||||
'sendMessage',
|
||||
403,
|
||||
'Forbidden: bot was blocked',
|
||||
null,
|
||||
),
|
||||
);
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(lockRowForUpdate).toHaveBeenCalledWith(
|
||||
tx,
|
||||
'NotificationPreference',
|
||||
'u1',
|
||||
);
|
||||
expect(tx.telegramLink.deleteMany).toHaveBeenCalledWith({
|
||||
where: { userId: 'u1', chatId: CHAT },
|
||||
});
|
||||
expect(tx.notificationPreference.updateMany).toHaveBeenCalledWith({
|
||||
where: { userId: 'u1' },
|
||||
data: { telegramEnabled: false },
|
||||
});
|
||||
expect(notifications.record).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
userId: 'u1',
|
||||
category: 'SYSTEM',
|
||||
message:
|
||||
'Telegram-Zustellung beendet. Der Bot wurde blockiert oder der Chat gelöscht.',
|
||||
}),
|
||||
tx,
|
||||
);
|
||||
expect(settleSkipped).toHaveBeenCalledWith(
|
||||
tx,
|
||||
'NotificationDelivery',
|
||||
'd1',
|
||||
CLAIMED_AT,
|
||||
'chat unavailable',
|
||||
);
|
||||
});
|
||||
|
||||
it('400 "chat not found" ends the link like a 403', async () => {
|
||||
claim([delivery('d1', 'u1')], context());
|
||||
client.sendMessage.mockRejectedValue(
|
||||
new TelegramApiError(
|
||||
'sendMessage',
|
||||
400,
|
||||
'Bad Request: chat not found',
|
||||
null,
|
||||
),
|
||||
);
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(tx.telegramLink.deleteMany).toHaveBeenCalledTimes(1);
|
||||
expect(settleSkipped).toHaveBeenCalledWith(
|
||||
tx,
|
||||
'NotificationDelivery',
|
||||
'd1',
|
||||
CLAIMED_AT,
|
||||
'chat unavailable',
|
||||
);
|
||||
});
|
||||
|
||||
it('any other 400 is a failed attempt, not the end of the link', async () => {
|
||||
claim([delivery('d1', 'u1', 1)], context());
|
||||
client.sendMessage.mockRejectedValue(
|
||||
new TelegramApiError(
|
||||
'sendMessage',
|
||||
400,
|
||||
'Bad Request: message is too long',
|
||||
null,
|
||||
),
|
||||
);
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(tx.telegramLink.deleteMany).not.toHaveBeenCalled();
|
||||
expect(settleFailure).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
'd1',
|
||||
CLAIMED_AT,
|
||||
1,
|
||||
'Telegram sendMessage failed: Bad Request: message is too long',
|
||||
);
|
||||
});
|
||||
|
||||
it('a 403 for an old chat does not remove a newer link of the same user', async () => {
|
||||
tx.telegramLink.deleteMany.mockResolvedValue({ count: 0 });
|
||||
claim([delivery('d1', 'u1')], context());
|
||||
client.sendMessage.mockRejectedValue(
|
||||
new TelegramApiError('sendMessage', 403, 'Forbidden', null),
|
||||
);
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(tx.notificationPreference.updateMany).not.toHaveBeenCalled();
|
||||
expect(notifications.record).not.toHaveBeenCalled();
|
||||
expect(settleSkipped).toHaveBeenCalledWith(
|
||||
tx,
|
||||
'NotificationDelivery',
|
||||
'd1',
|
||||
CLAIMED_AT,
|
||||
'chat unavailable',
|
||||
);
|
||||
});
|
||||
|
||||
it('a 403 for a bot reply only skips it', async () => {
|
||||
claim([template('t1', 'LINKED')]);
|
||||
client.sendMessage.mockRejectedValue(
|
||||
new TelegramApiError('sendMessage', 403, 'Forbidden', null),
|
||||
);
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(prisma.$transaction).not.toHaveBeenCalled();
|
||||
expect(settleSkipped).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
't1',
|
||||
CLAIMED_AT,
|
||||
'chat unavailable',
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['5xx', new TelegramApiError('sendMessage', 502, 'Bad Gateway', null)],
|
||||
['timeout', new TelegramApiError('sendMessage', null, 'timeout', null)],
|
||||
[
|
||||
'network error',
|
||||
new TelegramApiError('sendMessage', null, 'network error', null),
|
||||
],
|
||||
])('%s is a failed attempt under the retry rule', async (_name, error) => {
|
||||
claim([delivery('d1', 'u1', 3)], context());
|
||||
client.sendMessage.mockRejectedValue(error);
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(settleFailure).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
'd1',
|
||||
CLAIMED_AT,
|
||||
3,
|
||||
error.message,
|
||||
);
|
||||
expect(settleSuccess).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('settles an unexpected error as a failed attempt and goes on with the batch', async () => {
|
||||
prisma.$queryRaw.mockRejectedValueOnce(new Error('connection reset'));
|
||||
claim([delivery('d1'), delivery('d2')], context());
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(settleFailure).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
'd1',
|
||||
CLAIMED_AT,
|
||||
0,
|
||||
'connection reset',
|
||||
);
|
||||
expect(client.sendMessage).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('logs a failed settle instead of throwing', async () => {
|
||||
claim([delivery('d1')], context());
|
||||
client.sendMessage.mockRejectedValue(
|
||||
new TelegramApiError('sendMessage', 500, 'oops', null),
|
||||
);
|
||||
settleFailure.mockRejectedValue(new Error('db down'));
|
||||
|
||||
await expect(sender.runOnce(signal)).resolves.toBe(1);
|
||||
|
||||
expect(logError).toHaveBeenCalledWith(
|
||||
expect.stringContaining('d1 not settled: db down'),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('invalid token', () => {
|
||||
const unauthorized = () =>
|
||||
new TelegramApiError('sendMessage', 401, 'Unauthorized', null);
|
||||
|
||||
it('401 keeps the delivery pending, pauses 15 min and notifies system:manage once a day', async () => {
|
||||
claim([delivery('d1'), delivery('d2')], context());
|
||||
client.sendMessage.mockRejectedValue(unauthorized());
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(deferClaim).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
'd1',
|
||||
CLAIMED_AT,
|
||||
900_000,
|
||||
);
|
||||
expect(settleFailure).not.toHaveBeenCalled();
|
||||
expect(settleSkipped).not.toHaveBeenCalled();
|
||||
expect(notifications.recordForPermission).toHaveBeenCalledWith(
|
||||
'system:manage',
|
||||
expect.objectContaining({
|
||||
category: 'SYSTEM',
|
||||
priority: 'WARNUNG',
|
||||
message: 'Telegram: Bot-Token ungültig.',
|
||||
link: '/settings/system',
|
||||
dedupeKeyPrefix: expect.stringMatching(
|
||||
/^telegram-token-invalid:\d{4}-\d{2}-\d{2}$/,
|
||||
) as string,
|
||||
}),
|
||||
);
|
||||
// The second delivery of the batch is not tried with the bad token.
|
||||
expect(client.sendMessage).toHaveBeenCalledTimes(1);
|
||||
expect(deferClaim).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
'd2',
|
||||
CLAIMED_AT,
|
||||
expect.any(Number),
|
||||
);
|
||||
});
|
||||
|
||||
it('claims nothing while paused', async () => {
|
||||
claim([delivery('d1')], context());
|
||||
client.sendMessage.mockRejectedValue(unauthorized());
|
||||
await sender.runOnce(signal);
|
||||
claimBatch.mockClear();
|
||||
|
||||
expect(await sender.runOnce(signal)).toBe(0);
|
||||
expect(await sender.drain(signal)).toBe(0);
|
||||
expect(claimBatch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not let a failing notice break the settle', async () => {
|
||||
claim([delivery('d1')], context());
|
||||
client.sendMessage.mockRejectedValue(unauthorized());
|
||||
notifications.recordForPermission.mockRejectedValue(new Error('db down'));
|
||||
|
||||
await expect(sender.runOnce(signal)).resolves.toBe(1);
|
||||
|
||||
expect(deferClaim).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('never puts the token or a chat id into the log', async () => {
|
||||
claim([delivery('d1')], context());
|
||||
client.sendMessage.mockRejectedValue(unauthorized());
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
const logged = JSON.stringify(logError.mock.calls);
|
||||
expect(logged).not.toContain(String(CHAT));
|
||||
});
|
||||
});
|
||||
|
||||
describe('rates', () => {
|
||||
it('keeps 1.1 s between two messages to the same chat', async () => {
|
||||
claim([delivery('d1', 'u1')], context());
|
||||
await sender.runOnce(signal);
|
||||
deferClaim.mockClear();
|
||||
claim([delivery('d2', 'u1')], context());
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(client.sendMessage).toHaveBeenCalledTimes(1);
|
||||
const [, , id, , delay] = deferClaim.mock.calls[0];
|
||||
expect(id).toBe('d2');
|
||||
expect(delay).toBeGreaterThan(0);
|
||||
expect(delay).toBeLessThanOrEqual(1_100);
|
||||
});
|
||||
|
||||
it('sends at most 25 messages per second over all chats', async () => {
|
||||
const rows = Array.from({ length: 4 }, (_, i) => delivery(`d${i}`));
|
||||
claim(rows, ...rows.map((_, i) => context({ chatId: BigInt(i + 1) })));
|
||||
const started = performance.now();
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
// Four messages need three gaps of 40 ms.
|
||||
expect(performance.now() - started).toBeGreaterThanOrEqual(100);
|
||||
expect(client.sendMessage).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
|
||||
it('drain waits the chat spacing after a batch that held messages back, then sends the next', async () => {
|
||||
claim([delivery('d1', 'u1'), delivery('d2', 'u1')], context());
|
||||
claim([delivery('d2', 'u1')], context());
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const done = sender.drain(signal);
|
||||
await jest.advanceTimersByTimeAsync(2_000);
|
||||
await expect(done).resolves.toBe(3);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('drain stops after maxRounds', async () => {
|
||||
claimBatch.mockResolvedValue([template('t', 'HELP', 1n)]);
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const done = sender.drain(signal, 3);
|
||||
await jest.advanceTimersByTimeAsync(5_000);
|
||||
await expect(done).resolves.toBe(3);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
expect(claimBatch).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('releaseStaleClaims', () => {
|
||||
it('returns how many claims were released and logs the FAILED ones', async () => {
|
||||
releaseStaleClaimRows.mockResolvedValue([
|
||||
{ id: 'a', status: 'PENDING' },
|
||||
{ id: 'b', status: 'FAILED' },
|
||||
]);
|
||||
|
||||
expect(await sender.releaseStaleClaims()).toBe(2);
|
||||
expect(releaseStaleClaimRows).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
);
|
||||
expect(logError).toHaveBeenCalledWith(expect.stringContaining('b'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('fix round 1', () => {
|
||||
it('429 pauses the whole pod for retry_after, not only the row', async () => {
|
||||
claim([delivery('d1'), delivery('d2')], context());
|
||||
client.sendMessage.mockRejectedValue(
|
||||
new TelegramApiError('sendMessage', 429, 'Too Many Requests', 30),
|
||||
);
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
// The second delivery of the batch is put back, not tried.
|
||||
expect(client.sendMessage).toHaveBeenCalledTimes(1);
|
||||
expect(deferClaim).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
'd2',
|
||||
CLAIMED_AT,
|
||||
expect.any(Number),
|
||||
);
|
||||
claimBatch.mockClear();
|
||||
expect(await sender.runOnce(signal)).toBe(0);
|
||||
expect(claimBatch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('a batch stops after about 120 s and puts the rest back without an attempt', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
claim(
|
||||
[delivery('d1'), delivery('d2'), delivery('d3')],
|
||||
context({ chatId: 1n }),
|
||||
context({ chatId: 2n }),
|
||||
context({ chatId: 3n }),
|
||||
);
|
||||
client.sendMessage.mockImplementationOnce(async () => {
|
||||
jest.advanceTimersByTime(TELEGRAM_BATCH_BUDGET_MS + 1);
|
||||
return Promise.resolve();
|
||||
});
|
||||
|
||||
const done = sender.runOnce(signal);
|
||||
await jest.advanceTimersByTimeAsync(100);
|
||||
await done;
|
||||
|
||||
expect(client.sendMessage).toHaveBeenCalledTimes(1);
|
||||
for (const id of ['d2', 'd3']) {
|
||||
expect(deferClaim).toHaveBeenCalledWith(
|
||||
prisma,
|
||||
'NotificationDelivery',
|
||||
id,
|
||||
CLAIMED_AT,
|
||||
0,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
describe('missing FRONTEND_ORIGIN', () => {
|
||||
const environment = process.env.NODE_ENV;
|
||||
let warn: jest.SpyInstance;
|
||||
|
||||
beforeEach(() => {
|
||||
delete process.env.FRONTEND_ORIGIN;
|
||||
warn = jest
|
||||
.spyOn(Logger.prototype, 'warn')
|
||||
.mockImplementation(() => undefined);
|
||||
});
|
||||
afterEach(() => {
|
||||
(process.env as Record<string, string | undefined>).NODE_ENV =
|
||||
environment;
|
||||
});
|
||||
|
||||
it('outside development sends no link and warns once', async () => {
|
||||
(process.env as Record<string, string | undefined>).NODE_ENV =
|
||||
'production';
|
||||
claim([delivery('d1')], context());
|
||||
claim([delivery('d2')], context({ chatId: 77n }));
|
||||
|
||||
await sender.runOnce(signal);
|
||||
await sender.runOnce(signal);
|
||||
|
||||
const texts = client.sendMessage.mock.calls.map((call) => call[1]);
|
||||
expect(texts).toEqual([
|
||||
'Heute fällig: Futter bestellen',
|
||||
'Heute fällig: Futter bestellen',
|
||||
]);
|
||||
expect(warn).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('development falls back to localhost', async () => {
|
||||
(process.env as Record<string, string | undefined>).NODE_ENV =
|
||||
'development';
|
||||
claim([delivery('d1')], context());
|
||||
|
||||
await sender.runOnce(signal);
|
||||
|
||||
expect(client.sendMessage.mock.calls[0][1]).toContain(
|
||||
'http://localhost:3000/tasks',
|
||||
);
|
||||
expect(warn).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,634 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { Prisma } from '../../generated/prisma';
|
||||
import { PermissionResolverService } from '../auth/permissions/permission-resolver.service';
|
||||
import { berlinToday } from '../common/dates';
|
||||
import { loggableError } from '../common/loggable-error';
|
||||
import {
|
||||
claimBatch,
|
||||
deferClaim,
|
||||
releaseStaleClaimRows,
|
||||
settleFailure,
|
||||
settleSkipped,
|
||||
settleSuccess,
|
||||
} from '../common/outbox/outbox-claims';
|
||||
import { DRAIN_MAX_ROUNDS } from '../domain-events/domain-event-dispatcher';
|
||||
import { NotificationsService } from '../notifications/notifications.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { lockRowForUpdate } from '../prisma/row-lock';
|
||||
import { TelegramApiError } from './telegram-api-error';
|
||||
import { TelegramClient } from './telegram-client';
|
||||
import {
|
||||
renderTelegramText,
|
||||
TELEGRAM_TEMPLATES,
|
||||
TelegramTemplateCode,
|
||||
} from './telegram-texts';
|
||||
|
||||
export const TELEGRAM_SEND_POLL_MS = 5_000;
|
||||
export const TELEGRAM_SEND_BATCH = 25;
|
||||
export const TELEGRAM_MAX_PER_SECOND = 25;
|
||||
export const TELEGRAM_CHAT_SPACING_MS = 1_100;
|
||||
export const TELEGRAM_TOKEN_INVALID_PAUSE_MS = 900_000;
|
||||
|
||||
const TABLE = 'NotificationDelivery';
|
||||
const MIN_SEND_GAP_MS = Math.ceil(1_000 / TELEGRAM_MAX_PER_SECOND);
|
||||
const DEFAULT_RETRY_AFTER_SEC = 5;
|
||||
/** How long the memory of "last message to this chat" is kept. */
|
||||
const CHAT_MEMORY_MS = 10_000;
|
||||
|
||||
const CHAT_GONE_MESSAGE =
|
||||
'Telegram-Zustellung beendet. Der Bot wurde blockiert oder der Chat gelöscht.';
|
||||
const TOKEN_INVALID_MESSAGE = 'Telegram: Bot-Token ungültig.';
|
||||
|
||||
/** Reasons a delivery is skipped; they land in lastError, so no personal data. */
|
||||
const SKIP = {
|
||||
userInactive: 'user inactive',
|
||||
telegramDisabled: 'telegram disabled',
|
||||
notLinked: 'not linked',
|
||||
categoryOff: 'category off',
|
||||
permissionRevoked: 'permission revoked',
|
||||
stale: 'stale',
|
||||
taskDone: 'task already done',
|
||||
chatUnavailable: 'chat unavailable',
|
||||
unknownTemplate: 'unknown template',
|
||||
notificationGone: 'notification gone',
|
||||
} as const;
|
||||
|
||||
/** A claimed NotificationDelivery row, as RETURNING * yields it. */
|
||||
interface ClaimedDelivery {
|
||||
id: string;
|
||||
userId: string | null;
|
||||
chatId: bigint | null;
|
||||
notificationId: string | null;
|
||||
templateCode: string | null;
|
||||
attempts: number;
|
||||
claimedAt: Date;
|
||||
}
|
||||
|
||||
/** What the send-time check reads about a delivery, in one query. */
|
||||
interface DeliveryContext {
|
||||
userActive: boolean;
|
||||
telegramEnabled: boolean;
|
||||
categories: string[];
|
||||
detail: boolean;
|
||||
chatId: bigint | null;
|
||||
category: string;
|
||||
message: string;
|
||||
link: string | null;
|
||||
requiredPermissions: string[];
|
||||
stale: boolean;
|
||||
taskOpen: boolean;
|
||||
}
|
||||
|
||||
type Target = { skip: string } | { chatId: bigint; text: string };
|
||||
|
||||
/** 'spaced': put back only because the chat got a message a moment ago. */
|
||||
type Outcome = 'settled' | 'spaced';
|
||||
|
||||
interface RoundResult {
|
||||
claimed: number;
|
||||
spaced: number;
|
||||
}
|
||||
|
||||
export interface TelegramSendFilter {
|
||||
/** Only deliveries of these users (e2e tests: no foreign rows). */
|
||||
userIds?: readonly string[];
|
||||
}
|
||||
|
||||
function claimFilter(filter?: TelegramSendFilter): Prisma.Sql | undefined {
|
||||
if (filter?.userIds === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
return filter.userIds.length === 0
|
||||
? Prisma.sql`FALSE`
|
||||
: Prisma.sql`"userId" IN (${Prisma.join([...filter.userIds])})`;
|
||||
}
|
||||
|
||||
function targetKey(delivery: ClaimedDelivery): string {
|
||||
return delivery.userId !== null
|
||||
? `user:${delivery.userId}`
|
||||
: `chat:${String(delivery.chatId)}`;
|
||||
}
|
||||
|
||||
/** The oldest delivery of each user or chat; the claim is ordered by age. */
|
||||
function splitOldestPerTarget(claimed: readonly ClaimedDelivery[]): {
|
||||
oldest: ClaimedDelivery[];
|
||||
rest: ClaimedDelivery[];
|
||||
} {
|
||||
const seen = new Set<string>();
|
||||
const oldest: ClaimedDelivery[] = [];
|
||||
const rest: ClaimedDelivery[] = [];
|
||||
for (const delivery of claimed) {
|
||||
const key = targetKey(delivery);
|
||||
if (seen.has(key)) {
|
||||
rest.push(delivery);
|
||||
} else {
|
||||
seen.add(key);
|
||||
oldest.push(delivery);
|
||||
}
|
||||
}
|
||||
return { oldest, rest };
|
||||
}
|
||||
|
||||
function sleep(ms: number, signal: AbortSignal): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
if (signal.aborted) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const done = (): void => {
|
||||
clearTimeout(timer);
|
||||
signal.removeEventListener('abort', done);
|
||||
resolve();
|
||||
};
|
||||
const timer = setTimeout(done, ms);
|
||||
signal.addEventListener('abort', done, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
function isChatGone(error: TelegramApiError): boolean {
|
||||
return (
|
||||
error.statusCode === 403 ||
|
||||
(error.statusCode === 400 && /chat not found/i.test(error.description))
|
||||
);
|
||||
}
|
||||
|
||||
const DEVELOPMENT_ORIGIN = 'http://localhost:3000';
|
||||
/** A batch stops after this long; the claim is stale after 5 minutes. */
|
||||
export const TELEGRAM_BATCH_BUDGET_MS = 120_000;
|
||||
|
||||
/**
|
||||
* Sends the queued Telegram deliveries (Spec 9a.6): claims up to 25 due
|
||||
* rows, takes the oldest of each user or chat, checks again at send time
|
||||
* that the message may still go out, and settles each row. Bot replies
|
||||
* (templateCode) skip the checks: they answer a person in a chat.
|
||||
* Run by the worker's telegram-send loop. ASSUMPTION: one worker replica
|
||||
* (the worker deployment has replicas: 1). The per-chat spacing, the global
|
||||
* rate limit and the pause after a 401 or 429 live in this pod's memory; with
|
||||
* several pods each would keep its own (the one-delivery-per-target rule in
|
||||
* SQL and Telegram's 429 answers would still protect the bot). Never logs the token, a chat id
|
||||
* or a message text.
|
||||
*/
|
||||
@Injectable()
|
||||
export class TelegramSender {
|
||||
private readonly logger = new Logger(TelegramSender.name);
|
||||
private pausedUntil = 0;
|
||||
private warnedMissingOrigin = false;
|
||||
private lastSendAt = 0;
|
||||
private readonly lastSendToChat = new Map<string, number>();
|
||||
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly client: TelegramClient,
|
||||
private readonly permissions: PermissionResolverService,
|
||||
private readonly notifications: NotificationsService,
|
||||
) {}
|
||||
|
||||
/** One batch. Returns how many deliveries it claimed. */
|
||||
async runOnce(
|
||||
signal: AbortSignal,
|
||||
filter?: TelegramSendFilter,
|
||||
): Promise<number> {
|
||||
return (await this.round(signal, filter)).claimed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs batches until one claims nothing, at most `maxRounds`. After a
|
||||
* batch that put messages back for the chat spacing it waits that
|
||||
* spacing, so a burst for one person does not wait for the next poll.
|
||||
*/
|
||||
async drain(
|
||||
signal: AbortSignal,
|
||||
maxRounds = DRAIN_MAX_ROUNDS,
|
||||
filter?: TelegramSendFilter,
|
||||
): Promise<number> {
|
||||
let total = 0;
|
||||
for (let round = 0; round < maxRounds && !signal.aborted; round++) {
|
||||
const result = await this.round(signal, filter);
|
||||
if (result.claimed === 0) {
|
||||
break;
|
||||
}
|
||||
total += result.claimed;
|
||||
if (result.spaced > 0) {
|
||||
await sleep(TELEGRAM_CHAT_SPACING_MS, signal);
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
/** Puts deliveries whose worker died back in line (or to FAILED). */
|
||||
async releaseStaleClaims(): Promise<number> {
|
||||
const released = await releaseStaleClaimRows(this.prisma, TABLE);
|
||||
const failed = released
|
||||
.filter((claim) => claim.status === 'FAILED')
|
||||
.map((claim) => claim.id);
|
||||
if (failed.length > 0) {
|
||||
this.logger.error(
|
||||
`Telegram deliveries FAILED, an expired claim used up their attempts: ${failed.join(', ')}`,
|
||||
);
|
||||
}
|
||||
return released.length;
|
||||
}
|
||||
|
||||
private async round(
|
||||
signal: AbortSignal,
|
||||
filter?: TelegramSendFilter,
|
||||
): Promise<RoundResult> {
|
||||
if (signal.aborted || this.remainingPauseMs() > 0) {
|
||||
return { claimed: 0, spaced: 0 };
|
||||
}
|
||||
const claimed = await claimBatch<ClaimedDelivery>(this.prisma, TABLE, {
|
||||
limit: TELEGRAM_SEND_BATCH,
|
||||
orderBy: 'createdAt',
|
||||
filter: claimFilter(filter),
|
||||
});
|
||||
const { oldest, rest } = splitOldestPerTarget(claimed);
|
||||
let spaced = rest.length;
|
||||
for (const delivery of rest) {
|
||||
await deferClaim(
|
||||
this.prisma,
|
||||
TABLE,
|
||||
delivery.id,
|
||||
delivery.claimedAt,
|
||||
TELEGRAM_CHAT_SPACING_MS,
|
||||
);
|
||||
}
|
||||
const started = performance.now();
|
||||
for (const delivery of oldest) {
|
||||
if (performance.now() - started > TELEGRAM_BATCH_BUDGET_MS) {
|
||||
// Out of time: the claim would go stale. Back without an attempt.
|
||||
await deferClaim(
|
||||
this.prisma,
|
||||
TABLE,
|
||||
delivery.id,
|
||||
delivery.claimedAt,
|
||||
0,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const outcome = await this.handle(delivery, signal);
|
||||
if (outcome === 'spaced') {
|
||||
spaced++;
|
||||
}
|
||||
}
|
||||
return { claimed: claimed.length, spaced };
|
||||
}
|
||||
|
||||
/** Shutdown and a paused bot put the claim back without an attempt. */
|
||||
private async handle(
|
||||
delivery: ClaimedDelivery,
|
||||
signal: AbortSignal,
|
||||
): Promise<Outcome> {
|
||||
const pauseMs = this.remainingPauseMs();
|
||||
if (signal.aborted || pauseMs > 0) {
|
||||
await deferClaim(
|
||||
this.prisma,
|
||||
TABLE,
|
||||
delivery.id,
|
||||
delivery.claimedAt,
|
||||
pauseMs,
|
||||
);
|
||||
return 'settled';
|
||||
}
|
||||
try {
|
||||
return await this.process(delivery, signal);
|
||||
} catch (error: unknown) {
|
||||
await this.failSafely(delivery, error);
|
||||
return 'settled';
|
||||
}
|
||||
}
|
||||
|
||||
private async process(
|
||||
delivery: ClaimedDelivery,
|
||||
signal: AbortSignal,
|
||||
): Promise<Outcome> {
|
||||
const target =
|
||||
delivery.templateCode !== null
|
||||
? this.templateTarget(delivery)
|
||||
: await this.notificationTarget(delivery);
|
||||
if ('skip' in target) {
|
||||
await settleSkipped(
|
||||
this.prisma,
|
||||
TABLE,
|
||||
delivery.id,
|
||||
delivery.claimedAt,
|
||||
target.skip,
|
||||
);
|
||||
return 'settled';
|
||||
}
|
||||
const waitMs = this.chatWaitMs(target.chatId);
|
||||
if (waitMs > 0) {
|
||||
await deferClaim(
|
||||
this.prisma,
|
||||
TABLE,
|
||||
delivery.id,
|
||||
delivery.claimedAt,
|
||||
waitMs,
|
||||
);
|
||||
return 'spaced';
|
||||
}
|
||||
await this.keepRateLimit(signal);
|
||||
await this.send(delivery, target, signal);
|
||||
return 'settled';
|
||||
}
|
||||
|
||||
private templateTarget(delivery: ClaimedDelivery): Target {
|
||||
const code = delivery.templateCode;
|
||||
if (
|
||||
code === null ||
|
||||
delivery.chatId === null ||
|
||||
!Object.hasOwn(TELEGRAM_TEMPLATES, code)
|
||||
) {
|
||||
return { skip: SKIP.unknownTemplate };
|
||||
}
|
||||
return {
|
||||
chatId: delivery.chatId,
|
||||
text: TELEGRAM_TEMPLATES[code as TelegramTemplateCode],
|
||||
};
|
||||
}
|
||||
|
||||
/** Spec 9a.6: everything that may have changed since the delivery was queued. */
|
||||
private async notificationTarget(delivery: ClaimedDelivery): Promise<Target> {
|
||||
const context = await this.loadContext(delivery);
|
||||
if (context === null) {
|
||||
return { skip: SKIP.notificationGone };
|
||||
}
|
||||
if (!context.userActive) {
|
||||
return { skip: SKIP.userInactive };
|
||||
}
|
||||
if (!context.telegramEnabled) {
|
||||
return { skip: SKIP.telegramDisabled };
|
||||
}
|
||||
if (context.chatId === null) {
|
||||
return { skip: SKIP.notLinked };
|
||||
}
|
||||
if (!context.categories.includes(context.category)) {
|
||||
return { skip: SKIP.categoryOff };
|
||||
}
|
||||
// Fresh, not the 30 s cache: this process never hears the invalidation
|
||||
// of the API pods, so a revoked right would keep sending for 30 s.
|
||||
const held = await this.permissions.getFreshPermissions(
|
||||
delivery.userId as string,
|
||||
);
|
||||
if (!context.requiredPermissions.every((key) => held.has(key))) {
|
||||
return { skip: SKIP.permissionRevoked };
|
||||
}
|
||||
if (context.stale) {
|
||||
return { skip: SKIP.stale };
|
||||
}
|
||||
if (!context.taskOpen) {
|
||||
return { skip: SKIP.taskDone };
|
||||
}
|
||||
return {
|
||||
chatId: context.chatId,
|
||||
text: renderTelegramText(context, context.detail, this.frontendOrigin()),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* One query for user, settings, link, notification, age and task. Stale:
|
||||
* TASK_DUE and DIGEST only on the Berlin day they were written, every
|
||||
* other category for 24 hours. "createdAt" is a UTC timestamp without
|
||||
* zone; AT TIME ZONE 'UTC' makes it an instant first.
|
||||
*/
|
||||
private async loadContext(
|
||||
delivery: ClaimedDelivery,
|
||||
): Promise<DeliveryContext | null> {
|
||||
const rows = await this.prisma.$queryRaw<DeliveryContext[]>(Prisma.sql`
|
||||
SELECT
|
||||
(u."isActive" AND r."status" = 'ACTIVE') AS "userActive",
|
||||
COALESCE(p."telegramEnabled", FALSE) AS "telegramEnabled",
|
||||
COALESCE(p."telegramCategories"::text[], ARRAY[]::text[]) AS "categories",
|
||||
COALESCE(p."telegramDetail", TRUE) AS "detail",
|
||||
l."chatId" AS "chatId",
|
||||
n."category"::text AS "category",
|
||||
n."message" AS "message",
|
||||
n."link" AS "link",
|
||||
n."requiredPermissions" AS "requiredPermissions",
|
||||
CASE
|
||||
WHEN n."category" IN ('TASK_DUE', 'DIGEST') THEN
|
||||
to_char((n."createdAt" AT TIME ZONE 'UTC') AT TIME ZONE 'Europe/Berlin', 'YYYY-MM-DD')
|
||||
<> to_char(now() AT TIME ZONE 'Europe/Berlin', 'YYYY-MM-DD')
|
||||
ELSE now() - (n."createdAt" AT TIME ZONE 'UTC') > interval '24 hours'
|
||||
END AS "stale",
|
||||
CASE
|
||||
WHEN n."category" = 'TASK_DUE' AND n."relatedType" = 'Task' AND n."relatedId" IS NOT NULL THEN
|
||||
COALESCE((SELECT t."status" IN ('OPEN', 'IN_PROGRESS') FROM "Task" t WHERE t."id" = n."relatedId"), FALSE)
|
||||
ELSE TRUE
|
||||
END AS "taskOpen"
|
||||
FROM "NotificationDelivery" d
|
||||
JOIN "Notification" n ON n."id" = d."notificationId"
|
||||
JOIN "User" u ON u."id" = d."userId"
|
||||
JOIN "Role" r ON r."id" = u."roleId"
|
||||
LEFT JOIN "NotificationPreference" p ON p."userId" = u."id"
|
||||
LEFT JOIN "TelegramLink" l ON l."userId" = u."id"
|
||||
WHERE d."id" = ${delivery.id}`);
|
||||
return rows[0] ?? null;
|
||||
}
|
||||
|
||||
private async send(
|
||||
delivery: ClaimedDelivery,
|
||||
target: { chatId: bigint; text: string },
|
||||
signal: AbortSignal,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await this.client.sendMessage(target.chatId, target.text, signal);
|
||||
} catch (error: unknown) {
|
||||
await this.onSendError(delivery, target.chatId, error);
|
||||
return;
|
||||
}
|
||||
this.lastSendToChat.set(String(target.chatId), performance.now());
|
||||
await settleSuccess(
|
||||
this.prisma,
|
||||
TABLE,
|
||||
delivery.id,
|
||||
delivery.claimedAt,
|
||||
'DELIVERED',
|
||||
);
|
||||
}
|
||||
|
||||
private async onSendError(
|
||||
delivery: ClaimedDelivery,
|
||||
chatId: bigint,
|
||||
error: unknown,
|
||||
): Promise<void> {
|
||||
if (!(error instanceof TelegramApiError)) {
|
||||
await this.failSafely(delivery, error);
|
||||
return;
|
||||
}
|
||||
if (error.statusCode === 429) {
|
||||
const seconds = error.retryAfterSec ?? DEFAULT_RETRY_AFTER_SEC;
|
||||
// Telegram limits the whole bot: the pod waits, not only this row.
|
||||
this.pausedUntil = Math.max(
|
||||
this.pausedUntil,
|
||||
performance.now() + seconds * 1_000,
|
||||
);
|
||||
await deferClaim(
|
||||
this.prisma,
|
||||
TABLE,
|
||||
delivery.id,
|
||||
delivery.claimedAt,
|
||||
seconds * 1_000,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (isChatGone(error)) {
|
||||
await this.endDelivery(delivery, chatId);
|
||||
return;
|
||||
}
|
||||
if (error.statusCode === 401) {
|
||||
await this.pauseForInvalidToken(delivery);
|
||||
return;
|
||||
}
|
||||
await this.failSafely(delivery, error);
|
||||
}
|
||||
|
||||
/**
|
||||
* The bot is blocked or the chat is gone: the link ends, but only if it
|
||||
* still points to this chat - a newer link of the same user to another
|
||||
* chat stays. Settings, link and message to the user change together.
|
||||
*/
|
||||
private async endDelivery(
|
||||
delivery: ClaimedDelivery,
|
||||
chatId: bigint,
|
||||
): Promise<void> {
|
||||
const userId = delivery.userId;
|
||||
if (userId === null) {
|
||||
await settleSkipped(
|
||||
this.prisma,
|
||||
TABLE,
|
||||
delivery.id,
|
||||
delivery.claimedAt,
|
||||
SKIP.chatUnavailable,
|
||||
);
|
||||
return;
|
||||
}
|
||||
await this.prisma.$transaction(async (tx) => {
|
||||
await lockRowForUpdate(tx, 'NotificationPreference', userId);
|
||||
const { count } = await tx.telegramLink.deleteMany({
|
||||
where: { userId, chatId },
|
||||
});
|
||||
if (count === 1) {
|
||||
await tx.notificationPreference.updateMany({
|
||||
where: { userId },
|
||||
data: { telegramEnabled: false },
|
||||
});
|
||||
await this.notifications.record(
|
||||
{
|
||||
userId,
|
||||
priority: 'HINWEIS',
|
||||
category: 'SYSTEM',
|
||||
message: CHAT_GONE_MESSAGE,
|
||||
},
|
||||
tx,
|
||||
);
|
||||
}
|
||||
await settleSkipped(
|
||||
tx,
|
||||
TABLE,
|
||||
delivery.id,
|
||||
delivery.claimedAt,
|
||||
SKIP.chatUnavailable,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/** The token is rejected: nothing can go out, so the whole loop waits. */
|
||||
private async pauseForInvalidToken(delivery: ClaimedDelivery): Promise<void> {
|
||||
this.pausedUntil = performance.now() + TELEGRAM_TOKEN_INVALID_PAUSE_MS;
|
||||
await deferClaim(
|
||||
this.prisma,
|
||||
TABLE,
|
||||
delivery.id,
|
||||
delivery.claimedAt,
|
||||
TELEGRAM_TOKEN_INVALID_PAUSE_MS,
|
||||
);
|
||||
try {
|
||||
// One per user and day: the dedupe key carries the Berlin day.
|
||||
await this.notifications.recordForPermission('system:manage', {
|
||||
category: 'SYSTEM',
|
||||
priority: 'WARNUNG',
|
||||
message: TOKEN_INVALID_MESSAGE,
|
||||
link: '/settings/system',
|
||||
dedupeKeyPrefix: `telegram-token-invalid:${berlinToday()}`,
|
||||
});
|
||||
} catch (error: unknown) {
|
||||
this.logger.error(
|
||||
`Telegram token notice not recorded: ${loggableError(error).message}`,
|
||||
);
|
||||
}
|
||||
this.logger.error(
|
||||
'Telegram rejected the bot token (401); sending paused for 15 minutes',
|
||||
);
|
||||
}
|
||||
|
||||
/** A failed attempt under the shared retry rule; the error text carries no secret. */
|
||||
private async failSafely(
|
||||
delivery: ClaimedDelivery,
|
||||
error: unknown,
|
||||
): Promise<void> {
|
||||
const { message } = loggableError(error);
|
||||
try {
|
||||
await settleFailure(
|
||||
this.prisma,
|
||||
TABLE,
|
||||
delivery.id,
|
||||
delivery.claimedAt,
|
||||
delivery.attempts,
|
||||
message,
|
||||
);
|
||||
} catch (settleError: unknown) {
|
||||
this.logger.error(
|
||||
`Telegram delivery ${delivery.id} not settled: ${loggableError(settleError).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Only development may fall back to localhost. Elsewhere a missing origin
|
||||
* (the startup check normally prevents it) sends the text without a link.
|
||||
*/
|
||||
private frontendOrigin(): string | null {
|
||||
const configured = process.env.FRONTEND_ORIGIN;
|
||||
if (configured) {
|
||||
return configured;
|
||||
}
|
||||
const environment = process.env.NODE_ENV;
|
||||
if (environment === undefined || environment === 'development') {
|
||||
return DEVELOPMENT_ORIGIN;
|
||||
}
|
||||
if (!this.warnedMissingOrigin) {
|
||||
this.warnedMissingOrigin = true;
|
||||
this.logger.warn(
|
||||
'FRONTEND_ORIGIN is not set: Telegram messages go out without a link',
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private remainingPauseMs(): number {
|
||||
return Math.max(0, Math.ceil(this.pausedUntil - performance.now()));
|
||||
}
|
||||
|
||||
/** Milliseconds until the chat may get its next message; 0: now. */
|
||||
private chatWaitMs(chatId: bigint): number {
|
||||
const now = performance.now();
|
||||
for (const [chat, sentAt] of this.lastSendToChat) {
|
||||
if (now - sentAt > CHAT_MEMORY_MS) {
|
||||
this.lastSendToChat.delete(chat);
|
||||
}
|
||||
}
|
||||
const sentAt = this.lastSendToChat.get(String(chatId));
|
||||
return sentAt === undefined
|
||||
? 0
|
||||
: Math.max(0, Math.ceil(TELEGRAM_CHAT_SPACING_MS - (now - sentAt)));
|
||||
}
|
||||
|
||||
/** At most TELEGRAM_MAX_PER_SECOND messages per second over all chats. */
|
||||
private async keepRateLimit(signal: AbortSignal): Promise<void> {
|
||||
const waitMs = MIN_SEND_GAP_MS - (performance.now() - this.lastSendAt);
|
||||
if (waitMs > 0) {
|
||||
await sleep(waitMs, signal);
|
||||
}
|
||||
this.lastSendAt = performance.now();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import {
|
||||
renderTelegramText,
|
||||
TELEGRAM_TEMPLATES,
|
||||
TelegramTemplateCode,
|
||||
} from './telegram-texts';
|
||||
|
||||
const ORIGIN = 'https://app.example.org';
|
||||
|
||||
describe('TELEGRAM_TEMPLATES', () => {
|
||||
it('has the exact texts of the bot replies', () => {
|
||||
expect(TELEGRAM_TEMPLATES).toEqual({
|
||||
LINKED:
|
||||
'Verbunden mit ApiCore. Hier kommen deine Erinnerungen. /stop beendet das.',
|
||||
LINK_INVALID:
|
||||
'Der Link ist ungültig oder abgelaufen. Bitte in ApiCore neu verbinden.',
|
||||
STOPPED: 'Abgemeldet. Du bekommst keine Nachrichten mehr.',
|
||||
HELP: 'Dieser Bot schickt Erinnerungen aus ApiCore. Verbinden geht nur in ApiCore unter Profil > Benachrichtigungen. /stop beendet.',
|
||||
} satisfies Record<TelegramTemplateCode, string>);
|
||||
});
|
||||
});
|
||||
|
||||
describe('renderTelegramText', () => {
|
||||
it('sends the message, a blank line and the app link', () => {
|
||||
const text = renderTelegramText(
|
||||
{ message: 'Heute fällig: Futter bestellen', link: '/tasks' },
|
||||
true,
|
||||
ORIGIN,
|
||||
);
|
||||
|
||||
expect(text).toBe(
|
||||
'Heute fällig: Futter bestellen\n\nhttps://app.example.org/tasks',
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to /dashboard without a link', () => {
|
||||
const text = renderTelegramText({ message: 'm', link: null }, true, ORIGIN);
|
||||
|
||||
expect(text).toBe('m\n\nhttps://app.example.org/dashboard');
|
||||
});
|
||||
|
||||
it('without details only says there is a new message, plus the link', () => {
|
||||
const text = renderTelegramText(
|
||||
{ message: 'Heute fällig: Schwarmzelle Völker', link: '/tasks' },
|
||||
false,
|
||||
ORIGIN,
|
||||
);
|
||||
|
||||
expect(text).toBe(
|
||||
'Neue Meldung in ApiCore.\n\nhttps://app.example.org/tasks',
|
||||
);
|
||||
expect(text).not.toContain('Schwarm');
|
||||
});
|
||||
|
||||
it('tolerates a trailing slash of the origin', () => {
|
||||
const text = renderTelegramText(
|
||||
{ message: 'm', link: '/tasks' },
|
||||
true,
|
||||
`${ORIGIN}/`,
|
||||
);
|
||||
|
||||
expect(text.endsWith('https://app.example.org/tasks')).toBe(true);
|
||||
});
|
||||
|
||||
it('cuts a long message to at most 600 characters and keeps the link whole', () => {
|
||||
const text = renderTelegramText(
|
||||
{ message: 'ä'.repeat(2000), link: '/tasks' },
|
||||
true,
|
||||
ORIGIN,
|
||||
);
|
||||
|
||||
expect([...text].length).toBeLessThanOrEqual(600);
|
||||
expect(text.endsWith('\n\nhttps://app.example.org/tasks')).toBe(true);
|
||||
expect(text).toContain('…\n\n');
|
||||
});
|
||||
|
||||
it('counts code points, not UTF-16 units', () => {
|
||||
const text = renderTelegramText(
|
||||
{ message: '🐝'.repeat(1000), link: '/tasks' },
|
||||
true,
|
||||
ORIGIN,
|
||||
);
|
||||
|
||||
expect([...text].length).toBeLessThanOrEqual(600);
|
||||
expect(text).not.toMatch(/[\ud800-\udbff](?![\udc00-\udfff])/);
|
||||
});
|
||||
|
||||
it('removes control characters and invisible characters from the message', () => {
|
||||
const text = renderTelegramText(
|
||||
{ message: 'a\u0007bcd\r\ne', link: '/tasks' },
|
||||
true,
|
||||
ORIGIN,
|
||||
);
|
||||
|
||||
expect(text).toBe('abcd\ne\n\nhttps://app.example.org/tasks');
|
||||
});
|
||||
|
||||
it('without an origin sends the text alone, no link', () => {
|
||||
expect(
|
||||
renderTelegramText({ message: 'Hallo', link: '/tasks' }, true, null),
|
||||
).toBe('Hallo');
|
||||
expect(
|
||||
renderTelegramText({ message: 'Hallo', link: '/tasks' }, false, null),
|
||||
).toBe('Neue Meldung in ApiCore.');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
import {
|
||||
sanitizeTelegramText,
|
||||
TELEGRAM_MAX_MESSAGE_CHARS,
|
||||
} from './telegram-client';
|
||||
|
||||
export type TelegramTemplateCode =
|
||||
'LINKED' | 'LINK_INVALID' | 'STOPPED' | 'HELP';
|
||||
|
||||
/** The replies of the bot to what a person does in the chat (Spec 9a.5). */
|
||||
export const TELEGRAM_TEMPLATES: Record<TelegramTemplateCode, string> = {
|
||||
LINKED:
|
||||
'Verbunden mit ApiCore. Hier kommen deine Erinnerungen. /stop beendet das.',
|
||||
LINK_INVALID:
|
||||
'Der Link ist ungültig oder abgelaufen. Bitte in ApiCore neu verbinden.',
|
||||
STOPPED: 'Abgemeldet. Du bekommst keine Nachrichten mehr.',
|
||||
HELP: 'Dieser Bot schickt Erinnerungen aus ApiCore. Verbinden geht nur in ApiCore unter Profil > Benachrichtigungen. /stop beendet.',
|
||||
};
|
||||
|
||||
const DEFAULT_LINK = '/dashboard';
|
||||
const DETAIL_OFF_TEXT = 'Neue Meldung in ApiCore.';
|
||||
const ELLIPSIS = '…';
|
||||
|
||||
/**
|
||||
* The plain text of a notification: its message (or, without details, only
|
||||
* the hint that there is one), a blank line and the app link. At most 600
|
||||
* characters; a long message is cut, the link stays whole. No parse_mode on
|
||||
* the wire, so a title cannot inject formatting.
|
||||
*/
|
||||
export function renderTelegramText(
|
||||
notification: { message: string; link: string | null },
|
||||
detail: boolean,
|
||||
frontendOrigin: string | null,
|
||||
): string {
|
||||
// Without a known origin no link at all: never a localhost link.
|
||||
const tail =
|
||||
frontendOrigin === null
|
||||
? ''
|
||||
: `\n\n${sanitizeTelegramText(
|
||||
`${frontendOrigin.replace(/\/+$/, '')}${notification.link ?? DEFAULT_LINK}`,
|
||||
)}`;
|
||||
const head = detail
|
||||
? sanitizeTelegramText(notification.message)
|
||||
: DETAIL_OFF_TEXT;
|
||||
const budget = TELEGRAM_MAX_MESSAGE_CHARS - [...tail].length;
|
||||
const characters = [...head];
|
||||
if (characters.length <= budget) {
|
||||
return `${head}${tail}`;
|
||||
}
|
||||
const kept = characters.slice(0, Math.max(budget - 1, 0)).join('');
|
||||
return sanitizeTelegramText(`${kept}${ELLIPSIS}${tail}`);
|
||||
}
|
||||
@@ -1,17 +1,24 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { NotificationsModule } from '../notifications/notifications.module';
|
||||
import { telegramConfig } from './telegram-config';
|
||||
import { TELEGRAM_CONFIG, TelegramClient } from './telegram-client';
|
||||
import { TelegramSender } from './telegram-sender';
|
||||
|
||||
/**
|
||||
* The Bot API client for the worker's Telegram loops. Reads the token from
|
||||
* the environment once; without it the client reports isConfigured() false
|
||||
* and the loops do not run.
|
||||
* and the loops do not run. The sender needs the permission resolver, which
|
||||
* the global JobPermissionsModule (worker) or PermissionsModule (API) provides:
|
||||
* this module imports neither, so the permission catalog is never rewritten
|
||||
* by it.
|
||||
*/
|
||||
@Module({
|
||||
imports: [NotificationsModule],
|
||||
providers: [
|
||||
{ provide: TELEGRAM_CONFIG, useFactory: () => telegramConfig() },
|
||||
TelegramClient,
|
||||
TelegramSender,
|
||||
],
|
||||
exports: [TelegramClient],
|
||||
exports: [TelegramClient, TelegramSender],
|
||||
})
|
||||
export class TelegramModule {}
|
||||
Reference in new issue
Block a user