import { randomUUID } from 'node:crypto'; import { Inject, Injectable, Logger, Optional } from '@nestjs/common'; import { Prisma } from '../../generated/prisma'; import { loggableError } from '../common/loggable-error'; import { PrismaService } from '../prisma/prisma.service'; import { TelegramApiError } from './telegram-api-error'; import { TelegramClient, TelegramUpdate, TelegramUpdateBatch, } from './telegram-client'; import { TelegramLinkService } from './telegram-link.service'; import type { TelegramTemplateCode } from './telegram-texts'; export const LEASE_SECONDS = 60; /** Loop interval: also how long a receiver without the lease (or after a 409) waits. */ export const NO_LEASE_WAIT_MS = 30_000; export const HELP_REPLIES_PER_CHAT = 3; export const HELP_WINDOW_MS = 600_000; /** Tunables of the receiver; tests inject their own (limits, clock). */ export interface TelegramUpdatesOptions { /** Attempts after which a failing update is skipped. */ attempts: number; /** Time after which a failing update is skipped, in ms of `now()`. */ windowMs: number; /** Monotonic clock in ms. */ now: () => number; } export const TELEGRAM_UPDATES_OPTIONS = Symbol('TELEGRAM_UPDATES_OPTIONS'); /** An update that keeps failing is skipped after this many attempts or this long. */ export const MAX_UPDATE_ATTEMPTS = 5; export const MAX_UPDATE_RETRY_MS = 600_000; const MAX_RATE_LIMIT_PAUSE_MS = 3_600_000; /** Fixed texts for the system page: no ids, no message text. */ export const SKIPPED_UPDATE_TEXT = 'Telegram-Nachricht übersprungen'; const MAX_LAST_ERROR_CHARS = 300; const START_COMMAND = /^\/([A-Za-z]+)(?:@[A-Za-z0-9_]+)?(?:\s+(\S+))?/; export type PollOutcome = 'disabled' | 'no-lease' | 'polled' | 'conflict' | 'error'; type Command = { name: 'start'; token: string | null } | { name: 'stop' } | { name: 'help' }; /** The lease or the stored offset changed: nothing of this update may stay. */ class LeaseLostError extends Error { constructor() { super('Telegram poll lease lost'); this.name = 'LeaseLostError'; } } /** * What may go to a log or lastError: Prisma errors by name, code and model * (loggableError), every other error by its class name only - a message can * quote user data. */ function describeError(error: unknown): string { if ( error instanceof Prisma.PrismaClientKnownRequestError || error instanceof Prisma.PrismaClientUnknownRequestError || error instanceof Prisma.PrismaClientValidationError || error instanceof Prisma.PrismaClientInitializationError || error instanceof Prisma.PrismaClientRustPanicError ) { return loggableError(error).message; } return error instanceof Error ? error.name : 'unknown error'; } const CONNECTION_ERROR_CODES = new Set(['P1001', 'P1002', 'P1008', 'P1017']); const SOCKET_ERROR_CODES = new Set([ 'ECONNREFUSED', 'ECONNRESET', 'ETIMEDOUT', 'EHOSTUNREACH', ]); /** * The database or the network is unreachable (Prisma P1001, P1002, P1008, * P1017, an initialization error, a refused or timed-out socket, a Telegram * call without an answer): that says nothing about the update itself, so it * does not count toward skipping it. */ function isConnectionError(error: unknown): boolean { if (error instanceof Prisma.PrismaClientInitializationError) { return true; } if (error instanceof Prisma.PrismaClientKnownRequestError) { return CONNECTION_ERROR_CODES.has(error.code); } if (error instanceof TelegramApiError) { return error.statusCode === null; } const code = (error as { code?: unknown } | null)?.code; return typeof code === 'string' && SOCKET_ERROR_CODES.has(code); } function parseCommand(text: string | null): Command { const match = START_COMMAND.exec(text?.trim() ?? ''); const name = match?.[1].toLowerCase(); if (name === 'start') { return { name: 'start', token: match?.[2] ?? null }; } return name === 'stop' ? { name: 'stop' } : { name: 'help' }; } /** * At most HELP_REPLIES_PER_CHAT replies per chat in HELP_WINDOW_MS, counted * in memory (there is exactly one receiver, the lease sees to that). A * stranger writing to the bot cannot make it talk without end. Monotonic * clock, so a clock step changes nothing. */ class ReplyLimiter { private readonly sentAt = new Map(); constructor(private readonly now: () => number) {} /** Would one more reply to this chat be within the limit? Counts nothing. */ allows(chatId: bigint): boolean { this.prune(); return ( (this.sentAt.get(String(chatId))?.length ?? 0) < HELP_REPLIES_PER_CHAT ); } /** Counts a reply that was queued and committed. */ record(chatId: bigint): void { const key = String(chatId); this.sentAt.set(key, [...(this.sentAt.get(key) ?? []), this.now()]); } private prune(): void { const now = this.now(); for (const [chat, times] of this.sentAt) { const recent = times.filter((time) => now - time < HELP_WINDOW_MS); if (recent.length === 0) { this.sentAt.delete(chat); } else { this.sentAt.set(chat, recent); } } } } /** * Receives what people write to the bot (Spec 9a.5, 9a.7): long polling * getUpdates in the worker, no public endpoint. Exactly one receiver per * bot: the lease in TelegramPollState (database clock only) says who. Each * update runs in its own transaction together with the offset advance, which * is a compare-and-set on lease owner and old offset - a receiver that lost * the lease changes nothing, so every update takes effect once. Replies are * queued as deliveries (the sender talks to Telegram, not this class). * Never logs a token, a link code, a chat id or a message text. */ @Injectable() export class TelegramUpdatesService { private readonly logger = new Logger(TelegramUpdatesService.name); /** Names this receiver in the lease. */ private readonly instanceId = randomUUID(); private readonly replies: ReplyLimiter; private readonly limits: Pick< TelegramUpdatesOptions, 'attempts' | 'windowMs' >; private readonly now: () => number; private holdsLease = false; /** Monotonic time until which Telegram asked us to stay away (429). */ private rateLimitedUntil = 0; /** Failed attempts per update, in memory: a restart gives them a fresh start. */ private readonly failures = new Map< bigint, { attempts: number; firstAt: number } >(); constructor( private readonly prisma: PrismaService, private readonly client: TelegramClient, private readonly links: TelegramLinkService, @Optional() @Inject(TELEGRAM_UPDATES_OPTIONS) options?: Partial, ) { this.now = options?.now ?? (() => performance.now()); this.limits = { attempts: options?.attempts ?? MAX_UPDATE_ATTEMPTS, windowMs: options?.windowMs ?? MAX_UPDATE_RETRY_MS, }; this.replies = new ReplyLimiter(this.now); } /** * One cycle: take or extend the lease, fetch updates (long poll), handle * them one by one. 'polled' means the next cycle may start at once. */ async pollOnce(signal: AbortSignal): Promise { if (!this.client.isConfigured()) { return 'disabled'; } if (signal.aborted || this.now() < this.rateLimitedUntil) { return 'error'; } const offset = await this.acquireLease(); if (offset === null) { return 'no-lease'; } let batch: TelegramUpdateBatch; try { batch = await this.client.getUpdates(offset, signal); } catch (error: unknown) { return this.onFetchError(error, signal); } if (batch.received > 0 && batch.highestUpdateId === null) { // An answer nobody can move the offset past would be fetched again at // once: report it, so the 30 s wait applies. await this.setLastError('getUpdates answer without update ids'); return 'error'; } await this.markPolled(); return this.applyBatch(batch, offset, signal); } /** The updates in order, then the offset past dropped ones; a clean batch clears lastError. */ private async applyBatch( batch: TelegramUpdateBatch, offset: bigint, signal: AbortSignal, ): Promise { let current = offset; let skipped = false; for (const update of batch.updates) { if (signal.aborted) { return 'polled'; } try { const reply = await this.prisma.$transaction((tx) => this.applyUpdate(tx, update, current), ); // Counted only now: a rolled-back reply was never queued. if (reply !== null) { this.replies.record(reply); } this.failures.delete(update.updateId); } catch (error: unknown) { if (error instanceof LeaseLostError) { return 'no-lease'; } if (!(await this.shouldSkip(update, error))) { return 'error'; } try { await this.skipUpdate(update, current); } catch (skipError: unknown) { return this.onUpdateError(skipError); } skipped = true; } current = update.updateId + 1n; } const dropped = await this.skipDropped(batch, current, signal); if (dropped !== 'ok') { return dropped; } this.pruneFailures(current); if (!skipped) { await this.clearLastError(); } return 'polled'; } /** Forgets updates behind the offset and failures older than the window. */ private pruneFailures(offset: bigint): void { const now = this.now(); for (const [updateId, state] of this.failures) { if (updateId < offset || now - state.firstAt >= this.limits.windowMs) { this.failures.delete(updateId); } } } /** Dropped updates (no chat, wrong shape): move the offset past them, never fetch them again. */ private async skipDropped( batch: TelegramUpdateBatch, current: bigint, signal: AbortSignal, ): Promise<'ok' | PollOutcome> { const highest = batch.highestUpdateId; if (highest === null || highest < current || signal.aborted) { return 'ok'; } try { await this.prisma.$transaction((tx) => this.advanceOffsetIn(tx, current, highest + 1n), ); } catch (error: unknown) { return this.onUpdateError(error); } return 'ok'; } /** Lease SQL, database clock only. The stored offset, or null without the lease. */ private async acquireLease(): Promise { const rows = await this.prisma.$queryRaw<{ updateOffset: bigint }[]>` UPDATE "TelegramPollState" SET "leaseOwner" = ${this.instanceId}, "leaseUntil" = now() + make_interval(secs => ${LEASE_SECONDS}) WHERE "id" = 'singleton' AND ("leaseUntil" IS NULL OR "leaseUntil" < now() OR "leaseOwner" = ${this.instanceId}) RETURNING "updateOffset"`; this.holdsLease = rows.length > 0; return rows[0]?.updateOffset ?? null; } /** * Shutdown: the next receiver need not wait for the lease to run out. * Call it after the loop has stopped; it does nothing without a lease. */ async releaseLease(): Promise { if (!this.holdsLease) { return; } this.holdsLease = false; try { await this.prisma.$executeRaw` UPDATE "TelegramPollState" SET "leaseUntil" = now() WHERE "id" = 'singleton' AND "leaseOwner" = ${this.instanceId}`; } catch (error: unknown) { this.logger.warn(`Telegram lease not released: ${describeError(error)}`); } } private async markPolled(): Promise { await this.prisma.$executeRaw` UPDATE "TelegramPollState" SET "lastPollAt" = now() WHERE "id" = 'singleton' AND "leaseOwner" = ${this.instanceId}`; } private async clearLastError(): Promise { await this.prisma.$executeRaw` UPDATE "TelegramPollState" SET "lastError" = NULL WHERE "id" = 'singleton' AND "leaseOwner" = ${this.instanceId}`; } private async setLastError(text: string): Promise { await this.prisma.$executeRaw` UPDATE "TelegramPollState" SET "lastError" = ${text.slice(0, MAX_LAST_ERROR_CHARS)} WHERE "id" = 'singleton' AND "leaseOwner" = ${this.instanceId}`; } /** * 409: another getUpdates or a webhook; wait. 429: stay away until * retry_after has passed (at most an hour). The texts are fixed plus the * error class: nothing from the answer, no token, no URL. */ private async onFetchError( error: unknown, signal: AbortSignal, ): Promise { if (signal.aborted) { return 'error'; } const status = error instanceof TelegramApiError ? error.statusCode : null; if (status === 429) { const seconds = error instanceof TelegramApiError ? error.retryAfterSec : null; this.rateLimitedUntil = this.now() + Math.min( (seconds ?? NO_LEASE_WAIT_MS / 1_000) * 1_000, MAX_RATE_LIMIT_PAUSE_MS, ); } let label = 'getUpdates failed'; if (status === 409) { label = 'getUpdates conflict (409)'; } else if (status === 429) { label = 'getUpdates rate limited (429)'; } await this.setLastError(`${label}: ${describeError(error)}`); return status === 409 ? 'conflict' : 'error'; } private async onUpdateError(error: unknown): Promise { if (error instanceof LeaseLostError) { return 'no-lease'; } // The offset stays: the update is tried again in the next cycle. const text = `update not handled: ${describeError(error)}`; this.logger.error(`Telegram ${text}`); await this.setLastError(text); return 'error'; } /** * Counts a failed attempt of an update; true once it failed too often or * for too long and has to be skipped (otherwise the failure is recorded). */ private async shouldSkip( update: TelegramUpdate, error: unknown, ): Promise { if (isConnectionError(error)) { // The database or network is down: the update is not at fault. await this.onUpdateError(error); return false; } const now = this.now(); const known = this.failures.get(update.updateId); const state = { attempts: (known?.attempts ?? 0) + 1, firstAt: known?.firstAt ?? now, }; this.failures.set(update.updateId, state); const exhausted = state.attempts >= this.limits.attempts || now - state.firstAt >= this.limits.windowMs; if (!exhausted) { await this.onUpdateError(error); } return exhausted; } /** Offset past a poison update in its own transaction, so the queue moves on. */ private async skipUpdate( update: TelegramUpdate, oldOffset: bigint, ): Promise { await this.prisma.$transaction(async (tx) => { await this.advanceOffsetIn(tx, oldOffset, update.updateId + 1n); await tx.$executeRaw` UPDATE "TelegramPollState" SET "lastError" = ${SKIPPED_UPDATE_TEXT} WHERE "id" = 'singleton' AND "leaseOwner" = ${this.instanceId}`; }); this.failures.delete(update.updateId); this.logger.warn('Telegram update skipped after repeated failures'); } /** Offset compare-and-set on lease owner and old offset; count 0: lease lost. */ private async advanceOffsetIn( tx: Prisma.TransactionClient, oldOffset: bigint, newOffset: bigint, ): Promise { const advanced = await tx.$executeRaw` UPDATE "TelegramPollState" SET "updateOffset" = ${newOffset} WHERE "id" = 'singleton' AND "leaseOwner" = ${this.instanceId} AND "leaseUntil" > now() AND "updateOffset" = ${oldOffset}`; if (advanced === 0) { throw new LeaseLostError(); } } /** * Offset first (a lost lease ends here), then the effect, all in `tx`. * Returns the chat whose reply quota must be counted once `tx` commits. */ private async applyUpdate( tx: Prisma.TransactionClient, update: TelegramUpdate, oldOffset: bigint, ): Promise { await this.advanceOffsetIn(tx, oldOffset, update.updateId + 1n); // Groups and channels: the bot is not for them. if (update.chatType !== 'private') { return null; } const command = parseCommand(update.text); if (command.name === 'stop') { await this.links.stopIn(tx, update.chatId); return this.reply(tx, update.chatId, 'STOPPED'); } if (command.name === 'start' && command.token !== null) { const result = await this.links.linkFromStartIn( tx, command.token, update.chatId, update.firstName, ); return this.reply(tx, update.chatId, result); } return this.reply(tx, update.chatId, 'HELP'); } /** * LINKED always goes out (it needs a valid one-time token, so no * stranger can trigger it); every other reply counts against the limit. * Returns the chat to count after the commit, or null. */ private async reply( tx: Prisma.TransactionClient, chatId: bigint, code: TelegramTemplateCode, ): Promise { if (code === 'LINKED') { await this.links.queueReplyIn(tx, chatId, code); return null; } if (!this.replies.allows(chatId)) { return null; } await this.links.queueReplyIn(tx, chatId, code); return chatId; } }