Platform backbone (plan 2026-10-13-plattform-rueckgrat), all tasks reviewed: - webhooks with delivery table, retries and stable delivery id - calendar handlers; a finished task removes its Outlook event - tasks, automations, inspections, varroa, treatments, feedings, harvests, colonies, wintering, stock and bottling record domain events in the same transaction; legacy post-commit calls removed - row locks through one helper (lockRowForUpdate, lockRowsForShare, lockRowsForUpdate) - job framework with JobRun, cli exit codes, cleanup, task templates, daily evaluation, hourly app-backup and the rekey job - system API and page "Verwaltung > System" with retry - @nestjs/schedule removed, guard tests against in-process schedulers Field capture T3 (plan 2026-10-13-t3-erfassung-am-stand), all tasks reviewed: - inspection photo offline without metadata - bulk capture for treatments, feedings and varroa (backend + frontend) - offline stand mode with service worker, offline pages and packs - offline session, cold start and logout on shared devices Docs: ARCHITECTURE.md assumptions and sections updated, TASKS.md. Gates: backend lint, typecheck, 1770 unit, 420 e2e, build, no schema drift; frontend lint, typecheck, 913 tests, build. Not rolled out. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
426 lines
14 KiB
TypeScript
426 lines
14 KiB
TypeScript
import { Logger } from '@nestjs/common';
|
|
import * as path from 'node:path';
|
|
import { JobRun } from '../../generated/prisma';
|
|
import { PrismaService } from '../prisma/prisma.service';
|
|
import { JobRunService } from '../system/job-run.service';
|
|
import {
|
|
APP_BACKUP_LOCK_TIMEOUT_MS,
|
|
BackupSchedulerService,
|
|
} from './backup-scheduler.service';
|
|
import { BackupService } from './backup.service';
|
|
import { BackupSettingsService } from './backup-settings.service';
|
|
|
|
jest.mock('node:fs/promises', () => ({
|
|
mkdir: jest.fn(),
|
|
writeFile: jest.fn(),
|
|
readdir: jest.fn(),
|
|
stat: jest.fn(),
|
|
unlink: jest.fn(),
|
|
}));
|
|
|
|
import * as fs from 'node:fs/promises';
|
|
|
|
type RawCall = [readonly string[], ...unknown[]];
|
|
|
|
const SEALED = {
|
|
format: 'apicore-backup',
|
|
exportedAt: '2026-10-13T01:05:00.000Z',
|
|
ciphertext: 'x'.repeat(2000),
|
|
};
|
|
|
|
describe('BackupSchedulerService', () => {
|
|
const originalEnv = { ...process.env };
|
|
const directory = path.resolve('/data/backups');
|
|
let service: BackupSchedulerService;
|
|
let backupService: { exportSealed: jest.Mock };
|
|
let settings: { get: jest.Mock };
|
|
let tx: {
|
|
$executeRaw: jest.Mock<Promise<number>, RawCall>;
|
|
$queryRaw: jest.Mock<Promise<{ now: Date }[]>, RawCall>;
|
|
};
|
|
let prisma: { $transaction: jest.Mock };
|
|
let jobRuns: { recent: jest.Mock };
|
|
let error: jest.SpyInstance;
|
|
let warn: jest.SpyInstance;
|
|
|
|
const sqlText = (call: RawCall): string => call[0].join('?');
|
|
|
|
beforeEach(() => {
|
|
jest.clearAllMocks();
|
|
process.env.BACKUP_STORAGE_PATH = directory;
|
|
delete process.env.BACKUP_MAX_IMPORT_BYTES;
|
|
delete process.env.BACKUP_RETENTION_COUNT;
|
|
delete process.env.BACKUP_CRON_SCHEDULE;
|
|
error = jest.spyOn(Logger.prototype, 'error').mockImplementation(() => {});
|
|
warn = jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => {});
|
|
|
|
backupService = { exportSealed: jest.fn().mockResolvedValue(SEALED) };
|
|
settings = {
|
|
get: jest.fn().mockResolvedValue({
|
|
storagePath: null,
|
|
cronSchedule: null,
|
|
retentionCount: null,
|
|
}),
|
|
};
|
|
tx = {
|
|
$executeRaw: jest.fn<Promise<number>, RawCall>().mockResolvedValue(1),
|
|
$queryRaw: jest
|
|
.fn<Promise<{ now: Date }[]>, RawCall>()
|
|
.mockResolvedValue([{ now: new Date('2026-10-13T01:05:00Z') }]),
|
|
};
|
|
prisma = {
|
|
$transaction: jest.fn((work: (client: typeof tx) => Promise<unknown>) =>
|
|
work(tx),
|
|
),
|
|
};
|
|
jobRuns = { recent: jest.fn().mockResolvedValue([]) };
|
|
service = new BackupSchedulerService(
|
|
backupService as unknown as BackupService,
|
|
settings as unknown as BackupSettingsService,
|
|
prisma as unknown as PrismaService,
|
|
jobRuns as unknown as JobRunService,
|
|
);
|
|
(fs.mkdir as jest.Mock).mockResolvedValue(undefined);
|
|
(fs.writeFile as jest.Mock).mockResolvedValue(undefined);
|
|
(fs.readdir as jest.Mock).mockResolvedValue([]);
|
|
(fs.unlink as jest.Mock).mockResolvedValue(undefined);
|
|
});
|
|
|
|
afterEach(() => {
|
|
error.mockRestore();
|
|
warn.mockRestore();
|
|
process.env = { ...originalEnv };
|
|
});
|
|
|
|
function onDisk(files: { name: string; size: number; mtime: string }[]) {
|
|
(fs.readdir as jest.Mock).mockResolvedValue(files.map((f) => f.name));
|
|
(fs.stat as jest.Mock).mockImplementation((filePath: string) => {
|
|
const file = files.find((f) => filePath.endsWith(f.name));
|
|
return Promise.resolve({
|
|
size: file?.size,
|
|
mtime: new Date(file!.mtime),
|
|
});
|
|
});
|
|
}
|
|
|
|
function jobRun(overrides: Partial<JobRun> = {}): JobRun {
|
|
return {
|
|
id: 'run-1',
|
|
job: 'app-backup',
|
|
status: 'SUCCEEDED',
|
|
startedAt: new Date('2026-10-13T01:00:02.000Z'),
|
|
finishedAt: new Date('2026-10-13T01:00:09.000Z'),
|
|
result: { written: true },
|
|
error: null,
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
describe('getStatus', () => {
|
|
it('reports the import limit and the size of the newest backup', async () => {
|
|
process.env.BACKUP_MAX_IMPORT_BYTES = '1000';
|
|
onDisk([
|
|
{
|
|
name: 'apicore-backup-old.json',
|
|
size: 5000,
|
|
mtime: '2026-10-10T03:00:00.000Z',
|
|
},
|
|
{
|
|
name: 'apicore-backup-new.json',
|
|
size: 900,
|
|
mtime: '2026-10-11T03:00:00.000Z',
|
|
},
|
|
]);
|
|
|
|
const status = await service.getStatus();
|
|
|
|
expect(status).toEqual(
|
|
expect.objectContaining({
|
|
importLimitBytes: 1000,
|
|
lastBackupSizeBytes: 900,
|
|
lastBackupExceedsImportLimit: false,
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('warns when the newest backup could no longer be imported', async () => {
|
|
process.env.BACKUP_MAX_IMPORT_BYTES = '1000';
|
|
onDisk([
|
|
{
|
|
name: 'apicore-backup-new.json',
|
|
size: 1001,
|
|
mtime: '2026-10-11T03:00:00.000Z',
|
|
},
|
|
]);
|
|
|
|
const status = await service.getStatus();
|
|
|
|
expect(status.lastBackupSizeBytes).toBe(1001);
|
|
expect(status.lastBackupExceedsImportLimit).toBe(true);
|
|
});
|
|
|
|
it('still reports the import limit while no schedule is configured', async () => {
|
|
delete process.env.BACKUP_STORAGE_PATH;
|
|
|
|
const status = await service.getStatus();
|
|
|
|
expect(status).toEqual(
|
|
expect.objectContaining({
|
|
enabled: false,
|
|
importLimitBytes: 50 * 1024 * 1024,
|
|
lastBackupSizeBytes: null,
|
|
lastBackupExceedsImportLimit: false,
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('reports the newest app-backup job run', async () => {
|
|
jobRuns.recent.mockResolvedValue([
|
|
jobRun({
|
|
status: 'FAILED',
|
|
finishedAt: new Date('2026-10-13T01:00:05.000Z'),
|
|
result: null,
|
|
error: 'EACCES: permission denied',
|
|
}),
|
|
]);
|
|
|
|
const status = await service.getStatus();
|
|
|
|
expect(jobRuns.recent).toHaveBeenCalledWith(1, 'app-backup');
|
|
expect(status.lastJobRun).toEqual({
|
|
status: 'FAILED',
|
|
startedAt: '2026-10-13T01:00:02.000Z',
|
|
finishedAt: '2026-10-13T01:00:05.000Z',
|
|
error: 'EACCES: permission denied',
|
|
});
|
|
});
|
|
|
|
it('reports a running job run without an end and the run also without a path', async () => {
|
|
delete process.env.BACKUP_STORAGE_PATH;
|
|
jobRuns.recent.mockResolvedValue([
|
|
jobRun({ status: 'RUNNING', finishedAt: null, result: null }),
|
|
]);
|
|
|
|
const status = await service.getStatus();
|
|
|
|
expect(status.lastJobRun).toEqual({
|
|
status: 'RUNNING',
|
|
startedAt: '2026-10-13T01:00:02.000Z',
|
|
finishedAt: null,
|
|
error: null,
|
|
});
|
|
});
|
|
|
|
it('reports no job run before the first one', async () => {
|
|
const status = await service.getStatus();
|
|
|
|
expect(status.lastJobRun).toBeNull();
|
|
});
|
|
});
|
|
|
|
describe('runNow', () => {
|
|
it('keeps a backup larger than the import limit but logs an error', async () => {
|
|
process.env.BACKUP_MAX_IMPORT_BYTES = '1000';
|
|
|
|
const result = await service.runNow();
|
|
|
|
expect(fs.writeFile).toHaveBeenCalledTimes(1);
|
|
expect(result.fileName).toMatch(/^apicore-backup-.*\.json$/);
|
|
expect(error).toHaveBeenCalledTimes(1);
|
|
expect(error).toHaveBeenCalledWith(
|
|
expect.stringContaining('Import-Limit'),
|
|
);
|
|
});
|
|
|
|
it('logs nothing for a backup within the import limit', async () => {
|
|
await service.runNow();
|
|
|
|
expect(fs.writeFile).toHaveBeenCalledTimes(1);
|
|
expect(error).not.toHaveBeenCalled();
|
|
});
|
|
|
|
// A human asked for it: no due check, no wait for the schedule lock.
|
|
it('always writes, without the schedule lock', async () => {
|
|
onDisk([
|
|
{
|
|
name: 'apicore-backup-new.json',
|
|
size: 900,
|
|
mtime: new Date().toISOString(),
|
|
},
|
|
]);
|
|
|
|
await service.runNow();
|
|
|
|
expect(fs.writeFile).toHaveBeenCalledTimes(1);
|
|
expect(prisma.$transaction).not.toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
describe('runIfDue', () => {
|
|
const lastNight = '2026-10-12T01:00:00.000Z';
|
|
|
|
it('writes nothing without a path', async () => {
|
|
delete process.env.BACKUP_STORAGE_PATH;
|
|
|
|
const result = await service.runIfDue(new Date('2026-10-13T01:05:00Z'));
|
|
|
|
expect(result).toEqual({ written: false, reason: 'no-path' });
|
|
expect(backupService.exportSealed).not.toHaveBeenCalled();
|
|
expect(fs.writeFile).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('writes nothing while the newest backup covers the last scheduled time', async () => {
|
|
onDisk([{ name: 'apicore-backup-a.json', size: 900, mtime: lastNight }]);
|
|
|
|
const result = await service.runIfDue(new Date('2026-10-13T00:30:00Z'));
|
|
|
|
expect(result).toEqual({ written: false, reason: 'not-due' });
|
|
expect(backupService.exportSealed).not.toHaveBeenCalled();
|
|
expect(fs.writeFile).not.toHaveBeenCalled();
|
|
expect(fs.unlink).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('writes and prunes once the next scheduled time has passed', async () => {
|
|
process.env.BACKUP_RETENTION_COUNT = '1';
|
|
onDisk([
|
|
{ name: 'apicore-backup-a.json', size: 900, mtime: lastNight },
|
|
{
|
|
name: 'apicore-backup-b.json',
|
|
size: 800,
|
|
mtime: '2026-10-11T01:00:00.000Z',
|
|
},
|
|
]);
|
|
|
|
const result = await service.runIfDue(new Date('2026-10-13T01:05:00Z'));
|
|
|
|
expect(fs.writeFile).toHaveBeenCalledTimes(1);
|
|
expect(result).toEqual({
|
|
written: true,
|
|
fileName: 'apicore-backup-2026-10-13T01-05-00-000Z.json',
|
|
sizeBytes: Buffer.byteLength(JSON.stringify(SEALED)),
|
|
pruned: 1,
|
|
});
|
|
expect(fs.unlink).toHaveBeenCalledWith(
|
|
path.join(directory, 'apicore-backup-b.json'),
|
|
);
|
|
});
|
|
|
|
it('writes the first backup into an empty directory', async () => {
|
|
const result = await service.runIfDue(new Date('2026-10-13T00:30:00Z'));
|
|
|
|
expect(result).toEqual(expect.objectContaining({ written: true }));
|
|
expect(fs.writeFile).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('reads the schedule in Europe/Berlin', async () => {
|
|
process.env.BACKUP_CRON_SCHEDULE = '0 3 * * *';
|
|
onDisk([
|
|
{
|
|
name: 'apicore-backup-a.json',
|
|
size: 900,
|
|
mtime: '2026-10-12T03:00:00.000Z',
|
|
},
|
|
]);
|
|
|
|
// 03:00 Berlin = 01:00Z has passed, 03:00 UTC has not.
|
|
const result = await service.runIfDue(new Date('2026-10-13T02:00:00Z'));
|
|
|
|
expect(result.written).toBe(true);
|
|
});
|
|
|
|
it('does not count a file it could not delete as pruned', async () => {
|
|
process.env.BACKUP_RETENTION_COUNT = '1';
|
|
onDisk([
|
|
{ name: 'apicore-backup-a.json', size: 900, mtime: lastNight },
|
|
{
|
|
name: 'apicore-backup-b.json',
|
|
size: 800,
|
|
mtime: '2026-10-11T01:00:00.000Z',
|
|
},
|
|
{
|
|
name: 'apicore-backup-c.json',
|
|
size: 700,
|
|
mtime: '2026-10-10T01:00:00.000Z',
|
|
},
|
|
]);
|
|
(fs.unlink as jest.Mock)
|
|
.mockResolvedValueOnce(undefined)
|
|
.mockRejectedValueOnce(new Error('EBUSY'));
|
|
|
|
const result = await service.runIfDue(new Date('2026-10-13T01:05:00Z'));
|
|
|
|
expect(result).toEqual(expect.objectContaining({ pruned: 1 }));
|
|
expect(warn).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
// Two job pods (a CronJob run and a manual `kubectl create job`) must
|
|
// not both find the backup due: the check and the write happen under
|
|
// one database lock, so the second one sees the first one's file.
|
|
it('checks and writes only while holding the schedule lock', async () => {
|
|
await service.runIfDue(new Date('2026-10-13T01:05:00Z'));
|
|
|
|
expect(prisma.$transaction).toHaveBeenCalledWith(expect.any(Function), {
|
|
timeout: APP_BACKUP_LOCK_TIMEOUT_MS,
|
|
maxWait: expect.any(Number) as number,
|
|
});
|
|
expect(tx.$executeRaw).toHaveBeenCalledTimes(1);
|
|
const [lockCall] = tx.$executeRaw.mock.calls;
|
|
expect(sqlText(lockCall)).toContain('pg_advisory_xact_lock');
|
|
expect(lockCall.slice(1)).toEqual(['app-backup:schedule']);
|
|
const lockedAt = tx.$executeRaw.mock.invocationCallOrder[0];
|
|
expect(
|
|
(fs.readdir as jest.Mock).mock.invocationCallOrder[0],
|
|
).toBeGreaterThan(lockedAt);
|
|
expect(
|
|
(fs.writeFile as jest.Mock).mock.invocationCallOrder[0],
|
|
).toBeGreaterThan(lockedAt);
|
|
});
|
|
|
|
it('takes the time from the database clock when none is given', async () => {
|
|
onDisk([{ name: 'apicore-backup-a.json', size: 900, mtime: lastNight }]);
|
|
tx.$queryRaw.mockResolvedValue([
|
|
{ now: new Date('2026-10-13T00:30:00Z') },
|
|
]);
|
|
|
|
// The pod's own clock (2026-10-14) would say due; the database's says no.
|
|
jest.useFakeTimers({ now: new Date('2026-10-14T12:00:00Z') });
|
|
try {
|
|
await expect(service.runIfDue()).resolves.toEqual({
|
|
written: false,
|
|
reason: 'not-due',
|
|
});
|
|
} finally {
|
|
jest.useRealTimers();
|
|
}
|
|
expect(sqlText(tx.$queryRaw.mock.calls[0])).toContain('now()');
|
|
});
|
|
|
|
it('gives up the lock as soon as the backup is not due', async () => {
|
|
onDisk([{ name: 'apicore-backup-a.json', size: 900, mtime: lastNight }]);
|
|
|
|
await service.runIfDue(new Date('2026-10-13T00:30:00Z'));
|
|
|
|
expect(prisma.$transaction).toHaveBeenCalledTimes(1);
|
|
expect(fs.mkdir).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('fails the run when the export fails', async () => {
|
|
backupService.exportSealed.mockRejectedValue(new Error('export broke'));
|
|
|
|
await expect(
|
|
service.runIfDue(new Date('2026-10-13T01:05:00Z')),
|
|
).rejects.toThrow('export broke');
|
|
expect(fs.writeFile).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('fails the run on an invalid schedule instead of skipping silently', async () => {
|
|
process.env.BACKUP_CRON_SCHEDULE = 'every night';
|
|
|
|
await expect(
|
|
service.runIfDue(new Date('2026-10-13T01:05:00Z')),
|
|
).rejects.toThrow();
|
|
expect(fs.writeFile).not.toHaveBeenCalled();
|
|
});
|
|
});
|
|
});
|