feat: platform backbone and field capture complete (checkpoint)
Platform backbone (plan 2026-10-13-plattform-rueckgrat), all tasks reviewed: - webhooks with delivery table, retries and stable delivery id - calendar handlers; a finished task removes its Outlook event - tasks, automations, inspections, varroa, treatments, feedings, harvests, colonies, wintering, stock and bottling record domain events in the same transaction; legacy post-commit calls removed - row locks through one helper (lockRowForUpdate, lockRowsForShare, lockRowsForUpdate) - job framework with JobRun, cli exit codes, cleanup, task templates, daily evaluation, hourly app-backup and the rekey job - system API and page "Verwaltung > System" with retry - @nestjs/schedule removed, guard tests against in-process schedulers Field capture T3 (plan 2026-10-13-t3-erfassung-am-stand), all tasks reviewed: - inspection photo offline without metadata - bulk capture for treatments, feedings and varroa (backend + frontend) - offline stand mode with service worker, offline pages and packs - offline session, cold start and logout on shared devices Docs: ARCHITECTURE.md assumptions and sections updated, TASKS.md. Gates: backend lint, typecheck, 1770 unit, 420 e2e, build, no schema drift; frontend lint, typecheck, 913 tests, build. Not rolled out. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
1 parent
c01a38835a
commit
cdd30cf3ee
288 files changed
+39165
-2853
No files matched your search
@@ -0,0 +1,82 @@
|
||||
import { runInNewContext } from 'node:vm';
|
||||
import { BACKUP_SCHEDULE_TIME_ZONE, isBackupDue } from './backup-due';
|
||||
|
||||
describe('isBackupDue', () => {
|
||||
it('reads the schedule in Europe/Berlin by default', () => {
|
||||
expect(BACKUP_SCHEDULE_TIME_ZONE).toBe('Europe/Berlin');
|
||||
});
|
||||
|
||||
it('is due when there is no backup yet', () => {
|
||||
expect(
|
||||
isBackupDue('0 3 * * *', null, new Date('2026-10-13T00:30:00Z')),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('is not due again on the same day for "0 3 * * *"', () =>
|
||||
expect(
|
||||
isBackupDue(
|
||||
'0 3 * * *',
|
||||
new Date('2026-10-12T01:00:00Z'),
|
||||
new Date('2026-10-13T00:30:00Z'),
|
||||
),
|
||||
).toBe(false));
|
||||
|
||||
it('is due after the next 03:00 Europe/Berlin', () =>
|
||||
expect(
|
||||
isBackupDue(
|
||||
'0 3 * * *',
|
||||
new Date('2026-10-12T01:00:00Z'),
|
||||
new Date('2026-10-13T01:05:00Z'),
|
||||
),
|
||||
).toBe(true));
|
||||
|
||||
it('is due exactly at the next scheduled time', () =>
|
||||
expect(
|
||||
isBackupDue(
|
||||
'0 3 * * *',
|
||||
new Date('2026-10-12T01:00:00Z'),
|
||||
new Date('2026-10-13T01:00:00Z'),
|
||||
),
|
||||
).toBe(true));
|
||||
|
||||
it('counts a backup written after the scheduled time (e.g. "Jetzt sichern") as covering it', () =>
|
||||
expect(
|
||||
isBackupDue(
|
||||
'0 3 * * *',
|
||||
new Date('2026-10-13T01:10:00Z'),
|
||||
new Date('2026-10-13T02:00:00Z'),
|
||||
),
|
||||
).toBe(false));
|
||||
|
||||
it('stays due after a missed run until a backup exists', () =>
|
||||
expect(
|
||||
isBackupDue(
|
||||
'0 3 * * *',
|
||||
new Date('2026-10-10T01:00:00Z'),
|
||||
new Date('2026-10-13T05:00:00Z'),
|
||||
),
|
||||
).toBe(true));
|
||||
|
||||
it('uses the given time zone', () => {
|
||||
const lastBackupAt = new Date('2026-10-12T03:00:00Z');
|
||||
const now = new Date('2026-10-13T02:00:00Z');
|
||||
|
||||
// Next 03:00 UTC is 03:00Z, next 03:00 Berlin was 01:00Z.
|
||||
expect(isBackupDue('0 3 * * *', lastBackupAt, now, 'UTC')).toBe(false);
|
||||
expect(isBackupDue('0 3 * * *', lastBackupAt, now)).toBe(true);
|
||||
});
|
||||
|
||||
// fs.Stats.mtime is created by Node's core; under Jest it is not an
|
||||
// instance of the test's Date (cron checks `instanceof Date`).
|
||||
it('accepts a Date from another realm', () => {
|
||||
const foreign = runInNewContext('new Date("2026-10-12T01:00:00Z")') as Date;
|
||||
|
||||
expect(
|
||||
isBackupDue('0 3 * * *', foreign, new Date('2026-10-13T00:30:00Z')),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects an invalid schedule even without a backup', () => {
|
||||
expect(() => isBackupDue('every night', null, new Date())).toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,34 @@
|
||||
import { CronTime } from 'cron';
|
||||
|
||||
/**
|
||||
* The time zone a backup schedule is read in - the same as the CronJobs'
|
||||
* `timeZone`, so "0 3 * * *" means 03:00 in Germany, summer and winter.
|
||||
*/
|
||||
export const BACKUP_SCHEDULE_TIME_ZONE = 'Europe/Berlin';
|
||||
|
||||
/**
|
||||
* Whether the hourly app-backup check has to write a backup: true while no
|
||||
* backup exists, otherwise once the first scheduled time after the newest
|
||||
* backup has come. A backup written after a scheduled time (a manual
|
||||
* "Jetzt sichern", or a late run) covers that time; a missed or failed run
|
||||
* stays due until a backup exists. Throws on an invalid schedule, also
|
||||
* without a backup, so a broken schedule fails the run instead of being
|
||||
* skipped silently.
|
||||
*/
|
||||
export function isBackupDue(
|
||||
cronSchedule: string,
|
||||
lastBackupAt: Date | null,
|
||||
now: Date,
|
||||
timeZone: string = BACKUP_SCHEDULE_TIME_ZONE,
|
||||
): boolean {
|
||||
const schedule = new CronTime(cronSchedule, timeZone);
|
||||
if (lastBackupAt === null) {
|
||||
return true;
|
||||
}
|
||||
// A copy: fs.Stats.mtime comes from another realm than cron's Date check.
|
||||
const next = schedule.getNextDateFrom(
|
||||
new Date(lastBackupAt.getTime()),
|
||||
timeZone,
|
||||
);
|
||||
return next.toMillis() <= now.getTime();
|
||||
}
|
||||
@@ -1,9 +1,12 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { SchedulerRegistry } from '@nestjs/schedule';
|
||||
import { CronJob } from 'cron';
|
||||
import * as path from 'node:path';
|
||||
import { Prisma } from '../../generated/prisma';
|
||||
import { BackupSchedulerService } from './backup-scheduler.service';
|
||||
import { JobRun } from '../../generated/prisma';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { JobRunService } from '../system/job-run.service';
|
||||
import {
|
||||
APP_BACKUP_LOCK_TIMEOUT_MS,
|
||||
BackupSchedulerService,
|
||||
} from './backup-scheduler.service';
|
||||
import { BackupService } from './backup.service';
|
||||
import { BackupSettingsService } from './backup-settings.service';
|
||||
|
||||
@@ -17,12 +20,30 @@ jest.mock('node:fs/promises', () => ({
|
||||
|
||||
import * as fs from 'node:fs/promises';
|
||||
|
||||
type RawCall = [readonly string[], ...unknown[]];
|
||||
|
||||
const SEALED = {
|
||||
format: 'apicore-backup',
|
||||
exportedAt: '2026-10-13T01:05:00.000Z',
|
||||
ciphertext: 'x'.repeat(2000),
|
||||
};
|
||||
|
||||
describe('BackupSchedulerService', () => {
|
||||
const originalEnv = { ...process.env };
|
||||
const directory = path.resolve('/data/backups');
|
||||
let service: BackupSchedulerService;
|
||||
let backupService: { exportSealed: jest.Mock };
|
||||
let settings: { get: jest.Mock };
|
||||
let tx: {
|
||||
$executeRaw: jest.Mock<Promise<number>, RawCall>;
|
||||
$queryRaw: jest.Mock<Promise<{ now: Date }[]>, RawCall>;
|
||||
};
|
||||
let prisma: { $transaction: jest.Mock };
|
||||
let jobRuns: { recent: jest.Mock };
|
||||
let error: jest.SpyInstance;
|
||||
let warn: jest.SpyInstance;
|
||||
|
||||
const sqlText = (call: RawCall): string => call[0].join('?');
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
@@ -31,33 +52,43 @@ describe('BackupSchedulerService', () => {
|
||||
delete process.env.BACKUP_RETENTION_COUNT;
|
||||
delete process.env.BACKUP_CRON_SCHEDULE;
|
||||
error = jest.spyOn(Logger.prototype, 'error').mockImplementation(() => {});
|
||||
warn = jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => {});
|
||||
|
||||
backupService = {
|
||||
exportSealed: jest.fn().mockResolvedValue({
|
||||
format: 'apicore-backup',
|
||||
exportedAt: '2026-10-12T03:00:00.000Z',
|
||||
ciphertext: 'x'.repeat(2000),
|
||||
}),
|
||||
};
|
||||
const settings = {
|
||||
backupService = { exportSealed: jest.fn().mockResolvedValue(SEALED) };
|
||||
settings = {
|
||||
get: jest.fn().mockResolvedValue({
|
||||
storagePath: null,
|
||||
cronSchedule: null,
|
||||
retentionCount: null,
|
||||
}),
|
||||
};
|
||||
tx = {
|
||||
$executeRaw: jest.fn<Promise<number>, RawCall>().mockResolvedValue(1),
|
||||
$queryRaw: jest
|
||||
.fn<Promise<{ now: Date }[]>, RawCall>()
|
||||
.mockResolvedValue([{ now: new Date('2026-10-13T01:05:00Z') }]),
|
||||
};
|
||||
prisma = {
|
||||
$transaction: jest.fn((work: (client: typeof tx) => Promise<unknown>) =>
|
||||
work(tx),
|
||||
),
|
||||
};
|
||||
jobRuns = { recent: jest.fn().mockResolvedValue([]) };
|
||||
service = new BackupSchedulerService(
|
||||
backupService as unknown as BackupService,
|
||||
settings as unknown as BackupSettingsService,
|
||||
{} as SchedulerRegistry,
|
||||
prisma as unknown as PrismaService,
|
||||
jobRuns as unknown as JobRunService,
|
||||
);
|
||||
(fs.mkdir as jest.Mock).mockResolvedValue(undefined);
|
||||
(fs.writeFile as jest.Mock).mockResolvedValue(undefined);
|
||||
(fs.readdir as jest.Mock).mockResolvedValue([]);
|
||||
(fs.unlink as jest.Mock).mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
error.mockRestore();
|
||||
warn.mockRestore();
|
||||
process.env = { ...originalEnv };
|
||||
});
|
||||
|
||||
@@ -72,6 +103,19 @@ describe('BackupSchedulerService', () => {
|
||||
});
|
||||
}
|
||||
|
||||
function jobRun(overrides: Partial<JobRun> = {}): JobRun {
|
||||
return {
|
||||
id: 'run-1',
|
||||
job: 'app-backup',
|
||||
status: 'SUCCEEDED',
|
||||
startedAt: new Date('2026-10-13T01:00:02.000Z'),
|
||||
finishedAt: new Date('2026-10-13T01:00:09.000Z'),
|
||||
result: { written: true },
|
||||
error: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('getStatus', () => {
|
||||
it('reports the import limit and the size of the newest backup', async () => {
|
||||
process.env.BACKUP_MAX_IMPORT_BYTES = '1000';
|
||||
@@ -129,6 +173,49 @@ describe('BackupSchedulerService', () => {
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('reports the newest app-backup job run', async () => {
|
||||
jobRuns.recent.mockResolvedValue([
|
||||
jobRun({
|
||||
status: 'FAILED',
|
||||
finishedAt: new Date('2026-10-13T01:00:05.000Z'),
|
||||
result: null,
|
||||
error: 'EACCES: permission denied',
|
||||
}),
|
||||
]);
|
||||
|
||||
const status = await service.getStatus();
|
||||
|
||||
expect(jobRuns.recent).toHaveBeenCalledWith(1, 'app-backup');
|
||||
expect(status.lastJobRun).toEqual({
|
||||
status: 'FAILED',
|
||||
startedAt: '2026-10-13T01:00:02.000Z',
|
||||
finishedAt: '2026-10-13T01:00:05.000Z',
|
||||
error: 'EACCES: permission denied',
|
||||
});
|
||||
});
|
||||
|
||||
it('reports a running job run without an end and the run also without a path', async () => {
|
||||
delete process.env.BACKUP_STORAGE_PATH;
|
||||
jobRuns.recent.mockResolvedValue([
|
||||
jobRun({ status: 'RUNNING', finishedAt: null, result: null }),
|
||||
]);
|
||||
|
||||
const status = await service.getStatus();
|
||||
|
||||
expect(status.lastJobRun).toEqual({
|
||||
status: 'RUNNING',
|
||||
startedAt: '2026-10-13T01:00:02.000Z',
|
||||
finishedAt: null,
|
||||
error: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('reports no job run before the first one', async () => {
|
||||
const status = await service.getStatus();
|
||||
|
||||
expect(status.lastJobRun).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('runNow', () => {
|
||||
@@ -151,48 +238,188 @@ describe('BackupSchedulerService', () => {
|
||||
expect(fs.writeFile).toHaveBeenCalledTimes(1);
|
||||
expect(error).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// A human asked for it: no due check, no wait for the schedule lock.
|
||||
it('always writes, without the schedule lock', async () => {
|
||||
onDisk([
|
||||
{
|
||||
name: 'apicore-backup-new.json',
|
||||
size: 900,
|
||||
mtime: new Date().toISOString(),
|
||||
},
|
||||
]);
|
||||
|
||||
await service.runNow();
|
||||
|
||||
expect(fs.writeFile).toHaveBeenCalledTimes(1);
|
||||
expect(prisma.$transaction).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// The export reads every table through Prisma; a Prisma message quotes
|
||||
// query values, so a failed scheduled run logs name, code and model only.
|
||||
describe('scheduled run', () => {
|
||||
it('logs a failed scheduled backup without the Prisma message', async () => {
|
||||
const registry = {
|
||||
doesExist: jest.fn().mockReturnValue(false),
|
||||
addCronJob: jest.fn(),
|
||||
deleteCronJob: jest.fn(),
|
||||
};
|
||||
const scheduled = new BackupSchedulerService(
|
||||
backupService as unknown as BackupService,
|
||||
describe('runIfDue', () => {
|
||||
const lastNight = '2026-10-12T01:00:00.000Z';
|
||||
|
||||
it('writes nothing without a path', async () => {
|
||||
delete process.env.BACKUP_STORAGE_PATH;
|
||||
|
||||
const result = await service.runIfDue(new Date('2026-10-13T01:05:00Z'));
|
||||
|
||||
expect(result).toEqual({ written: false, reason: 'no-path' });
|
||||
expect(backupService.exportSealed).not.toHaveBeenCalled();
|
||||
expect(fs.writeFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('writes nothing while the newest backup covers the last scheduled time', async () => {
|
||||
onDisk([{ name: 'apicore-backup-a.json', size: 900, mtime: lastNight }]);
|
||||
|
||||
const result = await service.runIfDue(new Date('2026-10-13T00:30:00Z'));
|
||||
|
||||
expect(result).toEqual({ written: false, reason: 'not-due' });
|
||||
expect(backupService.exportSealed).not.toHaveBeenCalled();
|
||||
expect(fs.writeFile).not.toHaveBeenCalled();
|
||||
expect(fs.unlink).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('writes and prunes once the next scheduled time has passed', async () => {
|
||||
process.env.BACKUP_RETENTION_COUNT = '1';
|
||||
onDisk([
|
||||
{ name: 'apicore-backup-a.json', size: 900, mtime: lastNight },
|
||||
{
|
||||
get: jest.fn().mockResolvedValue({
|
||||
storagePath: null,
|
||||
cronSchedule: null,
|
||||
retentionCount: null,
|
||||
}),
|
||||
} as unknown as BackupSettingsService,
|
||||
registry as unknown as SchedulerRegistry,
|
||||
);
|
||||
backupService.exportSealed.mockRejectedValue(
|
||||
new Prisma.PrismaClientKnownRequestError('title "Max Mustermann"', {
|
||||
code: 'P2010',
|
||||
clientVersion: '7.9.1',
|
||||
meta: { modelName: 'Task' },
|
||||
}),
|
||||
);
|
||||
await scheduled.reload();
|
||||
const [[, job]] = registry.addCronJob.mock.calls as [[string, CronJob]];
|
||||
await job.stop();
|
||||
name: 'apicore-backup-b.json',
|
||||
size: 800,
|
||||
mtime: '2026-10-11T01:00:00.000Z',
|
||||
},
|
||||
]);
|
||||
|
||||
await job.fireOnTick();
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
const result = await service.runIfDue(new Date('2026-10-13T01:05:00Z'));
|
||||
|
||||
const failure = error.mock.calls.find(([message]) =>
|
||||
String(message).startsWith('Scheduled backup failed'),
|
||||
) as [string, unknown];
|
||||
expect(failure[0]).toContain('P2010');
|
||||
expect(failure[0]).not.toContain('Mustermann');
|
||||
expect(failure[1]).toBeUndefined();
|
||||
expect(fs.writeFile).toHaveBeenCalledTimes(1);
|
||||
expect(result).toEqual({
|
||||
written: true,
|
||||
fileName: 'apicore-backup-2026-10-13T01-05-00-000Z.json',
|
||||
sizeBytes: Buffer.byteLength(JSON.stringify(SEALED)),
|
||||
pruned: 1,
|
||||
});
|
||||
expect(fs.unlink).toHaveBeenCalledWith(
|
||||
path.join(directory, 'apicore-backup-b.json'),
|
||||
);
|
||||
});
|
||||
|
||||
it('writes the first backup into an empty directory', async () => {
|
||||
const result = await service.runIfDue(new Date('2026-10-13T00:30:00Z'));
|
||||
|
||||
expect(result).toEqual(expect.objectContaining({ written: true }));
|
||||
expect(fs.writeFile).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('reads the schedule in Europe/Berlin', async () => {
|
||||
process.env.BACKUP_CRON_SCHEDULE = '0 3 * * *';
|
||||
onDisk([
|
||||
{
|
||||
name: 'apicore-backup-a.json',
|
||||
size: 900,
|
||||
mtime: '2026-10-12T03:00:00.000Z',
|
||||
},
|
||||
]);
|
||||
|
||||
// 03:00 Berlin = 01:00Z has passed, 03:00 UTC has not.
|
||||
const result = await service.runIfDue(new Date('2026-10-13T02:00:00Z'));
|
||||
|
||||
expect(result.written).toBe(true);
|
||||
});
|
||||
|
||||
it('does not count a file it could not delete as pruned', async () => {
|
||||
process.env.BACKUP_RETENTION_COUNT = '1';
|
||||
onDisk([
|
||||
{ name: 'apicore-backup-a.json', size: 900, mtime: lastNight },
|
||||
{
|
||||
name: 'apicore-backup-b.json',
|
||||
size: 800,
|
||||
mtime: '2026-10-11T01:00:00.000Z',
|
||||
},
|
||||
{
|
||||
name: 'apicore-backup-c.json',
|
||||
size: 700,
|
||||
mtime: '2026-10-10T01:00:00.000Z',
|
||||
},
|
||||
]);
|
||||
(fs.unlink as jest.Mock)
|
||||
.mockResolvedValueOnce(undefined)
|
||||
.mockRejectedValueOnce(new Error('EBUSY'));
|
||||
|
||||
const result = await service.runIfDue(new Date('2026-10-13T01:05:00Z'));
|
||||
|
||||
expect(result).toEqual(expect.objectContaining({ pruned: 1 }));
|
||||
expect(warn).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
// Two job pods (a CronJob run and a manual `kubectl create job`) must
|
||||
// not both find the backup due: the check and the write happen under
|
||||
// one database lock, so the second one sees the first one's file.
|
||||
it('checks and writes only while holding the schedule lock', async () => {
|
||||
await service.runIfDue(new Date('2026-10-13T01:05:00Z'));
|
||||
|
||||
expect(prisma.$transaction).toHaveBeenCalledWith(expect.any(Function), {
|
||||
timeout: APP_BACKUP_LOCK_TIMEOUT_MS,
|
||||
maxWait: expect.any(Number) as number,
|
||||
});
|
||||
expect(tx.$executeRaw).toHaveBeenCalledTimes(1);
|
||||
const [lockCall] = tx.$executeRaw.mock.calls;
|
||||
expect(sqlText(lockCall)).toContain('pg_advisory_xact_lock');
|
||||
expect(lockCall.slice(1)).toEqual(['app-backup:schedule']);
|
||||
const lockedAt = tx.$executeRaw.mock.invocationCallOrder[0];
|
||||
expect(
|
||||
(fs.readdir as jest.Mock).mock.invocationCallOrder[0],
|
||||
).toBeGreaterThan(lockedAt);
|
||||
expect(
|
||||
(fs.writeFile as jest.Mock).mock.invocationCallOrder[0],
|
||||
).toBeGreaterThan(lockedAt);
|
||||
});
|
||||
|
||||
it('takes the time from the database clock when none is given', async () => {
|
||||
onDisk([{ name: 'apicore-backup-a.json', size: 900, mtime: lastNight }]);
|
||||
tx.$queryRaw.mockResolvedValue([
|
||||
{ now: new Date('2026-10-13T00:30:00Z') },
|
||||
]);
|
||||
|
||||
// The pod's own clock (2026-10-14) would say due; the database's says no.
|
||||
jest.useFakeTimers({ now: new Date('2026-10-14T12:00:00Z') });
|
||||
try {
|
||||
await expect(service.runIfDue()).resolves.toEqual({
|
||||
written: false,
|
||||
reason: 'not-due',
|
||||
});
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
expect(sqlText(tx.$queryRaw.mock.calls[0])).toContain('now()');
|
||||
});
|
||||
|
||||
it('gives up the lock as soon as the backup is not due', async () => {
|
||||
onDisk([{ name: 'apicore-backup-a.json', size: 900, mtime: lastNight }]);
|
||||
|
||||
await service.runIfDue(new Date('2026-10-13T00:30:00Z'));
|
||||
|
||||
expect(prisma.$transaction).toHaveBeenCalledTimes(1);
|
||||
expect(fs.mkdir).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('fails the run when the export fails', async () => {
|
||||
backupService.exportSealed.mockRejectedValue(new Error('export broke'));
|
||||
|
||||
await expect(
|
||||
service.runIfDue(new Date('2026-10-13T01:05:00Z')),
|
||||
).rejects.toThrow('export broke');
|
||||
expect(fs.writeFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('fails the run on an invalid schedule instead of skipping silently', async () => {
|
||||
process.env.BACKUP_CRON_SCHEDULE = 'every night';
|
||||
|
||||
await expect(
|
||||
service.runIfDue(new Date('2026-10-13T01:05:00Z')),
|
||||
).rejects.toThrow();
|
||||
expect(fs.writeFile).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,32 +1,51 @@
|
||||
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
|
||||
import { SchedulerRegistry } from '@nestjs/schedule';
|
||||
import { CronJob } from 'cron';
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import * as fs from 'node:fs/promises';
|
||||
import * as path from 'node:path';
|
||||
import { JobRunStatus, Prisma } from '../../generated/prisma';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { JobRunService } from '../system/job-run.service';
|
||||
import { BackupService } from './backup.service';
|
||||
import { BackupSettingsService } from './backup-settings.service';
|
||||
import { loggableError } from '../common/loggable-error';
|
||||
import {
|
||||
assertBackupCron,
|
||||
resolveGuiBackupPath,
|
||||
} from './backup-schedule-rules';
|
||||
import { isBackupDue } from './backup-due';
|
||||
import { resolveGuiBackupPath } from './backup-schedule-rules';
|
||||
import {
|
||||
backupMaxImportBytes,
|
||||
exceedsBackupImportLimit,
|
||||
formatMegabytes,
|
||||
} from './backup-import-limit';
|
||||
|
||||
const SCHEDULED_JOB_NAME = 'automatic-backup';
|
||||
const DEFAULT_CRON_SCHEDULE = '0 3 * * *';
|
||||
const DEFAULT_RETENTION_COUNT = 14;
|
||||
const BACKUP_FILE_PREFIX = 'apicore-backup-';
|
||||
const BACKUP_FILE_SUFFIX = '.json';
|
||||
|
||||
// Guards against two replicas (deployment runs with replicas: 2, no shared
|
||||
// DB table for scheduling state) both firing the same cron tick: if the most
|
||||
// recent backup file on this shared/mounted path is younger than this, a
|
||||
// concurrent replica already handled the tick, so this one skips.
|
||||
const DUPLICATE_RUN_GUARD_MS = 2 * 60 * 1000;
|
||||
/**
|
||||
* Advisory lock (pg_advisory_xact_lock on its hash) around the due check
|
||||
* and the write of a scheduled backup.
|
||||
*/
|
||||
const APP_BACKUP_LOCK = 'app-backup:schedule';
|
||||
|
||||
/**
|
||||
* How long a scheduled backup may hold the lock: the activeDeadlineSeconds
|
||||
* of the app-backup CronJob (3600 s), after which the pod is gone anyway.
|
||||
*/
|
||||
export const APP_BACKUP_LOCK_TIMEOUT_MS = 60 * 60 * 1000;
|
||||
|
||||
/** How long to wait for a free pooled connection to take the lock on. */
|
||||
const APP_BACKUP_LOCK_MAX_WAIT_MS = 10_000;
|
||||
|
||||
/** What one run of the app-backup job did (JobRun.result). */
|
||||
export type AppBackupResult =
|
||||
| { written: true; fileName: string; sizeBytes: number; pruned: number }
|
||||
| { written: false; reason: 'no-path' | 'not-due' };
|
||||
|
||||
/** The newest app-backup job run, as GET /backup/schedule shows it. */
|
||||
export interface BackupJobRunStatus {
|
||||
status: JobRunStatus;
|
||||
startedAt: string;
|
||||
finishedAt: string | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export interface BackupScheduleStatus {
|
||||
enabled: boolean;
|
||||
@@ -48,6 +67,9 @@ export interface BackupScheduleStatus {
|
||||
cronSchedule: boolean;
|
||||
retentionCount: boolean;
|
||||
};
|
||||
// The newest run of the hourly app-backup job (CronJob), null before the
|
||||
// first one. lastRunAt above is the newest backup file instead.
|
||||
lastJobRun: BackupJobRunStatus | null;
|
||||
}
|
||||
|
||||
interface EffectiveBackupSettings {
|
||||
@@ -57,20 +79,23 @@ interface EffectiveBackupSettings {
|
||||
envOverrides: BackupScheduleStatus['envOverrides'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Automatic backups. The schedule is not a timer in this process: the
|
||||
* app-backup CronJob runs every hour and calls runIfDue(), which reads path,
|
||||
* schedule and retention afresh (env before GUI), so a GUI change applies
|
||||
* from the next hourly check on. runNow() is the "Jetzt sichern" button.
|
||||
*/
|
||||
@Injectable()
|
||||
export class BackupSchedulerService implements OnModuleInit {
|
||||
export class BackupSchedulerService {
|
||||
private readonly logger = new Logger(BackupSchedulerService.name);
|
||||
|
||||
constructor(
|
||||
private readonly backupService: BackupService,
|
||||
private readonly backupSettingsService: BackupSettingsService,
|
||||
private readonly schedulerRegistry: SchedulerRegistry,
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly jobRuns: JobRunService,
|
||||
) {}
|
||||
|
||||
async onModuleInit(): Promise<void> {
|
||||
await this.reload();
|
||||
}
|
||||
|
||||
private async effectiveSettings(): Promise<EffectiveBackupSettings> {
|
||||
const envPath = process.env.BACKUP_STORAGE_PATH;
|
||||
const hasEnvPath = Boolean(envPath && envPath.trim().length > 0);
|
||||
@@ -106,55 +131,10 @@ export class BackupSchedulerService implements OnModuleInit {
|
||||
};
|
||||
}
|
||||
|
||||
// Re-registers (or tears down) the cron job to match the current effective
|
||||
// settings. Called at boot and again after a GUI settings update, so an
|
||||
// admin's schedule/path/retention change takes effect immediately without
|
||||
// a pod restart.
|
||||
async reload(): Promise<void> {
|
||||
const settings = await this.effectiveSettings();
|
||||
if (!settings.envOverrides.cronSchedule) {
|
||||
try {
|
||||
assertBackupCron(settings.cronSchedule);
|
||||
} catch {
|
||||
// Keep whatever is running instead of silently ending backups.
|
||||
this.logger.error(
|
||||
`Ungültiger Backup-Zeitplan "${settings.cronSchedule}" - bisheriger Zeitplan bleibt aktiv`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (this.schedulerRegistry.doesExist('cron', SCHEDULED_JOB_NAME)) {
|
||||
this.schedulerRegistry.deleteCronJob(SCHEDULED_JOB_NAME);
|
||||
}
|
||||
|
||||
if (!settings.path) {
|
||||
this.logger.log(
|
||||
'Kein Backup-Pfad konfiguriert (weder Env noch GUI) - automatische Backups deaktiviert',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const directory = settings.path;
|
||||
const job = CronJob.from({
|
||||
cronTime: settings.cronSchedule,
|
||||
onTick: () => {
|
||||
// Prisma errors by name/code/model only (see loggableError).
|
||||
this.runScheduled(directory).catch((error: unknown) => {
|
||||
const { message, stack } = loggableError(error);
|
||||
this.logger.error(`Scheduled backup failed: ${message}`, stack);
|
||||
});
|
||||
},
|
||||
start: true,
|
||||
});
|
||||
this.schedulerRegistry.addCronJob(SCHEDULED_JOB_NAME, job);
|
||||
this.logger.log(
|
||||
`Automatische Backups aktiv: schedule="${settings.cronSchedule}" path="${directory}" retention=${settings.retentionCount}`,
|
||||
);
|
||||
}
|
||||
|
||||
async getStatus(): Promise<BackupScheduleStatus> {
|
||||
const settings = await this.effectiveSettings();
|
||||
const importLimitBytes = backupMaxImportBytes();
|
||||
const lastJobRun = await this.lastJobRun();
|
||||
if (!settings.path) {
|
||||
return {
|
||||
enabled: false,
|
||||
@@ -166,6 +146,7 @@ export class BackupSchedulerService implements OnModuleInit {
|
||||
lastBackupSizeBytes: null,
|
||||
lastBackupExceedsImportLimit: false,
|
||||
envOverrides: settings.envOverrides,
|
||||
lastJobRun,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -185,11 +166,25 @@ export class BackupSchedulerService implements OnModuleInit {
|
||||
lastBackupSizeBytes !== null &&
|
||||
exceedsBackupImportLimit(lastBackupSizeBytes),
|
||||
envOverrides: settings.envOverrides,
|
||||
lastJobRun,
|
||||
};
|
||||
}
|
||||
|
||||
// Manual trigger (e.g. "Jetzt sichern" button) - always runs, no
|
||||
// duplicate-tick guard, since a human explicitly asked for it.
|
||||
private async lastJobRun(): Promise<BackupJobRunStatus | null> {
|
||||
const [run] = await this.jobRuns.recent(1, 'app-backup');
|
||||
if (!run) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
status: run.status,
|
||||
startedAt: run.startedAt.toISOString(),
|
||||
finishedAt: run.finishedAt?.toISOString() ?? null,
|
||||
error: run.error,
|
||||
};
|
||||
}
|
||||
|
||||
// Manual trigger ("Jetzt sichern") - always writes: no due check and no
|
||||
// schedule lock, since a human explicitly asked for it.
|
||||
async runNow(): Promise<{ fileName: string }> {
|
||||
const settings = await this.effectiveSettings();
|
||||
if (!settings.path) {
|
||||
@@ -197,32 +192,61 @@ export class BackupSchedulerService implements OnModuleInit {
|
||||
'Kein Backup-Pfad konfiguriert - weder per Umgebungsvariable noch über die GUI',
|
||||
);
|
||||
}
|
||||
const filePath = await this.writeBackup(settings.path);
|
||||
const { fileName } = await this.writeBackup(settings.path);
|
||||
await this.pruneOldBackups(settings.path, settings.retentionCount);
|
||||
// Only the name: the server's directory layout stays on the server.
|
||||
return { fileName: path.basename(filePath) };
|
||||
return { fileName };
|
||||
}
|
||||
|
||||
private async runScheduled(directory: string): Promise<void> {
|
||||
/**
|
||||
* One hourly check of the app-backup job: writes a backup if one is due
|
||||
* (isBackupDue against the newest backup file, schedule in
|
||||
* Europe/Berlin) and prunes beyond the retention. Check and write run
|
||||
* under one database advisory lock, so a second job pod that checks at
|
||||
* the same time waits and then finds the first pod's file - two pods
|
||||
* never both write. Failures are thrown and fail the job run.
|
||||
*/
|
||||
async runIfDue(now?: Date): Promise<AppBackupResult> {
|
||||
const settings = await this.effectiveSettings();
|
||||
const files = await this.listBackupFiles(directory);
|
||||
const mostRecent = files[0];
|
||||
if (
|
||||
mostRecent &&
|
||||
Date.now() - mostRecent.mtime.getTime() < DUPLICATE_RUN_GUARD_MS
|
||||
) {
|
||||
this.logger.log(
|
||||
'Skipping scheduled backup - another replica already wrote one for this tick',
|
||||
);
|
||||
return;
|
||||
if (!settings.path) {
|
||||
return { written: false, reason: 'no-path' };
|
||||
}
|
||||
|
||||
const filePath = await this.writeBackup(directory);
|
||||
this.logger.log(`Scheduled backup written to ${filePath}`);
|
||||
await this.pruneOldBackups(directory, settings.retentionCount);
|
||||
const directory = settings.path;
|
||||
return this.prisma.$transaction(
|
||||
async (tx): Promise<AppBackupResult> => {
|
||||
await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtextextended(${APP_BACKUP_LOCK}, 0))`;
|
||||
const [newest] = await this.listBackupFiles(directory);
|
||||
// The database clock, like every other time comparison of the
|
||||
// platform: all job pods see the same time.
|
||||
const checkedAt = now ?? (await this.databaseNow(tx));
|
||||
if (
|
||||
!isBackupDue(settings.cronSchedule, newest?.mtime ?? null, checkedAt)
|
||||
) {
|
||||
return { written: false, reason: 'not-due' };
|
||||
}
|
||||
const { fileName, sizeBytes } = await this.writeBackup(directory);
|
||||
this.logger.log(`Scheduled backup ${fileName} written`);
|
||||
const pruned = await this.pruneOldBackups(
|
||||
directory,
|
||||
settings.retentionCount,
|
||||
);
|
||||
return { written: true, fileName, sizeBytes, pruned };
|
||||
},
|
||||
{
|
||||
timeout: APP_BACKUP_LOCK_TIMEOUT_MS,
|
||||
maxWait: APP_BACKUP_LOCK_MAX_WAIT_MS,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async writeBackup(directory: string): Promise<string> {
|
||||
private async databaseNow(tx: Prisma.TransactionClient): Promise<Date> {
|
||||
const [row] = await tx.$queryRaw<{ now: Date }[]>`SELECT now() AS now`;
|
||||
return row.now;
|
||||
}
|
||||
|
||||
private async writeBackup(
|
||||
directory: string,
|
||||
): Promise<{ fileName: string; sizeBytes: number }> {
|
||||
const sealed = await this.backupService.exportSealed();
|
||||
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
const timestamp = sealed.exportedAt.replace(/[:.]/g, '-');
|
||||
@@ -241,26 +265,30 @@ export class BackupSchedulerService implements OnModuleInit {
|
||||
`Backup ${path.basename(filePath)} ist ${formatMegabytes(sizeBytes)} groß und überschreitet das Import-Limit von ${formatMegabytes(backupMaxImportBytes())} - eine Wiederherstellung würde scheitern (BACKUP_MAX_IMPORT_BYTES erhöhen)`,
|
||||
);
|
||||
}
|
||||
return filePath;
|
||||
return { fileName: path.basename(filePath), sizeBytes };
|
||||
}
|
||||
|
||||
/** Deletes all but the newest `retentionCount` backups; returns how many. */
|
||||
private async pruneOldBackups(
|
||||
directory: string,
|
||||
retentionCount: number,
|
||||
): Promise<void> {
|
||||
): Promise<number> {
|
||||
const files = await this.listBackupFiles(directory);
|
||||
const toDelete = files.slice(retentionCount);
|
||||
await Promise.all(
|
||||
const deleted = await Promise.all(
|
||||
toDelete.map((file) =>
|
||||
fs
|
||||
.unlink(file.filePath)
|
||||
.catch((error) =>
|
||||
fs.unlink(file.filePath).then(
|
||||
() => true,
|
||||
(error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Could not delete old backup ${file.filePath}: ${String(error)}`,
|
||||
),
|
||||
),
|
||||
);
|
||||
return false;
|
||||
},
|
||||
),
|
||||
),
|
||||
);
|
||||
return deleted.filter(Boolean).length;
|
||||
}
|
||||
|
||||
private async listBackupFiles(
|
||||
|
||||
@@ -11,7 +11,8 @@ describe('BackupController', () => {
|
||||
const originalLimit = process.env.BACKUP_MAX_IMPORT_BYTES;
|
||||
let controller: BackupController;
|
||||
let backupService: { importFile: jest.Mock; exportSealed: jest.Mock };
|
||||
let scheduler: { reload: jest.Mock };
|
||||
let scheduler: Record<string, never>;
|
||||
let settings: { update: jest.Mock };
|
||||
let reauth: { verify: jest.Mock };
|
||||
const actor = { id: 'admin-1' } as AuthenticatedUser;
|
||||
const file = {
|
||||
@@ -28,12 +29,13 @@ describe('BackupController', () => {
|
||||
ciphertext: 'x'.repeat(2000),
|
||||
}),
|
||||
};
|
||||
scheduler = { reload: jest.fn().mockResolvedValue(undefined) };
|
||||
scheduler = {};
|
||||
settings = { update: jest.fn() };
|
||||
reauth = { verify: jest.fn().mockResolvedValue(undefined) };
|
||||
controller = new BackupController(
|
||||
backupService as unknown as BackupService,
|
||||
scheduler as unknown as BackupSchedulerService,
|
||||
{} as BackupSettingsService,
|
||||
settings as unknown as BackupSettingsService,
|
||||
reauth as unknown as ReauthService,
|
||||
);
|
||||
});
|
||||
@@ -46,23 +48,29 @@ describe('BackupController', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('re-registers the backup schedule after an import, since it may restore other backup settings', async () => {
|
||||
await controller.import(file, { password: 'secret' }, actor);
|
||||
|
||||
expect(backupService.importFile).toHaveBeenCalledTimes(1);
|
||||
expect(scheduler.reload).toHaveBeenCalledTimes(1);
|
||||
expect(scheduler.reload.mock.invocationCallOrder[0]).toBeGreaterThan(
|
||||
backupService.importFile.mock.invocationCallOrder[0],
|
||||
);
|
||||
});
|
||||
|
||||
it('leaves the schedule alone when the import fails', async () => {
|
||||
backupService.importFile.mockRejectedValue(new Error('broken file'));
|
||||
|
||||
// The app-backup job reads the settings on every run, so nothing in the
|
||||
// API has to be told about a restore or a settings change. The scheduler
|
||||
// double has no methods: any call on it fails these tests.
|
||||
it('imports without touching the schedule', async () => {
|
||||
await expect(
|
||||
controller.import(file, { password: 'secret' }, actor),
|
||||
).rejects.toThrow('broken file');
|
||||
expect(scheduler.reload).not.toHaveBeenCalled();
|
||||
).resolves.toEqual({ imported: {}, skipped: [] });
|
||||
|
||||
expect(backupService.importFile).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('saves settings without touching the schedule', async () => {
|
||||
const saved = { id: 'singleton', cronSchedule: '0 4 * * *' };
|
||||
settings.update.mockResolvedValue(saved);
|
||||
|
||||
await expect(
|
||||
controller.updateSettings({ cronSchedule: '0 4 * * *' }, actor),
|
||||
).resolves.toBe(saved);
|
||||
|
||||
expect(settings.update).toHaveBeenCalledWith(
|
||||
{ cronSchedule: '0 4 * * *' },
|
||||
'admin-1',
|
||||
);
|
||||
});
|
||||
|
||||
describe('export', () => {
|
||||
|
||||
@@ -65,13 +65,12 @@ export class BackupController {
|
||||
|
||||
@RequirePermission('backup:configure')
|
||||
@Patch('settings')
|
||||
async updateSettings(
|
||||
updateSettings(
|
||||
@Body() dto: UpdateBackupSettingsDto,
|
||||
@CurrentUser() actor: AuthenticatedUser,
|
||||
): Promise<BackupSettings> {
|
||||
const settings = await this.backupSettingsService.update(dto, actor.id);
|
||||
await this.backupSchedulerService.reload();
|
||||
return settings;
|
||||
// Read afresh by the next hourly app-backup check.
|
||||
return this.backupSettingsService.update(dto, actor.id);
|
||||
}
|
||||
|
||||
@RequirePermission('backup:export')
|
||||
@@ -129,9 +128,7 @@ export class BackupController {
|
||||
} catch {
|
||||
throw new BadRequestException('Datei ist kein gültiges JSON');
|
||||
}
|
||||
const summary = await this.backupService.importFile(parsed, actor.id);
|
||||
// The backup may have restored the schedule's settings too.
|
||||
await this.backupSchedulerService.reload();
|
||||
return summary;
|
||||
// Restored backup settings apply from the next hourly app-backup check.
|
||||
return this.backupService.importFile(parsed, actor.id);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { MulterModule } from '@nestjs/platform-express';
|
||||
import { ReauthModule } from '../auth/reauth/reauth.module';
|
||||
import { SystemModule } from '../system/system.module';
|
||||
import { BackupController } from './backup.controller';
|
||||
import { BackupService } from './backup.service';
|
||||
import { BackupSchedulerService } from './backup-scheduler.service';
|
||||
@@ -10,6 +11,8 @@ import { backupMaxImportBytes } from './backup-import-limit';
|
||||
@Module({
|
||||
imports: [
|
||||
ReauthModule,
|
||||
// JobRunService: GET /backup/schedule shows the newest app-backup run.
|
||||
SystemModule,
|
||||
// Upload limit of POST /backup/import. A factory rather than a constant
|
||||
// in the decorator: it runs once the app starts, after ConfigModule has
|
||||
// loaded .env, so the limit is the same one exports are checked against.
|
||||
@@ -19,5 +22,7 @@ import { backupMaxImportBytes } from './backup-import-limit';
|
||||
],
|
||||
controllers: [BackupController],
|
||||
providers: [BackupService, BackupSchedulerService, BackupSettingsService],
|
||||
// For the app-backup job (JobsModule, dist/cli.js).
|
||||
exports: [BackupSchedulerService],
|
||||
})
|
||||
export class BackupModule {}
|
||||
Reference in new issue
Block a user