feat: platform backbone and field capture, first part (checkpoint)
Platform backbone (plan 2026-10-13-plattform-rueckgrat), reviewed tasks: - run roles api/worker/job/all with shared startup checks (Task 1) - schema and migration for DomainEvent, deliveries, JobRun, ThrottlerHit (Task 2) - audit, notifications and domain events written in the same transaction (Task 3) - dispatcher with SKIP LOCKED claims, retries and stale release (Task 4) - worker building blocks: tick, LISTEN/NOTIFY wake-up, health (Task 5) - worker process and entry points (Task 6) - throttler counters in Postgres (Task 19) - separate keys per purpose with key ring (Task 20) - public origin page without exact location (Task 22) - shop keys split into read and write (Task 23) - Kustomize base, components, overlay encry, CI validation (Task 26) Field capture T3 (plan 2026-10-13-t3-erfassung-am-stand), reviewed tasks: - brood frames, read endpoints, apiary overview and offline pack - network basics, offline queue v3, navigation sheets, form building blocks - new inspection with last-time hints and weather, varroa per day, treatment prefill, apiary page with round - inspection photo (Task 10) is implemented, review pending Not rolled out. Remaining tasks follow in later commits. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
1 parent
10cbe9bf2d
commit
c01a38835a
340 files changed
+25695
-1808
No files matched your search
@@ -1,5 +1,41 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { isSealedBackup, openBackup, sealBackup } from './backup-crypto';
|
||||
import { BadRequestException, Logger } from '@nestjs/common';
|
||||
import {
|
||||
isSealedBackup,
|
||||
openBackup,
|
||||
sealBackup,
|
||||
type SealedBackup,
|
||||
} from './backup-crypto';
|
||||
|
||||
// Sealed by the code before the key ring, JWT_ACCESS_SECRET = 'x' * 40:
|
||||
// LEGACY_JWT_BACKUP without BACKUP_ENCRYPTION_KEY (key derived from the JWT
|
||||
// secret), KEY_BACKUP with BACKUP_ENCRYPTION_KEY = 'k' * 40.
|
||||
const FIXTURE_BUNDLE = {
|
||||
version: 3,
|
||||
exportedAt: '2026-10-01T09:00:00.000Z',
|
||||
data: { user: [{ id: 'u1' }] },
|
||||
};
|
||||
const LEGACY_JWT_BACKUP: SealedBackup = {
|
||||
format: 'apicore-backup',
|
||||
version: 3,
|
||||
keyId: '6ac8404ff5ae',
|
||||
exportedAt: '2026-10-01T09:00:00.000Z',
|
||||
algorithm: 'A256GCM+gzip',
|
||||
iv: 'vD+47m3e5IFJAexz',
|
||||
tag: 'LOgrmURu9vAMalQmlduDpw==',
|
||||
ciphertext:
|
||||
'yvuAGhVALAxXkqZM8FpeiPnGHD9llp/SVtdQ6S58oS4ezussfP7Lqj8ZDuNbxabwjUP4QBJvVdEyhKCFZ9w5YRbCRjDozizJyE6YwjqiCsAXWPyVZ2lZ6Fo8/G+/0HsoQg==',
|
||||
};
|
||||
const KEY_BACKUP: SealedBackup = {
|
||||
format: 'apicore-backup',
|
||||
version: 3,
|
||||
keyId: '14b11a29dc33',
|
||||
exportedAt: '2026-10-01T09:00:00.000Z',
|
||||
algorithm: 'A256GCM+gzip',
|
||||
iv: 'NKe1zNJfRPyBqeYL',
|
||||
tag: 'zU/oVHyV4bj5+wwF0lvGiA==',
|
||||
ciphertext:
|
||||
'ArjmmWq18F3OkPQYfx6x9XcjWdqP1SZC9zarUCRfV7bo6KRYx3pi5sNMRro6mg0eqlr4+C4jRKn3y+110kaWEFrY6J/ashF76pWSl1gzfiewkpxdqoyjkXRxrd4AnPrnfw==',
|
||||
};
|
||||
|
||||
describe('backup-crypto', () => {
|
||||
const originalEnv = { ...process.env };
|
||||
@@ -12,10 +48,13 @@ describe('backup-crypto', () => {
|
||||
beforeEach(() => {
|
||||
process.env.JWT_ACCESS_SECRET = 'x'.repeat(40);
|
||||
delete process.env.BACKUP_ENCRYPTION_KEY;
|
||||
delete process.env.BACKUP_ENCRYPTION_KEY_PREVIOUS;
|
||||
jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env = { ...originalEnv };
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('round-trips a bundle and hides its contents', () => {
|
||||
@@ -45,7 +84,17 @@ describe('backup-crypto', () => {
|
||||
).toThrow(BadRequestException);
|
||||
});
|
||||
|
||||
it('rejects a file with a truncated auth tag', () => {
|
||||
const sealed = sealBackup(bundle, bundle.exportedAt);
|
||||
const tag = Buffer.from(sealed.tag, 'base64').subarray(0, 4);
|
||||
|
||||
expect(() =>
|
||||
openBackup({ ...sealed, tag: tag.toString('base64') }),
|
||||
).toThrow(BadRequestException);
|
||||
});
|
||||
|
||||
it('rejects a file sealed with another key', () => {
|
||||
process.env.BACKUP_ENCRYPTION_KEY = 'k'.repeat(40);
|
||||
const sealed = sealBackup(bundle, bundle.exportedAt);
|
||||
process.env.BACKUP_ENCRYPTION_KEY = 'y'.repeat(40);
|
||||
|
||||
@@ -64,4 +113,80 @@ describe('backup-crypto', () => {
|
||||
process.env.BACKUP_ENCRYPTION_KEY = 'short';
|
||||
expect(() => sealBackup(bundle, bundle.exportedAt)).toThrow();
|
||||
});
|
||||
|
||||
it('opens a backup sealed with BACKUP_ENCRYPTION_KEY_PREVIOUS', () => {
|
||||
process.env.BACKUP_ENCRYPTION_KEY = 'k'.repeat(40);
|
||||
const sealed = sealBackup(bundle, bundle.exportedAt);
|
||||
process.env.BACKUP_ENCRYPTION_KEY = 'y'.repeat(40);
|
||||
process.env.BACKUP_ENCRYPTION_KEY_PREVIOUS = 'k'.repeat(40);
|
||||
|
||||
expect(openBackup(sealed)).toEqual(bundle);
|
||||
expect(sealBackup(bundle, bundle.exportedAt).keyId).not.toBe(sealed.keyId);
|
||||
});
|
||||
|
||||
it('still opens files sealed before the key ring (file format v3 unchanged)', () => {
|
||||
process.env.BACKUP_ENCRYPTION_KEY = 'k'.repeat(40);
|
||||
|
||||
expect(openBackup(KEY_BACKUP)).toEqual(FIXTURE_BUNDLE);
|
||||
expect(openBackup(LEGACY_JWT_BACKUP)).toEqual(FIXTURE_BUNDLE);
|
||||
});
|
||||
|
||||
it('only reads with the JWT-derived key, never seals with it once BACKUP_ENCRYPTION_KEY is set', () => {
|
||||
process.env.BACKUP_ENCRYPTION_KEY = 'k'.repeat(40);
|
||||
|
||||
expect(sealBackup(bundle, bundle.exportedAt).keyId).toBe(KEY_BACKUP.keyId);
|
||||
});
|
||||
|
||||
it('fails closed on a JWT-derived backup once JWT_ACCESS_SECRET changed', () => {
|
||||
process.env.BACKUP_ENCRYPTION_KEY = 'k'.repeat(40);
|
||||
process.env.JWT_ACCESS_SECRET = 'z'.repeat(40);
|
||||
|
||||
expect(() => openBackup(LEGACY_JWT_BACKUP)).toThrow(BadRequestException);
|
||||
});
|
||||
|
||||
it('refuses to seal in production without BACKUP_ENCRYPTION_KEY', () => {
|
||||
process.env.NODE_ENV = 'production';
|
||||
|
||||
expect(() => sealBackup(bundle, bundle.exportedAt)).toThrow(
|
||||
'BACKUP_ENCRYPTION_KEY is not set',
|
||||
);
|
||||
});
|
||||
|
||||
it('warns once that a development key stands in for BACKUP_ENCRYPTION_KEY', () => {
|
||||
jest.isolateModules(() => {
|
||||
// A fresh module copy, so earlier tests cannot have used up the warning.
|
||||
const nest =
|
||||
jest.requireActual<typeof import('@nestjs/common')>('@nestjs/common');
|
||||
const fresh =
|
||||
jest.requireActual<typeof import('./backup-crypto')>('./backup-crypto');
|
||||
const warn = jest
|
||||
.spyOn(nest.Logger.prototype, 'warn')
|
||||
.mockImplementation(() => undefined);
|
||||
|
||||
const sealed = fresh.sealBackup(bundle, bundle.exportedAt);
|
||||
fresh.sealBackup(bundle, bundle.exportedAt);
|
||||
fresh.openBackup(sealed);
|
||||
|
||||
expect(warn).toHaveBeenCalledTimes(1);
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
'BACKUP_ENCRYPTION_KEY not set – using development key derived from JWT_ACCESS_SECRET; never in production',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it('does not warn when only reading a legacy file next to a configured key', () => {
|
||||
jest.isolateModules(() => {
|
||||
const nest =
|
||||
jest.requireActual<typeof import('@nestjs/common')>('@nestjs/common');
|
||||
const fresh =
|
||||
jest.requireActual<typeof import('./backup-crypto')>('./backup-crypto');
|
||||
const warn = jest
|
||||
.spyOn(nest.Logger.prototype, 'warn')
|
||||
.mockImplementation(() => undefined);
|
||||
process.env.BACKUP_ENCRYPTION_KEY = 'k'.repeat(40);
|
||||
|
||||
expect(fresh.openBackup(LEGACY_JWT_BACKUP)).toEqual(FIXTURE_BUNDLE);
|
||||
expect(warn).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,17 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import * as crypto from 'node:crypto';
|
||||
import * as zlib from 'node:zlib';
|
||||
import {
|
||||
ENCRYPTION_KEY_VARIABLES,
|
||||
devFallbackSecret,
|
||||
findRingKey,
|
||||
gcmOpen,
|
||||
gcmSeal,
|
||||
readRingSecrets,
|
||||
requireJwtAccessSecret,
|
||||
type KeyRing,
|
||||
type RingKey,
|
||||
} from '../common/crypto/key-ring';
|
||||
|
||||
/**
|
||||
* A backup holds everything - password hashes, encrypted MFA secrets,
|
||||
@@ -9,17 +20,17 @@ import * as zlib from 'node:zlib';
|
||||
* keeps it unreadable outside ApiCore AND makes any change to it (or a
|
||||
* hand-made file) fail authentication before a single row is touched.
|
||||
*
|
||||
* Key: BACKUP_ENCRYPTION_KEY when set (needed to restore into another
|
||||
* deployment), otherwise derived from JWT_ACCESS_SECRET - so a deployment
|
||||
* can always restore its own backups. Either way the AES key comes from
|
||||
* HKDF with a backup-specific label, never the raw secret.
|
||||
* Keys follow the key ring (common/crypto/key-ring.ts): BACKUP_ENCRYPTION_KEY
|
||||
* seals, BACKUP_ENCRYPTION_KEY_PREVIOUS still opens, and files sealed before
|
||||
* the key was mandatory - with a key derived from JWT_ACCESS_SECRET - are
|
||||
* still read (never written, except outside production without
|
||||
* BACKUP_ENCRYPTION_KEY). The file names its key in `keyId`. Either way the
|
||||
* AES key comes from HKDF with a backup-specific label, never the raw
|
||||
* secret; derivation and file format v3 are unchanged, so older files open.
|
||||
*/
|
||||
export const SEALED_BACKUP_FORMAT = 'apicore-backup';
|
||||
export const SEALED_BACKUP_VERSION = 3;
|
||||
|
||||
const ALGORITHM = 'aes-256-gcm';
|
||||
const IV_BYTES = 12;
|
||||
const MIN_KEY_LENGTH = 32;
|
||||
// Decompressed size cap: a small file must not unpack into gigabytes.
|
||||
const MAX_PLAINTEXT_BYTES = 512 * 1024 * 1024;
|
||||
|
||||
@@ -35,25 +46,9 @@ export interface SealedBackup {
|
||||
ciphertext: string;
|
||||
}
|
||||
|
||||
function rootSecret(): { secret: string; source: 'backup' | 'jwt' } {
|
||||
const backupKey = process.env.BACKUP_ENCRYPTION_KEY;
|
||||
if (backupKey) {
|
||||
if (backupKey.length < MIN_KEY_LENGTH) {
|
||||
throw new Error(
|
||||
`BACKUP_ENCRYPTION_KEY must be at least ${MIN_KEY_LENGTH} characters`,
|
||||
);
|
||||
}
|
||||
return { secret: backupKey, source: 'backup' };
|
||||
}
|
||||
const jwtSecret = process.env.JWT_ACCESS_SECRET;
|
||||
if (!jwtSecret) {
|
||||
throw new Error('JWT_ACCESS_SECRET is not set');
|
||||
}
|
||||
return { secret: jwtSecret, source: 'jwt' };
|
||||
}
|
||||
type KeySource = 'backup' | 'jwt';
|
||||
|
||||
function deriveKey(): { key: Buffer; keyId: string } {
|
||||
const { secret, source } = rootSecret();
|
||||
function deriveBackupKey(secret: string, source: KeySource): RingKey {
|
||||
const key = Buffer.from(
|
||||
crypto.hkdfSync(
|
||||
'sha256',
|
||||
@@ -63,12 +58,35 @@ function deriveKey(): { key: Buffer; keyId: string } {
|
||||
32,
|
||||
),
|
||||
);
|
||||
const keyId = crypto
|
||||
.createHash('sha256')
|
||||
.update(key)
|
||||
.digest('hex')
|
||||
.slice(0, 12);
|
||||
return { key, keyId };
|
||||
const id = crypto.createHash('sha256').update(key).digest('hex').slice(0, 12);
|
||||
return { id, key };
|
||||
}
|
||||
|
||||
function legacyBackupKey(): RingKey {
|
||||
return deriveBackupKey(requireJwtAccessSecret(), 'jwt');
|
||||
}
|
||||
|
||||
function loadBackupRing(): KeyRing {
|
||||
const variable = ENCRYPTION_KEY_VARIABLES.backup;
|
||||
const secrets = readRingSecrets(variable);
|
||||
return {
|
||||
current: secrets.current
|
||||
? deriveBackupKey(secrets.current, 'backup')
|
||||
: deriveBackupKey(devFallbackSecret(variable), 'jwt'),
|
||||
previous: secrets.previous
|
||||
? deriveBackupKey(secrets.previous, 'backup')
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
/** The ring's keys first; the JWT-derived key of older files only as a last resort for reading. */
|
||||
function keyForReading(keyId: string): RingKey | null {
|
||||
const ringKey = findRingKey(loadBackupRing(), keyId);
|
||||
if (ringKey) {
|
||||
return ringKey;
|
||||
}
|
||||
const legacy = legacyBackupKey();
|
||||
return legacy.id === keyId ? legacy : null;
|
||||
}
|
||||
|
||||
function associatedData(
|
||||
@@ -80,24 +98,21 @@ function associatedData(
|
||||
}
|
||||
|
||||
export function sealBackup(bundle: unknown, exportedAt: string): SealedBackup {
|
||||
const { key, keyId } = deriveKey();
|
||||
const iv = crypto.randomBytes(IV_BYTES);
|
||||
const { current } = loadBackupRing();
|
||||
const header = {
|
||||
format: SEALED_BACKUP_FORMAT,
|
||||
version: SEALED_BACKUP_VERSION,
|
||||
keyId,
|
||||
keyId: current.id,
|
||||
exportedAt,
|
||||
} as const;
|
||||
const cipher = crypto.createCipheriv(ALGORITHM, key, iv);
|
||||
cipher.setAAD(associatedData(header));
|
||||
const compressed = zlib.gzipSync(Buffer.from(JSON.stringify(bundle)));
|
||||
const ciphertext = Buffer.concat([cipher.update(compressed), cipher.final()]);
|
||||
const sealed = gcmSeal(current.key, compressed, associatedData(header));
|
||||
return {
|
||||
...header,
|
||||
algorithm: 'A256GCM+gzip',
|
||||
iv: iv.toString('base64'),
|
||||
tag: cipher.getAuthTag().toString('base64'),
|
||||
ciphertext: ciphertext.toString('base64'),
|
||||
iv: sealed.iv.toString('base64'),
|
||||
tag: sealed.tag.toString('base64'),
|
||||
ciphertext: sealed.ciphertext.toString('base64'),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -112,32 +127,30 @@ export function isSealedBackup(value: unknown): value is SealedBackup {
|
||||
);
|
||||
}
|
||||
|
||||
/** The bundle inside a sealed backup; throws if the file was not made with this key or was changed. */
|
||||
/** The bundle inside a sealed backup; throws if no known key made the file or it was changed. */
|
||||
export function openBackup(sealed: SealedBackup): unknown {
|
||||
if (sealed.version !== SEALED_BACKUP_VERSION) {
|
||||
throw new BadRequestException(
|
||||
`Backup-Version ${String(sealed.version)} wird nicht unterstützt`,
|
||||
);
|
||||
}
|
||||
const { key, keyId } = deriveKey();
|
||||
if (sealed.keyId !== keyId) {
|
||||
const ringKey = keyForReading(sealed.keyId);
|
||||
if (!ringKey) {
|
||||
throw new BadRequestException(
|
||||
'Backup wurde mit einem anderen Schlüssel erstellt (BACKUP_ENCRYPTION_KEY prüfen)',
|
||||
'Backup wurde mit einem anderen Schlüssel erstellt (BACKUP_ENCRYPTION_KEY bzw. BACKUP_ENCRYPTION_KEY_PREVIOUS prüfen)',
|
||||
);
|
||||
}
|
||||
let compressed: Buffer;
|
||||
try {
|
||||
const decipher = crypto.createDecipheriv(
|
||||
ALGORITHM,
|
||||
key,
|
||||
Buffer.from(sealed.iv, 'base64'),
|
||||
compressed = gcmOpen(
|
||||
ringKey.key,
|
||||
{
|
||||
iv: Buffer.from(sealed.iv, 'base64'),
|
||||
tag: Buffer.from(sealed.tag, 'base64'),
|
||||
ciphertext: Buffer.from(sealed.ciphertext, 'base64'),
|
||||
},
|
||||
associatedData(sealed),
|
||||
);
|
||||
decipher.setAAD(associatedData(sealed));
|
||||
decipher.setAuthTag(Buffer.from(sealed.tag, 'base64'));
|
||||
compressed = Buffer.concat([
|
||||
decipher.update(Buffer.from(sealed.ciphertext, 'base64')),
|
||||
decipher.final(),
|
||||
]);
|
||||
} catch {
|
||||
throw new BadRequestException(
|
||||
'Backup-Datei ist beschädigt oder wurde verändert',
|
||||
|
||||
@@ -216,6 +216,27 @@ describe('BackupService', () => {
|
||||
);
|
||||
expect(documents.find((d) => d.id === 'doc-2')?.contentBase64).toBeNull();
|
||||
});
|
||||
|
||||
it('exports and restores Inspection.broodFrames', async () => {
|
||||
const row = {
|
||||
id: 'inspection-1',
|
||||
colonyId: 'colony-1',
|
||||
userId: 'known-user',
|
||||
broodFrames: 5,
|
||||
};
|
||||
prisma.inspection.findMany.mockResolvedValue([row]);
|
||||
|
||||
const bundle = await service.exportAll();
|
||||
const exported = bundle.data.inspection as { broodFrames: number }[];
|
||||
expect(exported[0]?.broodFrames).toBe(5);
|
||||
|
||||
await service.importAll(bundle, 'admin-1', TRUSTED);
|
||||
|
||||
const [[call]] = tx.inspection.upsert.mock.calls as [
|
||||
[{ create: { broodFrames: number } }],
|
||||
];
|
||||
expect(call.create.broodFrames).toBe(5);
|
||||
});
|
||||
});
|
||||
|
||||
describe('importAll', () => {
|
||||
|
||||
@@ -127,6 +127,14 @@ export const MODELS_EXCLUDED_FROM_BACKUP: Readonly<
|
||||
'Machine credentials stay out on purpose (ARCHITECTURE.md assumption 14): a new key is issued after a restore. BatchMovement/ShopStockRequest keep apiKeyId only as a plain historical reference.',
|
||||
IdempotencyRecord:
|
||||
'Volatile cache of first responses to retried create requests (ARCHITECTURE.md assumption 11) - a restored entry could only replay a stale response.',
|
||||
DomainEvent:
|
||||
'Operational outbox - side effects of restored data must never be replayed after a restore.',
|
||||
DomainEventDelivery:
|
||||
'Per-handler progress of DomainEvent rows, excluded with them.',
|
||||
WebhookDelivery:
|
||||
'Delivery log and retry queue - restoring it would resend old events to receivers.',
|
||||
JobRun: "Run history of this deployment's CronJobs, meaningless elsewhere.",
|
||||
ThrottlerHit: 'Rate-limit counters (UNLOGGED), valid for seconds to minutes.',
|
||||
};
|
||||
|
||||
const DOCUMENT_ENTITY_TYPES = new Set<string>(
|
||||
|
||||
Reference in new issue
Block a user