feat: expand backup to all data and add automatic scheduled backups
Backup export/import now covers Role, User, RolePermission, Notification, AuditLog and Document (with actual file content), not just business data - matching "wirklich alle Daten". Role/User use natural keys (role.key, resource+action) so a bundle restores cleanly into a different database. Adds BackupSchedulerService (@nestjs/schedule) for cron-based automatic backups to a local path (BACKUP_STORAGE_PATH), with retention pruning and a file-mtime guard against duplicate runs across replicas. New GET /backup/schedule and POST /backup/schedule/run-now endpoints, plus a status card and manual trigger in the frontend Backup settings page.
This commit is contained in:
1 parent
d998f611e2
commit
7b23862a41
13 files changed
+853
-68
No files matched your search
@@ -1,9 +1,13 @@
|
||||
import { BadRequestException, Injectable } from '@nestjs/common';
|
||||
import * as crypto from 'node:crypto';
|
||||
import * as fs from 'node:fs/promises';
|
||||
import * as path from 'node:path';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { AuditLogService } from '../audit/audit-log.service';
|
||||
import { ALLOWED_DOCUMENT_MIME_TYPES } from '../documents/document-mime-types';
|
||||
|
||||
const BACKUP_VERSION = 1;
|
||||
const IMPORT_TRANSACTION_TIMEOUT_MS = 5 * 60 * 1000;
|
||||
const BACKUP_VERSION = 2;
|
||||
const IMPORT_TRANSACTION_TIMEOUT_MS = 10 * 60 * 1000;
|
||||
|
||||
export interface BackupBundle {
|
||||
version: number;
|
||||
@@ -16,21 +20,25 @@ export interface BackupImportSummary {
|
||||
}
|
||||
|
||||
// Excluded on purpose:
|
||||
// - User/Role/Permission/RolePermission/RefreshToken: access control is
|
||||
// managed live via the Benutzer & Rollen page, not round-tripped through a
|
||||
// JSON file - Permission rows are also code-derived (re-synced from the
|
||||
// catalog on every boot) and would just conflict with that.
|
||||
// - AuditLog: an immutable historical record, not data to restore.
|
||||
// - Notification: ephemeral, not worth backing up.
|
||||
// - Document: only the DB row (filename/path/size) would be backed up, not
|
||||
// the actual uploaded file bytes on disk - a restored row pointing at a
|
||||
// file that was never copied is worse than no row at all. File storage
|
||||
// needs its own (volume-level) backup strategy.
|
||||
// - Permission: code-derived, re-synced from the catalog on every boot -
|
||||
// including it would just fight that sync. RolePermission references
|
||||
// permissions by resource+action (not id) instead, resolved against
|
||||
// whatever the target database's own sync already produced.
|
||||
// - RefreshToken: live session tokens, meaningless once restored (expired,
|
||||
// or a stale session that should never be revivable from a backup file).
|
||||
//
|
||||
// Every model below keeps a `userId`-shaped column pointing at the (excluded)
|
||||
// User table. On import, an id that doesn't exist in the target database is
|
||||
// replaced with the id of the admin performing the import - see
|
||||
// resolveUserId/resolveOptionalUserId.
|
||||
// Role/User/RolePermission ARE included (v2) - Role is matched by its `key`
|
||||
// (not id, since a fresh deployment's seeded system roles get a new
|
||||
// database-generated id every time) and User carries a `roleKey` instead of
|
||||
// a raw roleId for the same reason. Every other model's `userId`-shaped
|
||||
// column is resolved against the now-imported User table; an id that still
|
||||
// doesn't exist there (e.g. a partial/older backup) falls back to the id of
|
||||
// the admin performing the import.
|
||||
//
|
||||
// Document is included with the actual file content (base64), not just the
|
||||
// DB row - the row alone would point at a path that was never copied. A row
|
||||
// whose file could not be read at export time is skipped on import rather
|
||||
// than restored with a dangling path.
|
||||
type Row = Record<string, unknown>;
|
||||
|
||||
function asString(row: Row, key: string): string | null {
|
||||
@@ -38,6 +46,10 @@ function asString(row: Row, key: string): string | null {
|
||||
return typeof value === 'string' ? value : null;
|
||||
}
|
||||
|
||||
function documentsStorageRoot(): string {
|
||||
return process.env.DOCUMENTS_STORAGE_PATH ?? './uploads';
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class BackupService {
|
||||
constructor(
|
||||
@@ -47,6 +59,28 @@ export class BackupService {
|
||||
|
||||
async exportAll(): Promise<BackupBundle> {
|
||||
const data: Record<string, unknown[]> = {};
|
||||
|
||||
data.role = await this.prisma.role.findMany();
|
||||
const rolePermissions = await this.prisma.rolePermission.findMany({
|
||||
include: {
|
||||
role: { select: { key: true } },
|
||||
permission: { select: { resource: true, action: true } },
|
||||
},
|
||||
});
|
||||
data.rolePermission = rolePermissions.map((rp) => ({
|
||||
roleKey: rp.role.key,
|
||||
resource: rp.permission.resource,
|
||||
action: rp.permission.action,
|
||||
}));
|
||||
const users = await this.prisma.user.findMany({
|
||||
include: { role: { select: { key: true } } },
|
||||
});
|
||||
data.user = users.map(({ role, ...rest }) => {
|
||||
const withoutRoleId: Record<string, unknown> = { ...rest };
|
||||
delete withoutRoleId.roleId;
|
||||
return { ...withoutRoleId, roleKey: role.key };
|
||||
});
|
||||
|
||||
data.apiary = await this.prisma.apiary.findMany();
|
||||
data.queen = await this.prisma.queen.findMany();
|
||||
data.colony = await this.prisma.colony.findMany();
|
||||
@@ -80,6 +114,26 @@ export class BackupService {
|
||||
data.automationRule = await this.prisma.automationRule.findMany();
|
||||
data.siteSettings = await this.prisma.siteSettings.findMany();
|
||||
|
||||
const documents = await this.prisma.document.findMany();
|
||||
data.document = await Promise.all(
|
||||
documents.map(async (document) => {
|
||||
const { storagePath, ...rest } = document;
|
||||
try {
|
||||
const contentBase64 = (await fs.readFile(storagePath)).toString(
|
||||
'base64',
|
||||
);
|
||||
return { ...rest, contentBase64 };
|
||||
} catch {
|
||||
// File missing on disk - keep the DB row's metadata visible in the
|
||||
// export, but with no content to restore from.
|
||||
return { ...rest, contentBase64: null };
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
data.notification = await this.prisma.notification.findMany();
|
||||
data.auditLog = await this.prisma.auditLog.findMany();
|
||||
|
||||
return {
|
||||
version: BACKUP_VERSION,
|
||||
exportedAt: new Date().toISOString(),
|
||||
@@ -100,18 +154,6 @@ export class BackupService {
|
||||
throw new BadRequestException('Invalid backup file');
|
||||
}
|
||||
|
||||
const knownUserIds = new Set(
|
||||
(await this.prisma.user.findMany({ select: { id: true } })).map(
|
||||
(user) => user.id,
|
||||
),
|
||||
);
|
||||
const resolveUserId = (rawUserId: string | null): string =>
|
||||
rawUserId !== null && knownUserIds.has(rawUserId)
|
||||
? rawUserId
|
||||
: actingUserId;
|
||||
const resolveOptionalUserId = (rawUserId: string | null): string | null =>
|
||||
rawUserId === null ? null : resolveUserId(rawUserId);
|
||||
|
||||
const rowsOf = (key: string): Row[] => {
|
||||
const value = bundle.data[key];
|
||||
return Array.isArray(value) ? (value as Row[]) : [];
|
||||
@@ -121,6 +163,86 @@ export class BackupService {
|
||||
|
||||
await this.prisma.$transaction(
|
||||
async (tx) => {
|
||||
const roleKeyToId = new Map<string, string>();
|
||||
for (const row of rowsOf('role')) {
|
||||
const withoutId: Row = { ...row };
|
||||
delete withoutId.id;
|
||||
const upserted = await tx.role.upsert({
|
||||
where: { key: row.key as string },
|
||||
create: row as never,
|
||||
update: withoutId as never,
|
||||
});
|
||||
roleKeyToId.set(row.key as string, upserted.id);
|
||||
}
|
||||
imported.role = rowsOf('role').length;
|
||||
|
||||
function resolveRoleId(roleKey: string): string {
|
||||
const id = roleKeyToId.get(roleKey);
|
||||
if (!id) {
|
||||
throw new BadRequestException(
|
||||
`Backup references unknown role "${roleKey}"`,
|
||||
);
|
||||
}
|
||||
return id;
|
||||
}
|
||||
|
||||
for (const row of rowsOf('user')) {
|
||||
const { roleKey, ...rest } = row;
|
||||
const data = {
|
||||
...rest,
|
||||
roleId: resolveRoleId(roleKey as string),
|
||||
};
|
||||
await tx.user.upsert({
|
||||
where: { id: row.id as string },
|
||||
create: data as never,
|
||||
update: data as never,
|
||||
});
|
||||
}
|
||||
imported.user = rowsOf('user').length;
|
||||
|
||||
for (const row of rowsOf('rolePermission')) {
|
||||
const roleId = resolveRoleId(row.roleKey as string);
|
||||
const permission = await tx.permission.findUnique({
|
||||
where: {
|
||||
resource_action: {
|
||||
resource: row.resource as string,
|
||||
action: row.action as string,
|
||||
},
|
||||
},
|
||||
});
|
||||
if (!permission) {
|
||||
// Permission catalog is code-derived and synced at boot; a
|
||||
// resource/action from an older export that no longer exists in
|
||||
// this version of the app is skipped rather than failing the
|
||||
// whole import.
|
||||
continue;
|
||||
}
|
||||
await tx.rolePermission.upsert({
|
||||
where: {
|
||||
roleId_permissionId: { roleId, permissionId: permission.id },
|
||||
},
|
||||
create: { roleId, permissionId: permission.id },
|
||||
update: {},
|
||||
});
|
||||
}
|
||||
imported.rolePermission = rowsOf('rolePermission').length;
|
||||
|
||||
// Built after role/user import so a user id that only exists
|
||||
// because THIS SAME bundle just created it is still recognised.
|
||||
const knownUserIds = new Set(
|
||||
(await tx.user.findMany({ select: { id: true } })).map(
|
||||
(user) => user.id,
|
||||
),
|
||||
);
|
||||
const resolveUserId = (rawUserId: string | null): string =>
|
||||
rawUserId !== null && knownUserIds.has(rawUserId)
|
||||
? rawUserId
|
||||
: actingUserId;
|
||||
const resolveOptionalUserId = (
|
||||
rawUserId: string | null,
|
||||
): string | null =>
|
||||
rawUserId === null ? null : resolveUserId(rawUserId);
|
||||
|
||||
for (const row of rowsOf('apiary')) {
|
||||
await tx.apiary.upsert({
|
||||
where: { id: row.id as string },
|
||||
@@ -543,6 +665,78 @@ export class BackupService {
|
||||
});
|
||||
}
|
||||
imported.siteSettings = rowsOf('siteSettings').length;
|
||||
|
||||
// Writes the file to disk as part of this loop, before the
|
||||
// surrounding transaction is known to commit - if a later entity in
|
||||
// this same import fails and the transaction rolls back, a file
|
||||
// written here is simply orphaned on disk (wasted space), not a
|
||||
// correctness problem, since no DB row ends up pointing at it.
|
||||
let documentCount = 0;
|
||||
for (const row of rowsOf('document')) {
|
||||
const contentBase64 = row.contentBase64 as string | null | undefined;
|
||||
if (!contentBase64) {
|
||||
continue;
|
||||
}
|
||||
const buffer = Buffer.from(contentBase64, 'base64');
|
||||
const extension =
|
||||
ALLOWED_DOCUMENT_MIME_TYPES[row.mimeType as string] ?? '';
|
||||
const directory = path.join(
|
||||
documentsStorageRoot(),
|
||||
(row.entityType as string).toLowerCase(),
|
||||
);
|
||||
await fs.mkdir(directory, { recursive: true });
|
||||
const storagePath = path.join(
|
||||
directory,
|
||||
`${crypto.randomUUID()}${extension}`,
|
||||
);
|
||||
await fs.writeFile(storagePath, buffer);
|
||||
const withoutContent: Row = { ...row };
|
||||
delete withoutContent.contentBase64;
|
||||
const data = {
|
||||
...withoutContent,
|
||||
storagePath,
|
||||
uploadedByUserId: resolveOptionalUserId(
|
||||
asString(row, 'uploadedByUserId'),
|
||||
),
|
||||
};
|
||||
await tx.document.upsert({
|
||||
where: { id: row.id as string },
|
||||
create: data as never,
|
||||
update: data as never,
|
||||
});
|
||||
documentCount += 1;
|
||||
}
|
||||
imported.document = documentCount;
|
||||
|
||||
for (const row of rowsOf('notification')) {
|
||||
await tx.notification.upsert({
|
||||
where: { id: row.id as string },
|
||||
create: {
|
||||
...row,
|
||||
userId: resolveOptionalUserId(asString(row, 'userId')),
|
||||
} as never,
|
||||
update: {
|
||||
...row,
|
||||
userId: resolveOptionalUserId(asString(row, 'userId')),
|
||||
} as never,
|
||||
});
|
||||
}
|
||||
imported.notification = rowsOf('notification').length;
|
||||
|
||||
for (const row of rowsOf('auditLog')) {
|
||||
await tx.auditLog.upsert({
|
||||
where: { id: row.id as string },
|
||||
create: {
|
||||
...row,
|
||||
userId: resolveOptionalUserId(asString(row, 'userId')),
|
||||
} as never,
|
||||
update: {
|
||||
...row,
|
||||
userId: resolveOptionalUserId(asString(row, 'userId')),
|
||||
} as never,
|
||||
});
|
||||
}
|
||||
imported.auditLog = rowsOf('auditLog').length;
|
||||
},
|
||||
{ timeout: IMPORT_TRANSACTION_TIMEOUT_MS },
|
||||
);
|
||||
|
||||
Reference in new issue
Block a user