feat: platform backbone and field capture, first part (checkpoint)
Platform backbone (plan 2026-10-13-plattform-rueckgrat), reviewed tasks: - run roles api/worker/job/all with shared startup checks (Task 1) - schema and migration for DomainEvent, deliveries, JobRun, ThrottlerHit (Task 2) - audit, notifications and domain events written in the same transaction (Task 3) - dispatcher with SKIP LOCKED claims, retries and stale release (Task 4) - worker building blocks: tick, LISTEN/NOTIFY wake-up, health (Task 5) - worker process and entry points (Task 6) - throttler counters in Postgres (Task 19) - separate keys per purpose with key ring (Task 20) - public origin page without exact location (Task 22) - shop keys split into read and write (Task 23) - Kustomize base, components, overlay encry, CI validation (Task 26) Field capture T3 (plan 2026-10-13-t3-erfassung-am-stand), reviewed tasks: - brood frames, read endpoints, apiary overview and offline pack - network basics, offline queue v3, navigation sheets, form building blocks - new inspection with last-time hints and weather, varroa per day, treatment prefill, apiary page with round - inspection photo (Task 10) is implemented, review pending Not rolled out. Remaining tasks follow in later commits. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
Nick SzaunigandClaude Opus 5.5 committed
1 parent
10cbe9bf2d
commit
c01a38835a
340 files changed
+25695
-1808
No files matched your search
@@ -0,0 +1,459 @@
|
||||
// Policy checks for the rendered overlay (spec 2026-10-12 sections 5.2, 5.3, 6, 7).
|
||||
// The CI renders deploy/k8s/overlays/encry and every rollout stage and hands
|
||||
// them over as JSON (see .gitea/workflows/manifests.yml). Locally:
|
||||
// kubectl kustomize deploy/k8s/overlays/encry > /tmp/rendered.yaml
|
||||
// yq ea -o=json '[.]' /tmp/rendered.yaml > /tmp/rendered.json
|
||||
// node deploy/k8s/tests/render-stages.mjs /tmp/stages
|
||||
// RENDERED_MANIFESTS=/tmp/rendered.json RENDERED_STAGES=/tmp/stages/stages.json \
|
||||
// node --test deploy/k8s/tests/manifest-policy.test.mjs
|
||||
// RENDERED_MANIFESTS is the target state (all components) and gets the full
|
||||
// security checks. RENDERED_STAGES holds every intermediate rollout stage;
|
||||
// each must keep the application reachable.
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import {
|
||||
NAMESPACE, ip, isEmptySelector, matchesSelector, modelOf, podTemplateOf, policyTypes, syntheticPod,
|
||||
} from './netpol-model.mjs';
|
||||
|
||||
const APP_PODS = ['apicore-backend', 'apicore-worker', 'apicore-job', 'apicore-frontend'];
|
||||
const DB_CLIENT_APPS = ['apicore-backend', 'apicore-worker', 'apicore-job', 'apicore-db-backup'];
|
||||
const KNOWN_JOBS = ['daily-evaluation', 'task-templates', 'cleanup', 'app-backup', 'rekey'];
|
||||
const PRIVATE_V4 = ['10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16', '100.64.0.0/10', '169.254.0.0/16', '127.0.0.0/8'];
|
||||
const PRIVATE_V6 = ['fc00::/7', 'fe80::/10', '::1/128'];
|
||||
const MIGRATE = ['node', 'node_modules/prisma/build/index.js', 'migrate', 'deploy'];
|
||||
const SEED = ['node', 'dist/seed.js'];
|
||||
const POSTGRES = 'StatefulSet/apicore-postgres';
|
||||
const res = (cpuReq, cpuLim, memReq, memLim) => ({
|
||||
requests: { cpu: cpuReq, memory: memReq },
|
||||
limits: { cpu: cpuLim, memory: memLim },
|
||||
});
|
||||
const RESOURCES = {
|
||||
api: res('50m', '500m', '192Mi', '512Mi'),
|
||||
worker: res('20m', '200m', '128Mi', '256Mi'),
|
||||
frontend: res('30m', '300m', '128Mi', '256Mi'),
|
||||
job: res('50m', '500m', '192Mi', '512Mi'),
|
||||
};
|
||||
const NEVER = '0 0 31 2 *';
|
||||
const CRON_JOBS = {
|
||||
cleanup: { args: ['cleanup'], schedule: '0 4 * * *', suspend: false, backoffLimit: 1, deadline: 1800 },
|
||||
'daily-evaluation': { args: ['daily-evaluation'], schedule: '30 4 * * *', suspend: false, backoffLimit: 1, deadline: 1800 },
|
||||
'task-templates': { args: ['task-templates'], schedule: '0 5 * * *', suspend: false, backoffLimit: 1, deadline: 1800 },
|
||||
'app-backup': { args: ['app-backup'], schedule: '0 * * * *', suspend: false, backoffLimit: 0, deadline: 3600 },
|
||||
rekey: { args: ['rekey'], schedule: NEVER, suspend: true, backoffLimit: 0, deadline: 1800 },
|
||||
'rekey-dry-run': { args: ['rekey', '--dry-run'], schedule: NEVER, suspend: true, backoffLimit: 0, deadline: 1800 },
|
||||
};
|
||||
// Selectors are immutable (StatefulSet) or carry live traffic (Services), so
|
||||
// they must equal what runs in the cluster (read with kubectl on 2026-10-13).
|
||||
const LIVE_SELECTORS = {
|
||||
'Deployment/apicore-backend': { app: 'apicore-backend' },
|
||||
'Deployment/apicore-frontend': { app: 'apicore-frontend' },
|
||||
'StatefulSet/apicore-postgres': { 'app.kubernetes.io/name': 'apicore-postgres' },
|
||||
};
|
||||
const LIVE_SERVICES = {
|
||||
'apicore-backend': { selector: { app: 'apicore-backend' }, ports: [{ port: 80, targetPort: 3001 }] },
|
||||
'apicore-frontend': { selector: { app: 'apicore-frontend' }, ports: [{ port: 80, targetPort: 3000 }] },
|
||||
'apicore-postgres': {
|
||||
selector: { 'app.kubernetes.io/name': 'apicore-postgres' },
|
||||
ports: [{ name: 'postgres', port: 5432, targetPort: 'postgres' }],
|
||||
},
|
||||
};
|
||||
// Live hardening of the postgres container; the repo must never weaken it.
|
||||
const POSTGRES_SECURITY = {
|
||||
runAsUser: 70,
|
||||
runAsGroup: 70,
|
||||
runAsNonRoot: true,
|
||||
readOnlyRootFilesystem: true,
|
||||
allowPrivilegeEscalation: false,
|
||||
capabilities: { drop: ['ALL'] },
|
||||
};
|
||||
// Pods created by controllers outside this repo; policies may target them.
|
||||
const SOLVER_LABELS = { 'acme.cert-manager.io/http01-solver': 'true' };
|
||||
const TRAEFIK = syntheticPod('kube-system', { 'app.kubernetes.io/name': 'rke2-traefik' });
|
||||
const KUBE_DNS = syntheticPod('kube-system', { 'k8s-app': 'kube-dns' });
|
||||
const SOLVER = syntheticPod(NAMESPACE, SOLVER_LABELS, [{ name: 'http', containerPort: 8089 }]);
|
||||
|
||||
function loadJson(variable) {
|
||||
const file = process.env[variable];
|
||||
if (!file) throw new Error(`${variable} must point to rendered manifests as JSON (see header)`);
|
||||
return JSON.parse(readFileSync(file, 'utf8'));
|
||||
}
|
||||
|
||||
const target = modelOf(loadJson('RENDERED_MANIFESTS').filter(Boolean));
|
||||
const stages = Object.entries(loadJson('RENDERED_STAGES')).map(([name, docs]) => [name, modelOf(docs.filter(Boolean))]);
|
||||
|
||||
function onePod(model, app) {
|
||||
const found = model.byApp(app);
|
||||
assert.equal(found.length, 1, `expected exactly one pod template with app=${app}`);
|
||||
return found[0];
|
||||
}
|
||||
const mainContainer = (pod) => pod.containers[0];
|
||||
const envOf = (container) => Object.fromEntries((container.env ?? []).map((e) => [e.name, e.value]));
|
||||
const postgresOf = (model) => {
|
||||
const postgres = model.byOwner(POSTGRES);
|
||||
assert.ok(postgres, 'postgres StatefulSet missing');
|
||||
return postgres;
|
||||
};
|
||||
|
||||
// --- target state: security --------------------------------------------------
|
||||
|
||||
test('default deny for ingress and egress exists', () => {
|
||||
const denyAll = target.policies.filter(
|
||||
(p) =>
|
||||
isEmptySelector(p.spec.podSelector) &&
|
||||
policyTypes(p).includes('Ingress') &&
|
||||
policyTypes(p).includes('Egress') &&
|
||||
(p.spec.ingress ?? []).length === 0 &&
|
||||
(p.spec.egress ?? []).length === 0,
|
||||
);
|
||||
assert.equal(denyAll.length, 1, 'expected one namespace-wide default-deny policy');
|
||||
for (const pod of target.pods) {
|
||||
assert.ok(target.policiesFor(pod, 'Ingress').length > 0, `${pod.owner} is not isolated for ingress`);
|
||||
assert.ok(target.policiesFor(pod, 'Egress').length > 0, `${pod.owner} is not isolated for egress`);
|
||||
}
|
||||
});
|
||||
|
||||
test('every pod template can resolve DNS (UDP+TCP 53 to kube-dns)', () => {
|
||||
assert.ok(target.pods.length > 0, 'no pod templates rendered');
|
||||
const otherKubeSystemPod = syntheticPod('kube-system', { app: 'something-else' });
|
||||
for (const pod of [...target.pods, SOLVER]) {
|
||||
for (const protocol of ['UDP', 'TCP']) {
|
||||
assert.ok(target.canConnect(pod, KUBE_DNS, 53, protocol), `${pod.owner} cannot reach kube-dns on ${protocol}/53`);
|
||||
assert.ok(!target.canConnect(pod, otherKubeSystemPod, 53, protocol), `${pod.owner} reaches any kube-system pod on 53`);
|
||||
assert.ok(!target.canConnect(pod, ip('1.1.1.1'), 53, protocol), `${pod.owner} reaches an outside resolver on 53`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('postgres accepts 5432 only from db-client pods; only api, worker, jobs and db-backup carry db-client', () => {
|
||||
const postgres = postgresOf(target);
|
||||
const clients = target.pods.filter((p) => p.labels['db-client'] === 'true');
|
||||
assert.deepEqual([...new Set(clients.map((p) => p.labels.app))].sort(), [...DB_CLIENT_APPS].sort());
|
||||
|
||||
for (const client of clients) {
|
||||
assert.ok(target.canConnect(client, postgres, 5432), `${client.owner} cannot reach postgres on 5432`);
|
||||
assert.ok(!target.canConnect(client, postgres, 22), `${client.owner} reaches postgres on a port other than 5432`);
|
||||
}
|
||||
const strangers = [
|
||||
TRAEFIK,
|
||||
SOLVER,
|
||||
syntheticPod('other', { 'db-client': 'true' }),
|
||||
syntheticPod(NAMESPACE, { app: 'intruder' }),
|
||||
ip('10.42.0.99'),
|
||||
];
|
||||
for (const stranger of strangers) {
|
||||
assert.ok(!target.allowsIngress(postgres, stranger, 5432), `${stranger.owner} reaches postgres`);
|
||||
}
|
||||
for (const pod of target.pods.filter((p) => p.labels['db-client'] !== 'true' && p !== postgres)) {
|
||||
assert.ok(!target.canConnect(pod, postgres, 5432), `${pod.owner} reaches postgres without db-client`);
|
||||
}
|
||||
});
|
||||
|
||||
test('frontend has no path to 5432 and no ipBlock egress', () => {
|
||||
const frontend = onePod(target, 'apicore-frontend');
|
||||
// Egress alone must block it, whatever ingress policies exist live.
|
||||
assert.ok(!target.allowsEgress(frontend, postgresOf(target), 5432), 'frontend egress allows postgres');
|
||||
for (const address of ['10.43.0.5', '10.42.3.7', '1.1.1.1']) {
|
||||
assert.ok(!target.allowsEgress(frontend, ip(address), 5432), `frontend egress allows ${address}:5432`);
|
||||
}
|
||||
for (const policy of target.policiesFor(frontend, 'Egress')) {
|
||||
for (const rule of policy.spec.egress ?? []) {
|
||||
assert.ok((rule.to ?? []).length > 0, `${policy.metadata.name} allows egress to everything`);
|
||||
assert.ok(!rule.to.some((peer) => peer.ipBlock), `${policy.metadata.name} gives the frontend ipBlock egress`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('worker and job pods accept no ingress', () => {
|
||||
const targets = target.pods.filter((p) => ['apicore-worker', 'apicore-job', 'apicore-db-backup'].includes(p.labels.app));
|
||||
assert.ok(targets.some((p) => p.labels.app === 'apicore-worker'), 'worker missing');
|
||||
assert.ok(targets.some((p) => p.labels.app === 'apicore-job'), 'job pods missing');
|
||||
for (const pod of targets) {
|
||||
const ingressPolicies = target.policiesFor(pod, 'Ingress');
|
||||
assert.ok(ingressPolicies.length > 0, `${pod.owner} is not isolated for ingress`);
|
||||
for (const policy of ingressPolicies) {
|
||||
assert.equal((policy.spec.ingress ?? []).length, 0, `${policy.metadata.name} opens ingress to ${pod.owner}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('every 0.0.0.0/0 and ::/0 egress excludes private ranges and allows only TCP 443', () => {
|
||||
for (const policy of target.policies) {
|
||||
for (const rule of policy.spec.egress ?? []) {
|
||||
assert.ok((rule.to ?? []).length > 0, `${policy.metadata.name} has an egress rule without destinations`);
|
||||
for (const peer of rule.to.filter((p) => p.ipBlock)) {
|
||||
const { cidr, except = [] } = peer.ipBlock;
|
||||
const required = cidr === '0.0.0.0/0' ? PRIVATE_V4 : cidr === '::/0' ? PRIVATE_V6 : null;
|
||||
assert.ok(required, `${policy.metadata.name}: unexpected ipBlock ${cidr}`);
|
||||
for (const range of required) assert.ok(except.includes(range), `${policy.metadata.name}: ${cidr} lacks except ${range}`);
|
||||
assert.deepEqual(
|
||||
(rule.ports ?? []).map((p) => [p.protocol ?? 'TCP', p.port, p.endPort]),
|
||||
[['TCP', 443, undefined]],
|
||||
`${policy.metadata.name}: public egress must be TCP 443 only`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
const blocked = ['10.0.0.1', '172.16.0.1', '192.168.1.10', '100.64.0.1', '169.254.169.254', '127.0.0.1', 'fc00::1', 'fe80::1', '::1'];
|
||||
for (const app of ['apicore-backend', 'apicore-worker']) {
|
||||
const pod = onePod(target, app);
|
||||
assert.ok(target.canConnect(pod, ip('1.1.1.1'), 443), `${app} cannot reach a public IPv4 on 443`);
|
||||
assert.ok(target.canConnect(pod, ip('2606:4700::1111'), 443), `${app} cannot reach a public IPv6 on 443`);
|
||||
assert.ok(!target.canConnect(pod, ip('1.1.1.1'), 80), `${app} reaches a public IP on 80`);
|
||||
for (const address of blocked) assert.ok(!target.canConnect(pod, ip(address), 443), `${app} reaches ${address}:443`);
|
||||
}
|
||||
for (const pod of [...target.pods.filter((p) => !['apicore-backend', 'apicore-worker'].includes(p.labels.app)), SOLVER]) {
|
||||
assert.ok(!target.canConnect(pod, ip('1.1.1.1'), 443), `${pod.owner} reaches the internet`);
|
||||
}
|
||||
});
|
||||
|
||||
test('traefik reaches frontend 3000 and api 3001; the api takes nothing else', () => {
|
||||
const frontend = onePod(target, 'apicore-frontend');
|
||||
const api = onePod(target, 'apicore-backend');
|
||||
assert.ok(target.canConnect(TRAEFIK, frontend, 3000), 'traefik cannot reach the frontend');
|
||||
assert.ok(target.canConnect(TRAEFIK, api, 3001), 'traefik cannot reach the api');
|
||||
assert.ok(target.canConnect(frontend, api, 3001), 'frontend cannot reach the api');
|
||||
assert.ok(!target.allowsIngress(api, syntheticPod('other', { app: 'apicore-frontend' }), 3001), 'api open to other namespaces');
|
||||
assert.ok(!target.allowsIngress(api, onePod(target, 'apicore-worker'), 3001), 'api open to the worker');
|
||||
assert.ok(!target.allowsIngress(frontend, api, 3000), 'frontend open to the api');
|
||||
assert.ok(!target.allowsIngress(api, TRAEFIK, 3002), 'api open on another port');
|
||||
});
|
||||
|
||||
test('cert-manager http01 solver pods: traefik reaches 8089, nothing else does', () => {
|
||||
assert.ok(target.canConnect(TRAEFIK, SOLVER, 8089), 'traefik cannot reach the http01 solver');
|
||||
assert.ok(!target.allowsIngress(SOLVER, TRAEFIK, 3000), 'solver open on another port');
|
||||
const others = [
|
||||
...target.pods,
|
||||
syntheticPod('other', { 'app.kubernetes.io/name': 'rke2-traefik' }),
|
||||
syntheticPod(NAMESPACE, { app: 'intruder' }),
|
||||
ip('1.1.1.1'),
|
||||
];
|
||||
for (const other of others) {
|
||||
assert.ok(!target.allowsIngress(SOLVER, other, 8089), `${other.owner} reaches the solver`);
|
||||
}
|
||||
assert.ok(!target.canConnect(SOLVER, postgresOf(target), 5432), 'solver reaches postgres');
|
||||
});
|
||||
|
||||
test('all app pods: automountServiceAccountToken false, readOnlyRootFilesystem true, /tmp emptyDir', () => {
|
||||
const appPods = target.pods.filter((p) => APP_PODS.includes(p.labels.app));
|
||||
assert.deepEqual([...new Set(appPods.map((p) => p.labels.app))].sort(), [...APP_PODS].sort());
|
||||
for (const pod of appPods) {
|
||||
assert.equal(pod.spec.automountServiceAccountToken, false, `${pod.owner} mounts the service account token`);
|
||||
assert.equal(pod.spec.securityContext?.runAsNonRoot, true, `${pod.owner} may run as root`);
|
||||
const emptyDirs = new Set((pod.spec.volumes ?? []).filter((v) => v.emptyDir).map((v) => v.name));
|
||||
for (const c of [...pod.initContainers, ...pod.containers]) {
|
||||
const where = `${pod.owner} container ${c.name}`;
|
||||
assert.equal(c.securityContext?.readOnlyRootFilesystem, true, `${where}: root filesystem writable`);
|
||||
assert.equal(c.securityContext?.allowPrivilegeEscalation, false, `${where}: privilege escalation`);
|
||||
assert.deepEqual(c.securityContext?.capabilities?.drop, ['ALL'], `${where}: capabilities not dropped`);
|
||||
const tmp = (c.volumeMounts ?? []).find((m) => m.mountPath === '/tmp');
|
||||
assert.ok(tmp && emptyDirs.has(tmp.name), `${where}: /tmp is not an emptyDir`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('cronjobs: Forbid, Europe/Berlin, command node dist/cli.js <known job>', () => {
|
||||
const cronJobs = target.docs.filter((d) => d.kind === 'CronJob');
|
||||
for (const cj of cronJobs) assert.equal(cj.spec.concurrencyPolicy, 'Forbid', `${cj.metadata.name}`);
|
||||
const appJobs = cronJobs.filter((cj) => podTemplateOf(cj).metadata?.labels?.app === 'apicore-job');
|
||||
assert.deepEqual(
|
||||
appJobs.map((cj) => cj.metadata.name).sort(),
|
||||
Object.keys(CRON_JOBS).map((name) => `apicore-job-${name}`).sort(),
|
||||
);
|
||||
for (const cj of appJobs) {
|
||||
const name = cj.metadata.name.replace(/^apicore-job-/, '');
|
||||
const want = CRON_JOBS[name];
|
||||
const template = podTemplateOf(cj);
|
||||
const container = template.spec.containers[0];
|
||||
assert.equal(template.spec.containers.length, 1, `${name}: one container`);
|
||||
assert.equal(cj.spec.timeZone, 'Europe/Berlin', `${name}: timeZone`);
|
||||
assert.equal(cj.spec.schedule, want.schedule, `${name}: schedule`);
|
||||
assert.equal(cj.spec.suspend ?? false, want.suspend, `${name}: suspend`);
|
||||
assert.equal(cj.spec.startingDeadlineSeconds, 600, `${name}: startingDeadlineSeconds`);
|
||||
assert.equal(cj.spec.successfulJobsHistoryLimit, 3, `${name}: successfulJobsHistoryLimit`);
|
||||
assert.equal(cj.spec.failedJobsHistoryLimit, 3, `${name}: failedJobsHistoryLimit`);
|
||||
assert.equal(cj.spec.jobTemplate.spec.ttlSecondsAfterFinished, 86400, `${name}: ttlSecondsAfterFinished`);
|
||||
assert.equal(cj.spec.jobTemplate.spec.backoffLimit, want.backoffLimit, `${name}: backoffLimit`);
|
||||
assert.equal(cj.spec.jobTemplate.spec.activeDeadlineSeconds, want.deadline, `${name}: activeDeadlineSeconds`);
|
||||
assert.equal(template.spec.restartPolicy, 'Never', `${name}: restartPolicy`);
|
||||
assert.equal(template.metadata.labels['apicore.io/job'], name, `${name}: apicore.io/job label`);
|
||||
assert.deepEqual(container.command, ['node', 'dist/cli.js', ...want.args], `${name}: command`);
|
||||
assert.ok(KNOWN_JOBS.includes(container.command[2]), `${name}: unknown job ${container.command[2]}`);
|
||||
assert.equal(envOf(container).APICORE_ROLE, 'job', `${name}: APICORE_ROLE`);
|
||||
}
|
||||
const backup = appJobs.find((cj) => cj.metadata.name === 'apicore-job-app-backup');
|
||||
const backupPod = podTemplateOf(backup).spec;
|
||||
const dataVolume = backupPod.volumes.find((v) => v.persistentVolumeClaim?.claimName === 'apicore-data');
|
||||
assert.ok(dataVolume, 'app-backup does not use the apicore-data claim');
|
||||
assert.ok(
|
||||
backupPod.containers[0].volumeMounts.some((m) => m.name === dataVolume.name && m.mountPath === '/data'),
|
||||
'app-backup does not mount apicore-data on /data',
|
||||
);
|
||||
});
|
||||
|
||||
test('resources match the spec table; HPA api 1–2 at 70 % CPU', () => {
|
||||
const api = onePod(target, 'apicore-backend');
|
||||
const worker = onePod(target, 'apicore-worker');
|
||||
assert.deepEqual(mainContainer(api).resources, RESOURCES.api, 'api resources');
|
||||
assert.deepEqual(mainContainer(worker).resources, RESOURCES.worker, 'worker resources');
|
||||
assert.deepEqual(mainContainer(onePod(target, 'apicore-frontend')).resources, RESOURCES.frontend, 'frontend resources');
|
||||
for (const job of target.byApp('apicore-job')) assert.deepEqual(mainContainer(job).resources, RESOURCES.job, `${job.owner} resources`);
|
||||
for (const pod of [api, worker]) {
|
||||
for (const c of pod.initContainers) assert.deepEqual(c.resources, RESOURCES.job, `${pod.owner} init ${c.name} resources`);
|
||||
}
|
||||
|
||||
const hpas = target.docs.filter((d) => d.kind === 'HorizontalPodAutoscaler');
|
||||
assert.equal(hpas.length, 1, 'expected one HPA');
|
||||
const [hpa] = hpas;
|
||||
assert.equal(hpa.apiVersion, 'autoscaling/v2');
|
||||
assert.deepEqual(hpa.spec.scaleTargetRef, { apiVersion: 'apps/v1', kind: 'Deployment', name: 'apicore-backend' });
|
||||
assert.equal(hpa.spec.minReplicas, 1);
|
||||
assert.equal(hpa.spec.maxReplicas, 2);
|
||||
assert.deepEqual(hpa.spec.metrics, [
|
||||
{ type: 'Resource', resource: { name: 'cpu', target: { type: 'Utilization', averageUtilization: 70 } } },
|
||||
]);
|
||||
assert.equal(api.doc.spec.replicas, undefined, 'the HPA owns the api replica count');
|
||||
});
|
||||
|
||||
test('api and worker run prisma migrate deploy as initContainer; only api seeds', () => {
|
||||
const api = onePod(target, 'apicore-backend');
|
||||
const worker = onePod(target, 'apicore-worker');
|
||||
assert.deepEqual(api.initContainers.map((c) => c.name), ['migrate', 'seed']);
|
||||
assert.deepEqual(worker.initContainers.map((c) => c.name), ['migrate']);
|
||||
for (const pod of [api, worker]) {
|
||||
const [migrate] = pod.initContainers;
|
||||
assert.deepEqual(migrate.command, MIGRATE, `${pod.owner}: migrate command`);
|
||||
assert.equal(envOf(migrate).HOME, '/tmp', `${pod.owner}: migrate HOME`);
|
||||
assert.equal(envOf(migrate).CHECKPOINT_DISABLE, '1', `${pod.owner}: migrate CHECKPOINT_DISABLE`);
|
||||
for (const c of pod.initContainers) assert.equal(c.image, mainContainer(pod).image, `${pod.owner}: ${c.name} image`);
|
||||
}
|
||||
assert.deepEqual(api.initContainers[1].command, SEED);
|
||||
const all = (p) => [...p.initContainers, ...p.containers];
|
||||
const seeding = target.pods.filter((p) => all(p).some((c) => c.command?.includes('dist/seed.js')));
|
||||
assert.deepEqual(seeding.map((p) => p.owner), [api.owner], 'only the api seeds');
|
||||
const migrating = target.pods.filter((p) => p.initContainers.some((c) => c.command?.includes('migrate')));
|
||||
assert.deepEqual(migrating.map((p) => p.owner).sort(), [api.owner, worker.owner].sort());
|
||||
|
||||
assert.equal(mainContainer(api).command, undefined, 'api runs the image CMD node dist/main.js');
|
||||
assert.equal(envOf(mainContainer(api)).APICORE_ROLE, 'api');
|
||||
assert.deepEqual(mainContainer(worker).command, ['node', 'dist/worker.js']);
|
||||
assert.equal(envOf(mainContainer(worker)).APICORE_ROLE, 'worker');
|
||||
assert.equal(envOf(mainContainer(worker)).WORKER_HEALTH_PORT, '3002');
|
||||
});
|
||||
|
||||
test('every image is pinned by sha256 digest', () => {
|
||||
for (const pod of target.pods) {
|
||||
for (const c of [...pod.initContainers, ...pod.containers]) {
|
||||
assert.match(c.image, /@sha256:[0-9a-f]{64}$/, `${pod.owner} container ${c.name}: ${c.image}`);
|
||||
assert.doesNotMatch(c.image, /:latest(@|$)/, `${pod.owner} container ${c.name} uses latest`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// --- consistency with the cluster and between objects ------------------------
|
||||
|
||||
test('workload and Service selectors equal the live cluster', () => {
|
||||
for (const [owner, selector] of Object.entries(LIVE_SELECTORS)) {
|
||||
const pod = target.byOwner(owner);
|
||||
assert.ok(pod, `${owner} missing`);
|
||||
assert.deepEqual(pod.doc.spec.selector, { matchLabels: selector }, `${owner}: selector differs from live`);
|
||||
}
|
||||
for (const [name, live] of Object.entries(LIVE_SERVICES)) {
|
||||
const service = target.docs.find((d) => d.kind === 'Service' && d.metadata.name === name);
|
||||
assert.ok(service, `Service ${name} missing`);
|
||||
assert.deepEqual(service.spec.selector, live.selector, `Service ${name}: selector differs from live`);
|
||||
const ports = service.spec.ports.map(({ name: n, port, targetPort }) => ({ ...(n ? { name: n } : {}), port, targetPort }));
|
||||
assert.deepEqual(ports, live.ports, `Service ${name}: ports differ from live`);
|
||||
}
|
||||
});
|
||||
|
||||
test('postgres keeps the live hardening and its static volume in every stage', () => {
|
||||
for (const [name, model] of [...stages, ['target', target]]) {
|
||||
const postgres = postgresOf(model);
|
||||
const { spec } = postgres;
|
||||
assert.equal(postgres.doc.spec.volumeClaimTemplates, undefined, `${name}: volumeClaimTemplates instead of the static volume`);
|
||||
assert.equal(spec.automountServiceAccountToken, false, `${name}: service account token mounted`);
|
||||
assert.equal(spec.enableServiceLinks, false, `${name}: service links enabled`);
|
||||
assert.deepEqual(spec.securityContext, { seccompProfile: { type: 'RuntimeDefault' } }, `${name}: pod securityContext`);
|
||||
assert.equal(postgres.containers.length, 1, `${name}: one container`);
|
||||
const [container] = postgres.containers;
|
||||
assert.deepEqual(container.securityContext, POSTGRES_SECURITY, `${name}: container securityContext differs from live`);
|
||||
assert.deepEqual(container.resources, res('30m', '1', '128Mi', '1Gi'), `${name}: resources differ from live`);
|
||||
for (const probe of ['startupProbe', 'livenessProbe', 'readinessProbe']) {
|
||||
assert.ok(container[probe]?.exec, `${name}: ${probe} missing`);
|
||||
}
|
||||
const volumes = Object.fromEntries((spec.volumes ?? []).map((v) => [v.name, v]));
|
||||
const mountedAt = Object.fromEntries((container.volumeMounts ?? []).map((m) => [m.mountPath, volumes[m.name]]));
|
||||
assert.equal(mountedAt['/var/lib/postgresql']?.persistentVolumeClaim?.claimName, 'apicore-postgres-data', `${name}: data volume`);
|
||||
for (const path of ['/tmp', '/var/run/postgresql', '/dev/shm']) {
|
||||
assert.ok(mountedAt[path]?.emptyDir?.sizeLimit, `${name}: ${path} is not a size-limited emptyDir`);
|
||||
}
|
||||
const claim = model.docs.find((d) => d.kind === 'PersistentVolumeClaim' && d.metadata.name === 'apicore-postgres-data');
|
||||
assert.ok(claim, `${name}: PVC apicore-postgres-data missing`);
|
||||
assert.deepEqual(
|
||||
claim.spec,
|
||||
{ accessModes: ['ReadWriteMany'], storageClassName: '', volumeName: 'pv-apicore-postgres', resources: { requests: { storage: '10Gi' } } },
|
||||
`${name}: PVC differs from the live static binding`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('every NetworkPolicy selector names pods that exist', () => {
|
||||
const existing = (selector) => target.pods.some((p) => matchesSelector(selector, p.labels));
|
||||
const outsideRepo = (selector) => Object.entries(SOLVER_LABELS).every(([k, v]) => selector.matchLabels?.[k] === v);
|
||||
for (const policy of target.policies) {
|
||||
const { podSelector } = policy.spec;
|
||||
if (!isEmptySelector(podSelector) && !outsideRepo(podSelector)) {
|
||||
assert.ok(existing(podSelector), `${policy.metadata.name}: podSelector matches no rendered pod`);
|
||||
}
|
||||
const peers = [...(policy.spec.ingress ?? []).flatMap((r) => r.from ?? []), ...(policy.spec.egress ?? []).flatMap((r) => r.to ?? [])];
|
||||
for (const peer of peers.filter((p) => p.podSelector && !p.namespaceSelector && !isEmptySelector(p.podSelector))) {
|
||||
assert.ok(existing(peer.podSelector), `${policy.metadata.name}: peer ${JSON.stringify(peer.podSelector)} matches no rendered pod`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// --- every rollout stage keeps the application reachable ---------------------
|
||||
|
||||
test('the stage file covers several rollout stages', () => {
|
||||
assert.ok(stages.length >= 4, `expected the rollout stages, got ${stages.map(([n]) => n)}`);
|
||||
assert.ok(stages.some(([, m]) => m.policies.length === 0), 'no stage without NetworkPolicies (Runbook step 1)');
|
||||
});
|
||||
|
||||
for (const [name, model] of [...stages, ['target', target]]) {
|
||||
test(`stage ${name}: every Service selects pods that expose its target port`, () => {
|
||||
for (const service of model.docs.filter((d) => d.kind === 'Service')) {
|
||||
const selected = model.pods.filter((p) => matchesSelector({ matchLabels: service.spec.selector }, p.labels));
|
||||
assert.ok(selected.length > 0, `Service ${service.metadata.name} selects no rendered pod`);
|
||||
for (const { port, targetPort = port } of service.spec.ports) {
|
||||
const exposes = (p) =>
|
||||
p.ports.some((c) => (typeof targetPort === 'string' ? c.name === targetPort : c.containerPort === targetPort));
|
||||
assert.ok(selected.every(exposes), `Service ${service.metadata.name}: target port ${targetPort} not exposed`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test(`stage ${name}: DNS, ingress, the api and every db client keep working`, () => {
|
||||
const api = onePod(model, 'apicore-backend');
|
||||
const frontend = onePod(model, 'apicore-frontend');
|
||||
const postgres = postgresOf(model);
|
||||
for (const pod of model.pods) {
|
||||
for (const protocol of ['UDP', 'TCP']) {
|
||||
assert.ok(model.canConnect(pod, KUBE_DNS, 53, protocol), `${pod.owner} cannot resolve names (${protocol})`);
|
||||
}
|
||||
}
|
||||
assert.ok(model.canConnect(TRAEFIK, frontend, 3000), 'traefik cannot reach the frontend');
|
||||
assert.ok(model.canConnect(TRAEFIK, api, 3001), 'traefik cannot reach the api');
|
||||
assert.ok(model.canConnect(frontend, api, 3001), 'frontend cannot reach the api');
|
||||
assert.ok(model.canConnect(TRAEFIK, SOLVER, 8089), 'traefik cannot reach the http01 solver');
|
||||
assert.ok(model.canConnect(api, postgres, 5432), 'api cannot reach postgres');
|
||||
assert.ok(model.canConnect(api, ip('1.1.1.1'), 443), 'api cannot reach the internet on 443');
|
||||
const dbClients = model.pods.filter((p) => ['apicore-worker', 'apicore-job', 'apicore-db-backup'].includes(p.labels.app));
|
||||
for (const client of dbClients) {
|
||||
assert.ok(model.canConnect(client, postgres, 5432), `${client.owner} cannot reach postgres`);
|
||||
}
|
||||
for (const worker of model.byApp('apicore-worker')) {
|
||||
assert.ok(model.canConnect(worker, ip('1.1.1.1'), 443), 'worker cannot reach the internet on 443');
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
// Kubernetes semantics for rendered manifests: pod templates, label
|
||||
// selectors and NetworkPolicies (the union of all policies that select a
|
||||
// pod, both the egress and the ingress side). Used by
|
||||
// manifest-policy.test.mjs to ask "can A reach B on port P".
|
||||
|
||||
export const NAMESPACE = 'apicore';
|
||||
|
||||
/** Pod template of a workload, or undefined for anything else. */
|
||||
export function podTemplateOf(doc) {
|
||||
if (doc.kind === 'CronJob') return doc.spec?.jobTemplate?.spec?.template;
|
||||
if (['Deployment', 'StatefulSet', 'DaemonSet', 'Job'].includes(doc.kind)) return doc.spec?.template;
|
||||
return doc.kind === 'Pod' ? doc : undefined;
|
||||
}
|
||||
|
||||
function podsOf(docs) {
|
||||
return docs.flatMap((doc) => {
|
||||
const template = podTemplateOf(doc);
|
||||
if (!template) return [];
|
||||
const namespace = doc.metadata.namespace ?? NAMESPACE;
|
||||
const containers = template.spec.containers ?? [];
|
||||
return [{
|
||||
kind: 'pod',
|
||||
owner: `${doc.kind}/${doc.metadata.name}`,
|
||||
doc,
|
||||
namespace,
|
||||
nsLabels: { 'kubernetes.io/metadata.name': namespace },
|
||||
labels: template.metadata?.labels ?? {},
|
||||
spec: template.spec,
|
||||
containers,
|
||||
initContainers: template.spec.initContainers ?? [],
|
||||
ports: containers.flatMap((c) => c.ports ?? []),
|
||||
}];
|
||||
});
|
||||
}
|
||||
|
||||
/** A pod that is not rendered here (Traefik, CoreDNS, cert-manager solver). */
|
||||
export function syntheticPod(namespace, labels, ports = []) {
|
||||
return {
|
||||
kind: 'pod', owner: `synthetic/${namespace}/${JSON.stringify(labels)}`, namespace,
|
||||
nsLabels: { 'kubernetes.io/metadata.name': namespace }, labels, ports,
|
||||
};
|
||||
}
|
||||
|
||||
export const ip = (address) => ({ kind: 'ip', owner: `ip/${address}`, ip: address });
|
||||
|
||||
export function matchesSelector(selector, labels) {
|
||||
if (!selector) return true;
|
||||
const plain = Object.entries(selector.matchLabels ?? {}).every(([k, v]) => labels[k] === v);
|
||||
const expressions = (selector.matchExpressions ?? []).every(({ key, operator, values = [] }) => {
|
||||
if (operator === 'In') return values.includes(labels[key]);
|
||||
if (operator === 'NotIn') return !values.includes(labels[key]);
|
||||
if (operator === 'Exists') return key in labels;
|
||||
if (operator === 'DoesNotExist') return !(key in labels);
|
||||
throw new Error(`unsupported selector operator ${operator}`);
|
||||
});
|
||||
return plain && expressions;
|
||||
}
|
||||
|
||||
export const isEmptySelector = (s) =>
|
||||
!s || (Object.keys(s.matchLabels ?? {}).length === 0 && (s.matchExpressions ?? []).length === 0);
|
||||
|
||||
export function policyTypes(policy) {
|
||||
return policy.spec.policyTypes ?? (policy.spec.egress ? ['Ingress', 'Egress'] : ['Ingress']);
|
||||
}
|
||||
|
||||
function parseIp(address) {
|
||||
if (address.includes(':')) {
|
||||
const [head, tail] = address.includes('::') ? address.split('::') : [address, null];
|
||||
const h = head ? head.split(':') : [];
|
||||
const t = tail ? tail.split(':') : [];
|
||||
const groups = tail === null ? h : [...h, ...Array(8 - h.length - t.length).fill('0'), ...t];
|
||||
return { bits: 128, value: groups.reduce((acc, g) => (acc << 16n) + BigInt(parseInt(g, 16)), 0n) };
|
||||
}
|
||||
return { bits: 32, value: address.split('.').reduce((acc, o) => (acc << 8n) + BigInt(Number(o)), 0n) };
|
||||
}
|
||||
|
||||
export function cidrContains(cidr, address) {
|
||||
const [network, prefix] = cidr.split('/');
|
||||
const net = parseIp(network);
|
||||
const host = parseIp(address);
|
||||
if (net.bits !== host.bits) return false;
|
||||
const shift = BigInt(net.bits - Number(prefix));
|
||||
return net.value >> shift === host.value >> shift;
|
||||
}
|
||||
|
||||
function peerMatches(peer, endpoint, policyNamespace) {
|
||||
if (peer.ipBlock) {
|
||||
return (
|
||||
endpoint.kind === 'ip' &&
|
||||
cidrContains(peer.ipBlock.cidr, endpoint.ip) &&
|
||||
!(peer.ipBlock.except ?? []).some((cidr) => cidrContains(cidr, endpoint.ip))
|
||||
);
|
||||
}
|
||||
if (endpoint.kind !== 'pod') return false;
|
||||
const namespaceOk = peer.namespaceSelector
|
||||
? matchesSelector(peer.namespaceSelector, endpoint.nsLabels)
|
||||
: endpoint.namespace === policyNamespace;
|
||||
return namespaceOk && matchesSelector(peer.podSelector, endpoint.labels);
|
||||
}
|
||||
|
||||
function portMatches(ports, port, protocol, destination) {
|
||||
if (!ports || ports.length === 0) return true;
|
||||
return ports.some((p) => {
|
||||
if ((p.protocol ?? 'TCP') !== protocol) return false;
|
||||
if (p.port === undefined) return true;
|
||||
if (typeof p.port === 'string') {
|
||||
return (destination.ports ?? []).some(
|
||||
(c) => c.name === p.port && c.containerPort === port && (c.protocol ?? 'TCP') === protocol,
|
||||
);
|
||||
}
|
||||
return p.endPort ? port >= p.port && port <= p.endPort : p.port === port;
|
||||
});
|
||||
}
|
||||
|
||||
/** Everything the tests need to reason about one rendered document set. */
|
||||
export function modelOf(docs) {
|
||||
const policies = docs.filter((d) => d.kind === 'NetworkPolicy');
|
||||
const pods = podsOf(docs);
|
||||
|
||||
const policiesFor = (pod, type) =>
|
||||
policies.filter(
|
||||
(p) =>
|
||||
(p.metadata.namespace ?? NAMESPACE) === pod.namespace &&
|
||||
matchesSelector(p.spec.podSelector, pod.labels) &&
|
||||
policyTypes(p).includes(type),
|
||||
);
|
||||
|
||||
function allows(direction, pod, peer, port, protocol) {
|
||||
const type = direction === 'egress' ? 'Egress' : 'Ingress';
|
||||
const relevant = policiesFor(pod, type);
|
||||
if (relevant.length === 0) return true;
|
||||
const destination = direction === 'egress' ? peer : pod;
|
||||
return relevant.some((policy) =>
|
||||
(policy.spec[direction] ?? []).some((rule) => {
|
||||
const peers = rule[direction === 'egress' ? 'to' : 'from'];
|
||||
const peerOk =
|
||||
!peers || peers.length === 0 ||
|
||||
peers.some((p) => peerMatches(p, peer, policy.metadata.namespace ?? NAMESPACE));
|
||||
return peerOk && portMatches(rule.ports, port, protocol, destination);
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const allowsEgress = (src, dst, port, protocol = 'TCP') => allows('egress', src, dst, port, protocol);
|
||||
const allowsIngress = (dst, src, port, protocol = 'TCP') => allows('ingress', dst, src, port, protocol);
|
||||
const canConnect = (src, dst, port, protocol = 'TCP') =>
|
||||
(src.kind !== 'pod' || allowsEgress(src, dst, port, protocol)) &&
|
||||
(dst.kind !== 'pod' || allowsIngress(dst, src, port, protocol));
|
||||
|
||||
return {
|
||||
docs,
|
||||
pods,
|
||||
policies,
|
||||
policiesFor,
|
||||
allowsEgress,
|
||||
allowsIngress,
|
||||
canConnect,
|
||||
byApp: (app) => pods.filter((p) => p.labels.app === app),
|
||||
byOwner: (owner) => pods.find((p) => p.owner === owner),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
// Renders every rollout stage of overlays/encry (Runbook steps 1, 2, 3 and 5)
|
||||
// so manifest-policy.test.mjs can check that each intermediate state keeps
|
||||
// the application reachable. A stage is the committed overlay with only the
|
||||
// components released up to that step; the overlay in the repo is not
|
||||
// touched, each stage is rendered from a temporary copy.
|
||||
//
|
||||
// Usage: node deploy/k8s/tests/render-stages.mjs <outDir>
|
||||
// Env: KUBECTL (default "kubectl"), YQ (default "yq")
|
||||
// Output: <outDir>/<stage>.yaml per stage (for kubeconform) and
|
||||
// <outDir>/stages.json = { "<stage>": [rendered documents] }
|
||||
import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { dirname, isAbsolute, join, relative, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const K8S_DIR = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const OVERLAY_FILE = join('overlays', 'encry', 'kustomization.yaml');
|
||||
const ROLLOUT_ORDER = ['network-policies', 'pod-hardening', 'backend-roles', 'api-hpa'];
|
||||
const STAGES = {
|
||||
'step1-ist-stand': [],
|
||||
'step2-network-policies': ROLLOUT_ORDER.slice(0, 1),
|
||||
'step3-roles-and-hardening': ROLLOUT_ORDER.slice(0, 3),
|
||||
'step5-hpa': ROLLOUT_ORDER,
|
||||
};
|
||||
const COMPONENT_LINE = /^\s+-\s+\.\.\/\.\.\/components\/([\w-]+)/;
|
||||
|
||||
/** Splits the overlay into the lines before and after its components list. */
|
||||
function splitOverlay(text) {
|
||||
const lines = text.split('\n');
|
||||
const start = lines.findIndex((l) => l.startsWith('components:'));
|
||||
if (start < 0) throw new Error(`${OVERLAY_FILE}: no components list`);
|
||||
let end = start + 1;
|
||||
while (end < lines.length && COMPONENT_LINE.test(lines[end])) end += 1;
|
||||
const listed = lines.slice(start + 1, end).map((l) => l.match(COMPONENT_LINE)[1]);
|
||||
if (listed.join() !== ROLLOUT_ORDER.join()) {
|
||||
throw new Error(
|
||||
`${OVERLAY_FILE} lists components [${listed}], render-stages.mjs expects [${ROLLOUT_ORDER}]. ` +
|
||||
'Update ROLLOUT_ORDER and STAGES together with the overlay and the runbook.',
|
||||
);
|
||||
}
|
||||
return { before: lines.slice(0, start), after: lines.slice(end) };
|
||||
}
|
||||
|
||||
function overlayFor(parts, components) {
|
||||
const list = components.length === 0
|
||||
? ['components: []']
|
||||
: ['components:', ...components.map((c) => ` - ../../components/${c}`)];
|
||||
return [...parts.before, ...list, ...parts.after].join('\n');
|
||||
}
|
||||
|
||||
function renderStage(parts, components, outDir, name) {
|
||||
const tree = mkdtempSync(join(outDir, `${name}-tree-`));
|
||||
try {
|
||||
cpSync(K8S_DIR, tree, { recursive: true });
|
||||
writeFileSync(join(tree, OVERLAY_FILE), overlayFor(parts, components));
|
||||
const yaml = execFileSync(process.env.KUBECTL ?? 'kubectl', ['kustomize', join(tree, 'overlays', 'encry')], {
|
||||
encoding: 'utf8',
|
||||
});
|
||||
const yamlFile = join(outDir, `${name}.yaml`);
|
||||
writeFileSync(yamlFile, yaml);
|
||||
const json = execFileSync(process.env.YQ ?? 'yq', ['ea', '-o=json', '[.]', yamlFile], { encoding: 'utf8' });
|
||||
return JSON.parse(json).filter(Boolean);
|
||||
} finally {
|
||||
rmSync(tree, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function main() {
|
||||
const outDir = process.argv[2] && resolve(process.argv[2]);
|
||||
if (!outDir) throw new Error('usage: node render-stages.mjs <outDir>');
|
||||
// The copy of deploy/k8s lands in outDir; inside deploy/k8s it would copy itself.
|
||||
const fromK8s = relative(K8S_DIR, outDir);
|
||||
if (!fromK8s.startsWith('..') && !isAbsolute(fromK8s)) throw new Error('outDir must be outside deploy/k8s');
|
||||
mkdirSync(outDir, { recursive: true });
|
||||
const parts = splitOverlay(readFileSync(join(K8S_DIR, OVERLAY_FILE), 'utf8'));
|
||||
const stages = Object.fromEntries(
|
||||
Object.entries(STAGES).map(([name, components]) => [name, renderStage(parts, components, outDir, name)]),
|
||||
);
|
||||
writeFileSync(join(outDir, 'stages.json'), JSON.stringify(stages));
|
||||
for (const [name, docs] of Object.entries(stages)) process.stdout.write(`${name}: ${docs.length} documents\n`);
|
||||
}
|
||||
|
||||
main();
|
||||
Reference in new issue
Block a user