ApiCore/ApiCore-Backend/test/system.e2e-spec.ts

import { INestApplication } from '@nestjs/common';
import * as bcrypt from 'bcryptjs';
import request from 'supertest';
import { App } from 'supertest/types';
import { DomainEventDispatcher } from '../src/domain-events/domain-event-dispatcher';
import { JOB_NAMES } from '../src/jobs/job-names';
import { PrismaService } from '../src/prisma/prisma.service';
import { createTestApp } from './utils/create-test-app';
import { login } from './utils/api';

interface FailedEventBody {
  id: string;
  failedHandlers: unknown[];
  [key: string]: unknown;
}

interface FailedDeliveryBody {
  id: string;
  targetHost: string;
  [key: string]: unknown;
}

interface DeliveriesBody {
  telegram: {
    botNameSet: boolean;
    sender: { failed24h: number };
  };
  deliveries: { id: string; [key: string]: unknown }[];
}

interface OverviewBody {
  jobs: unknown[];
  domainEvents: { failed: number; oldestPendingAgeSeconds: number | null };
  webhookDeliveries: { failed: number };
}

describe('System API (e2e)', () => {
  let app: INestApplication<App>;
  let prisma: PrismaService;
  const suffix = `${Date.now()}-${Math.round(Math.random() * 1e6)}`;
  const password = 'SuperSecret123!';
  let adminToken: string;
  let staffToken: string;
  const server = () => app.getHttpServer();
  const auth = (token: string) => ({ Authorization: `Bearer ${token}` });

  beforeAll(async () => {
    app = await createTestApp();
    prisma = app.get(PrismaService);
    const passwordHash = await bcrypt.hash(password, 12);
    for (const [key, role] of [
      ['admin', 'ADMIN'],
      ['staff', 'STAFF'],
    ] as const) {
      await prisma.user.create({
        data: {
          email: `system-${key}-${suffix}@example.com`,
          passwordHash,
          name: key,
          role: { connect: { key: role } },
        },
      });
    }
    adminToken = (
      await login(app, `system-admin-${suffix}@example.com`, password)
    ).accessToken;
    staffToken = (
      await login(app, `system-staff-${suffix}@example.com`, password)
    ).accessToken;
  });

  afterAll(async () => {
    await app.close();
  });

  it('staff without system:read gets 403; API keys never reach /system', async () => {
    await request(server())
      .get('/api/v1/system/overview')
      .set(auth(staffToken))
      .expect(403);
    await request(server())
      .post('/api/v1/system/domain-events/x/retry')
      .set(auth(staffToken))
      .expect(403);

    const created = await request(server())
      .post('/api/v1/api-keys')
      .set(auth(adminToken))
      .send({ name: `System ${suffix}`, scopes: ['shop-sync:read'] })
      .expect(201);
    const { token } = created.body as { token: string };
    await request(server())
      .get('/api/v1/system/overview')
      .set('Authorization', `Bearer ${token}`)
      .expect(403);
    await request(server())
      .get('/api/v1/system/job-runs')
      .set('Authorization', `Bearer ${token}`)
      .expect(403);
  });

  it('shows the overview and validates the list limit', async () => {
    const response = await request(server())
      .get('/api/v1/system/overview')
      .set(auth(adminToken))
      .expect(200);
    const body = response.body as OverviewBody;
    expect(body.jobs).toHaveLength(JOB_NAMES.length);
    expect(typeof body.domainEvents.failed).toBe('number');

    await request(server())
      .get('/api/v1/system/job-runs?limit=0')
      .set(auth(adminToken))
      .expect(400);
    await request(server())
      .get('/api/v1/system/job-runs?limit=101')
      .set(auth(adminToken))
      .expect(400);
    await request(server())
      .get('/api/v1/system/job-runs?job=nope')
      .set(auth(adminToken))
      .expect(400);
    await request(server())
      .get('/api/v1/system/job-runs?job=cleanup&limit=5')
      .set(auth(adminToken))
      .expect(200);
  });

  it('admin retries a FAILED event and the next drain completes it', async () => {
    const correlationId = `system-${suffix}`;
    const event = await prisma.domainEvent.create({
      data: {
        type: 'test.system-retry',
        aggregateType: 'SystemTest',
        aggregateId: `agg-${suffix}`,
        payload: { secret: 'must-not-leak' },
        correlationId,
        status: 'FAILED',
        attempts: 8,
        lastError: 'boom',
        availableAt: new Date(Date.now() + 86_400_000),
      },
    });

    const failed = await request(server())
      .get('/api/v1/system/domain-events/failed?limit=100')
      .set(auth(adminToken))
      .expect(200);
    const listed = (failed.body as FailedEventBody[]).find(
      (e) => e.id === event.id,
    );
    expect(listed).toBeDefined();
    expect(JSON.stringify(failed.body)).not.toContain('must-not-leak');
    expect(listed).not.toHaveProperty('payload');

    await request(server())
      .post(`/api/v1/system/domain-events/${event.id}/retry`)
      .set(auth(adminToken))
      .expect(204);

    const reset = await prisma.domainEvent.findUniqueOrThrow({
      where: { id: event.id },
    });
    expect(reset).toMatchObject({
      status: 'PENDING',
      attempts: 0,
      lastError: null,
    });
    expect(reset.availableAt.getTime()).toBeLessThanOrEqual(Date.now() + 5_000);
    await expect(
      prisma.auditLog.count({
        where: { entityType: 'DomainEvent', entityId: event.id },
      }),
    ).resolves.toBe(1);

    await app
      .get(DomainEventDispatcher)
      .drain({ correlationIds: [correlationId] });
    const done = await prisma.domainEvent.findUniqueOrThrow({
      where: { id: event.id },
    });
    expect(done.status).toBe('DONE');

    // DONE is not FAILED any more: a second retry is a conflict.
    await request(server())
      .post(`/api/v1/system/domain-events/${event.id}/retry`)
      .set(auth(adminToken))
      .expect(409);
    await request(server())
      .post('/api/v1/system/domain-events/does-not-exist/retry')
      .set(auth(adminToken))
      .expect(404);
  });

  it('lists a failed webhook delivery with host only and retries it', async () => {
    const subscription = await prisma.webhookSubscription.create({
      data: {
        url: 'https://hooks.example.com/very/secret/path?token=abc',
        events: ['harvest.created'],
        secret: 'enc-secret',
      },
    });
    const delivery = await prisma.webhookDelivery.create({
      data: {
        subscriptionId: subscription.id,
        event: 'harvest.created',
        body: '{"private":"record"}',
        status: 'FAILED',
        attempts: 8,
        lastStatusCode: 500,
        lastError: 'HTTP 500',
      },
    });

    const response = await request(server())
      .get('/api/v1/system/webhook-deliveries/failed?limit=100')
      .set(auth(adminToken))
      .expect(200);
    const listed = (response.body as FailedDeliveryBody[]).find(
      (d) => d.id === delivery.id,
    );
    expect(listed?.targetHost).toBe('hooks.example.com');
    const raw = JSON.stringify(listed);
    expect(raw).not.toMatch(/secret|token|private/);

    await request(server())
      .post(`/api/v1/system/webhook-deliveries/${delivery.id}/retry`)
      .set(auth(adminToken))
      .expect(204);
    const reset = await prisma.webhookDelivery.findUniqueOrThrow({
      where: { id: delivery.id },
    });
    expect(reset).toMatchObject({
      status: 'PENDING',
      attempts: 0,
      lastError: null,
      lastStatusCode: null,
    });
    await request(server())
      .post(`/api/v1/system/webhook-deliveries/${delivery.id}/retry`)
      .set(auth(adminToken))
      .expect(409);

    await prisma.webhookSubscription.delete({ where: { id: subscription.id } });
  });

  describe('notification deliveries (Telegram card)', () => {
    const CHAT_ID = 987654321n;
    let userId: string;
    const createdNotificationIds: string[] = [];

    const templateDeliveryIds: string[] = [];

    /** A bot reply: addressed by chat id, no user, no notification. */
    async function createTemplateDelivery(): Promise<string> {
      const delivery = await prisma.notificationDelivery.create({
        data: {
          channel: 'TELEGRAM',
          chatId: CHAT_ID,
          templateCode: 'HELP',
          status: 'FAILED',
          attempts: 8,
          lastError: 'HTTP 502',
        },
      });
      templateDeliveryIds.push(delivery.id);
      return delivery.id;
    }

    async function createDelivery(
      status: 'FAILED' | 'DELIVERED',
    ): Promise<string> {
      const notification = await prisma.notification.create({
        data: {
          userId,
          priority: 'HINWEIS',
          category: 'TASK_DUE',
          message: 'private reminder text',
          dedupeKey: `system-e2e:${suffix}:${createdNotificationIds.length}:${Date.now()}`,
        },
      });
      createdNotificationIds.push(notification.id);
      const delivery = await prisma.notificationDelivery.create({
        data: {
          channel: 'TELEGRAM',
          userId,
          notificationId: notification.id,
          status,
          attempts: status === 'FAILED' ? 8 : 1,
          lastStatusCode: status === 'FAILED' ? 502 : null,
          lastError: status === 'FAILED' ? 'HTTP 502' : null,
        },
      });
      return delivery.id;
    }

    beforeAll(async () => {
      const admin = await prisma.user.findUniqueOrThrow({
        where: { email: `system-admin-${suffix}@example.com` },
        select: { id: true },
      });
      userId = admin.id;
    });

    afterAll(async () => {
      // Only this block's own notifications (deliveries cascade).
      await prisma.notification.deleteMany({
        where: { id: { in: createdNotificationIds } },
      });
      await prisma.notificationDelivery.deleteMany({
        where: { id: { in: templateDeliveryIds } },
      });
    });

    it('GET /system/notification-deliveries shows FAILED entries and retry moves them back to PENDING', async () => {
      const failedId = await createDelivery('FAILED');
      const deliveredId = await createDelivery('DELIVERED');
      const templateId = await createTemplateDelivery();

      const response = await request(server())
        .get('/api/v1/system/notification-deliveries?status=FAILED')
        .set(auth(adminToken))
        .expect(200);
      const body = response.body as DeliveriesBody;
      const listed = body.deliveries.find((d) => d.id === failedId);
      expect(listed).toMatchObject({
        status: 'FAILED',
        category: 'TASK_DUE',
        attempts: 8,
        lastStatusCode: 502,
      });
      expect(body.deliveries.some((d) => d.id === deliveredId)).toBe(false);
      expect(body.deliveries.find((d) => d.id === templateId)).toMatchObject({
        category: null,
        templateCode: 'HELP',
      });
      expect(typeof body.telegram.botNameSet).toBe('boolean');
      expect(body.telegram.sender.failed24h).toBeGreaterThanOrEqual(1);
      const raw = JSON.stringify(body);
      expect(raw).not.toContain(CHAT_ID.toString());
      expect(raw).not.toContain('private reminder text');
      expect(listed).not.toHaveProperty('chatId');

      await request(server())
        .get('/api/v1/system/notification-deliveries?status=NOPE')
        .set(auth(adminToken))
        .expect(400);

      await request(server())
        .post(`/api/v1/system/notification-deliveries/${failedId}/retry`)
        .set(auth(adminToken))
        .expect(204);
      const reset = await prisma.notificationDelivery.findUniqueOrThrow({
        where: { id: failedId },
      });
      expect(reset).toMatchObject({
        status: 'PENDING',
        attempts: 0,
        lastError: null,
        lastStatusCode: null,
        claimedAt: null,
      });
      expect(reset.availableAt.getTime()).toBeLessThanOrEqual(
        Date.now() + 5_000,
      );
      await expect(
        prisma.auditLog.count({
          where: { entityType: 'NotificationDelivery', entityId: failedId },
        }),
      ).resolves.toBe(1);

      // PENDING is not FAILED any more, a DELIVERED one never was.
      await request(server())
        .post(`/api/v1/system/notification-deliveries/${failedId}/retry`)
        .set(auth(adminToken))
        .expect(409);
      await request(server())
        .post(`/api/v1/system/notification-deliveries/${deliveredId}/retry`)
        .set(auth(adminToken))
        .expect(409);
      await request(server())
        .post('/api/v1/system/notification-deliveries/does-not-exist/retry')
        .set(auth(adminToken))
        .expect(404);
    });

    it('retry-failed requeues the FAILED entries and reports how many', async () => {
      const own = [
        await createDelivery('FAILED'),
        await createDelivery('FAILED'),
      ];
      // Other files share this DB: remember their FAILED rows to restore them.
      const others = await prisma.notificationDelivery.findMany({
        where: { status: 'FAILED', id: { notIn: own } },
      });
      const ownAudits = () =>
        prisma.auditLog.count({
          where: { entityType: 'NotificationDelivery', entityId: { in: own } },
        });
      try {
        const response = await request(server())
          .post('/api/v1/system/notification-deliveries/retry-failed')
          .set(auth(adminToken))
          .expect(200);
        expect(
          (response.body as { retried: number }).retried,
        ).toBeGreaterThanOrEqual(2);

        const rows = await prisma.notificationDelivery.findMany({
          where: { id: { in: own } },
          select: { status: true, attempts: true },
        });
        expect(rows).toEqual([
          { status: 'PENDING', attempts: 0 },
          { status: 'PENDING', attempts: 0 },
        ]);
        await expect(ownAudits()).resolves.toBe(2);

        // A second call leaves the own rows alone: no new audit rows.
        await request(server())
          .post('/api/v1/system/notification-deliveries/retry-failed')
          .set(auth(adminToken))
          .expect(200);
        await expect(
          prisma.notificationDelivery.count({
            where: { id: { in: own }, status: 'PENDING' },
          }),
        ).resolves.toBe(2);
        await expect(ownAudits()).resolves.toBe(2);
      } finally {
        for (const row of others) {
          await prisma.notificationDelivery.updateMany({
            where: { id: row.id },
            data: {
              status: row.status,
              attempts: row.attempts,
              availableAt: row.availableAt,
              claimedAt: row.claimedAt,
              lastStatusCode: row.lastStatusCode,
              lastError: row.lastError,
            },
          });
        }
      }
    });

    it('staff without system:manage gets 403', async () => {
      await request(server())
        .get('/api/v1/system/notification-deliveries')
        .set(auth(staffToken))
        .expect(403);
      await request(server())
        .post('/api/v1/system/notification-deliveries/x/retry')
        .set(auth(staffToken))
        .expect(403);
      await request(server())
        .post('/api/v1/system/notification-deliveries/retry-failed')
        .set(auth(staffToken))
        .expect(403);
    });
  });
});