AI Prompt: Forma3D.Connect — Phase 6: Hardening (Production Readiness)¶

Testing → Monitoring → Documentation → Security → Production Ready ✅

┌──────────────────────────────────────────────────────────────────┐
│                    PHASE 6: HARDENING                            │
├──────────────────────────────────────────────────────────────────┤
│                                                                  │
│  ┌────────────────┐  ┌────────────────┐  ┌────────────────┐     │
│  │   TESTING      │  │   MONITORING   │  │   SECURITY     │     │
│  │                │  │                │  │                │     │
│  │ • Unit 80%+    │  │ • Health checks│  │ • Dep scan     │     │
│  │ • Integration  │  │ • Alerting     │  │ • Rate limits  │     │
│  │ • E2E paths    │  │ • Metrics      │  │ • Headers      │     │
│  │ • Load 500/day │  │ • Runbooks     │  │ • Auth audit   │     │
│  └────────────────┘  └────────────────┘  └────────────────┘     │
│                                                                  │
│  ┌─────────────────────────────────────────────────────────┐    │
│  │                  DOCUMENTATION                          │    │
│  │                                                         │    │
│  │  • README complete       • API docs (Swagger)           │    │
│  │  • Architecture docs     • Troubleshooting guide        │    │
│  │  • Runbook operations    • Environment setup            │    │
│  └─────────────────────────────────────────────────────────┘    │
│                                                                  │
│                          ↓                                       │
│              ┌──────────────────────┐                            │
│              │  PRODUCTION READY ✅  │                            │
│              └──────────────────────┘                            │
└──────────────────────────────────────────────────────────────────┘

apps/api/src/
├── **/__tests__/                      # Unit tests for all modules
│   ├── orders.controller.spec.ts      # Controller tests (if not done in 5h)
│   ├── print-jobs.controller.spec.ts
│   └── ...
│
├── test/
│   ├── integration/
│   │   ├── order-flow.spec.ts         # Order → Print → Fulfill flow
│   │   ├── cancellation-flow.spec.ts  # Cancellation handling
│   │   └── error-recovery.spec.ts     # Error recovery scenarios
│   └── e2e/
│       └── critical-paths.spec.ts     # E2E critical path tests

apps/web/src/
├── **/__tests__/                      # Additional component tests

load-tests/
├── k6/
│   ├── config.js                      # K6 configuration
│   ├── scenarios/
│   │   ├── order-throughput.js        # 500 orders/day simulation
│   │   ├── dashboard-load.js          # Dashboard concurrent users
│   │   └── webhook-burst.js           # Webhook burst handling
│   └── reports/                       # Generated reports

apps/api/src/
├── health/
│   ├── health.module.ts               # UPDATE: Add external service checks
│   ├── health.controller.ts           # UPDATE: Enhanced health endpoints
│   └── indicators/
│       ├── shopify.indicator.ts       # Shopify API health check
│       ├── simplyprint.indicator.ts   # SimplyPrint API health check
│       ├── sendcloud.indicator.ts     # Sendcloud API health check
│       └── database.indicator.ts      # Database health check

deployment/
├── monitoring/
│   ├── alerting-rules.yml             # Alert definitions
│   ├── runbook.md                     # Operations runbook
│   └── dashboard.json                 # Metrics dashboard config

apps/api/src/
├── common/
│   ├── guards/
│   │   └── throttler.guard.ts         # Rate limiting guard
│   ├── middleware/
│   │   └── security-headers.middleware.ts  # Security headers
│   └── filters/
│       └── global-exception.filter.ts # UPDATE: Enhanced error handling

.github/workflows/
└── security-scan.yml                  # Dependency security scan workflow

docs/
├── 04-development/
│   ├── runbook.md                     # NEW: Operations runbook
│   ├── troubleshooting.md             # NEW: Troubleshooting guide
│   └── environment-setup.md           # NEW: Environment setup guide
├── 03-architecture/
│   └── (validate and update all diagrams)
└── README.md                          # UPDATE: Complete project documentation

# Generate coverage report for backend
pnpm nx test api --coverage

# Generate coverage report for frontend
pnpm nx test web --coverage

# Identify files with low coverage
# Target: 80%+ statements, functions, branches, lines

import { Test, TestingModule } from '@nestjs/testing';
import { INestApplication } from '@nestjs/common';
import * as request from 'supertest';
import { AppModule } from '../../src/app.module';
import { PrismaService } from '../../src/database/prisma.service';
import { OrderStatus, PrintJobStatus } from '@prisma/client';

describe('Order Flow Integration (Integration)', () => {
  let app: INestApplication;
  let prisma: PrismaService;

  beforeAll(async () => {
    const moduleRef: TestingModule = await Test.createTestingModule({
      imports: [AppModule],
    }).compile();

    app = moduleRef.createNestApplication();
    prisma = moduleRef.get<PrismaService>(PrismaService);
    await app.init();
  });

  afterAll(async () => {
    await app.close();
  });

  beforeEach(async () => {
    // Clean up test data
    await prisma.eventLog.deleteMany();
    await prisma.printJob.deleteMany();
    await prisma.lineItem.deleteMany();
    await prisma.shipment.deleteMany();
    await prisma.order.deleteMany();
  });

  describe('Complete Order Flow', () => {
    it('should process order from creation to fulfillment', async () => {
      // 1. Create order via webhook simulation
      const webhookPayload = createMockShopifyOrderWebhook();

      const orderResponse = await request(app.getHttpServer())
        .post('/api/v1/webhooks/shopify')
        .set('X-Shopify-Topic', 'orders/create')
        .set('X-Shopify-Hmac-SHA256', calculateHmac(webhookPayload))
        .send(webhookPayload)
        .expect(200);

      expect(orderResponse.body.success).toBe(true);

      // 2. Verify order was created
      const order = await prisma.order.findFirst({
        where: { shopifyOrderId: webhookPayload.id.toString() },
        include: { lineItems: true, printJobs: true },
      });

      expect(order).toBeDefined();
      expect(order!.status).toBe(OrderStatus.PENDING);
      expect(order!.lineItems).toHaveLength(webhookPayload.line_items.length);

      // 3. Simulate print job completion
      for (const printJob of order!.printJobs) {
        await prisma.printJob.update({
          where: { id: printJob.id },
          data: { status: PrintJobStatus.COMPLETED },
        });
      }

      // 4. Trigger orchestration check
      // (In real flow this happens via events)

      // 5. Verify order is ready for fulfillment
      const updatedOrder = await prisma.order.findUnique({
        where: { id: order!.id },
      });

      // Order should be completed when all print jobs are done
      expect(updatedOrder!.status).toBe(OrderStatus.COMPLETED);
    });

    it('should handle order cancellation during printing', async () => {
      // Create order and start printing
      const order = await createTestOrder(prisma);

      // Simulate cancellation webhook
      const cancelWebhook = createMockCancellationWebhook(order.shopifyOrderId);

      await request(app.getHttpServer())
        .post('/api/v1/webhooks/shopify')
        .set('X-Shopify-Topic', 'orders/cancelled')
        .set('X-Shopify-Hmac-SHA256', calculateHmac(cancelWebhook))
        .send(cancelWebhook)
        .expect(200);

      // Verify order and print jobs are cancelled
      const cancelledOrder = await prisma.order.findUnique({
        where: { id: order.id },
        include: { printJobs: true },
      });

      expect(cancelledOrder!.status).toBe(OrderStatus.CANCELLED);
      cancelledOrder!.printJobs.forEach((job) => {
        expect([PrintJobStatus.CANCELLED, PrintJobStatus.COMPLETED]).toContain(job.status);
      });
    });
  });

  describe('Error Recovery', () => {
    it('should retry failed print jobs', async () => {
      // Create order with a print job that will fail
      const order = await createTestOrder(prisma);
      const printJob = order.printJobs[0];

      // Mark as failed
      await prisma.printJob.update({
        where: { id: printJob.id },
        data: { 
          status: PrintJobStatus.FAILED,
          errorMessage: 'Simulated failure',
        },
      });

      // Trigger retry via API
      await request(app.getHttpServer())
        .post(`/api/v1/print-jobs/${printJob.id}/retry`)
        .set('X-API-Key', process.env.API_KEY || 'test-key')
        .expect(200);

      // Verify job is queued again
      const retriedJob = await prisma.printJob.findUnique({
        where: { id: printJob.id },
      });

      expect(retriedJob!.status).toBe(PrintJobStatus.QUEUED);
    });
  });
});

// Helper functions
function createMockShopifyOrderWebhook() {
  return {
    id: Date.now(),
    order_number: 1001,
    email: 'test@example.com',
    total_price: '49.99',
    currency: 'EUR',
    shipping_address: {
      first_name: 'Test',
      last_name: 'Customer',
      address1: '123 Test St',
      city: 'Brussels',
      zip: '1000',
      country_code: 'BE',
    },
    line_items: [
      {
        id: Date.now(),
        variant_id: 12345,
        title: 'Test Product',
        quantity: 1,
        price: '49.99',
        sku: 'TEST-SKU-001',
      },
    ],
  };
}

function createMockCancellationWebhook(shopifyOrderId: string) {
  return {
    id: shopifyOrderId,
    cancelled_at: new Date().toISOString(),
  };
}

function calculateHmac(payload: object): string {
  const crypto = require('crypto');
  const secret = process.env.SHOPIFY_WEBHOOK_SECRET || 'test-secret';
  return crypto
    .createHmac('sha256', secret)
    .update(JSON.stringify(payload))
    .digest('base64');
}

async function createTestOrder(prisma: PrismaService) {
  // Create a test order with line items and print jobs
  return prisma.order.create({
    data: {
      shopifyOrderId: `test-${Date.now()}`,
      shopifyOrderNumber: 'TEST-1001',
      customerName: 'Test Customer',
      customerEmail: 'test@example.com',
      shippingAddress: {
        first_name: 'Test',
        last_name: 'Customer',
        address1: '123 Test St',
        city: 'Brussels',
        zip: '1000',
        country_code: 'BE',
      },
      totalPrice: 49.99,
      currency: 'EUR',
      status: OrderStatus.PROCESSING,
      lineItems: {
        create: [
          {
            shopifyLineItemId: `line-${Date.now()}`,
            shopifyVariantId: '12345',
            title: 'Test Product',
            quantity: 1,
            price: 49.99,
            sku: 'TEST-SKU-001',
          },
        ],
      },
      printJobs: {
        create: [
          {
            simplyPrintJobId: `sp-${Date.now()}`,
            status: PrintJobStatus.PRINTING,
          },
        ],
      },
    },
    include: {
      lineItems: true,
      printJobs: true,
    },
  });
}

import http from 'k6/http';
import { check, sleep } from 'k6';
import { Rate, Trend } from 'k6/metrics';

// Custom metrics
const orderCreationRate = new Rate('order_creation_success');
const orderCreationDuration = new Trend('order_creation_duration');

// Test configuration
export const options = {
  scenarios: {
    // Simulate 500 orders/day = ~21 orders/hour = ~0.35 orders/minute
    // But we want to test burst capacity too
    sustained_load: {
      executor: 'constant-arrival-rate',
      rate: 1, // 1 order per second (3600/hour for stress test)
      timeUnit: '1s',
      duration: '5m',
      preAllocatedVUs: 10,
      maxVUs: 50,
    },
    spike_test: {
      executor: 'ramping-arrival-rate',
      startRate: 0,
      timeUnit: '1s',
      preAllocatedVUs: 50,
      maxVUs: 100,
      stages: [
        { target: 10, duration: '30s' }, // Ramp up to 10/s
        { target: 10, duration: '1m' },  // Hold at 10/s
        { target: 50, duration: '30s' }, // Spike to 50/s
        { target: 50, duration: '30s' }, // Hold spike
        { target: 0, duration: '30s' },  // Ramp down
      ],
      startTime: '6m', // Start after sustained load
    },
  },
  thresholds: {
    'http_req_duration': ['p(95)<2000'], // 95th percentile < 2s
    'http_req_failed': ['rate<0.01'],    // Error rate < 1%
    'order_creation_success': ['rate>0.99'], // 99% success rate
  },
};

const BASE_URL = __ENV.API_URL || 'http://localhost:3000';
const API_KEY = __ENV.API_KEY || 'test-api-key';

// Webhook simulation (read-only for load test)
export default function () {
  // Test 1: Health check
  const healthRes = http.get(`${BASE_URL}/health`);
  check(healthRes, {
    'health check status is 200': (r) => r.status === 200,
  });

  // Test 2: Orders list (dashboard simulation)
  const ordersRes = http.get(`${BASE_URL}/api/v1/orders?page=1&pageSize=20`, {
    headers: { 'X-API-Key': API_KEY },
  });
  check(ordersRes, {
    'orders list status is 200': (r) => r.status === 200,
    'orders list has data': (r) => {
      const body = JSON.parse(r.body);
      return Array.isArray(body.orders);
    },
  });

  // Test 3: Single order detail
  const orderId = getRandomOrderId();
  if (orderId) {
    const orderDetailRes = http.get(`${BASE_URL}/api/v1/orders/${orderId}`, {
      headers: { 'X-API-Key': API_KEY },
    });
    check(orderDetailRes, {
      'order detail status is 200 or 404': (r) => [200, 404].includes(r.status),
    });
  }

  // Test 4: Shipping methods (unauthenticated)
  const shippingRes = http.get(`${BASE_URL}/api/v1/shipping/methods?country=BE`);
  check(shippingRes, {
    'shipping methods status is 200': (r) => r.status === 200,
  });

  sleep(0.1); // 100ms between iterations
}

function getRandomOrderId() {
  // In real test, fetch from a pool of known order IDs
  // For now, return null to skip order detail test
  return null;
}

export const environments = {
  local: {
    baseUrl: 'http://localhost:3000',
    apiKey: 'dev-api-key',
  },
  staging: {
    baseUrl: 'https://staging-connect-api.forma3d.be',
    apiKey: '__STAGING_API_KEY__',
  },
};

{
  "scripts": {
    "load-test:local": "k6 run --env API_URL=http://localhost:3000 load-tests/k6/scenarios/order-throughput.js",
    "load-test:staging": "k6 run --env API_URL=https://staging-connect-api.forma3d.be load-tests/k6/scenarios/order-throughput.js"
  }
}

/**
 * E2E Critical Path Tests
 * 
 * These tests verify the complete automation flow works end-to-end
 * against a real database (test environment).
 */
import { Test, TestingModule } from '@nestjs/testing';
import { INestApplication } from '@nestjs/common';
import { AppModule } from '../../src/app.module';

describe('Critical Paths E2E', () => {
  let app: INestApplication;

  beforeAll(async () => {
    const moduleRef: TestingModule = await Test.createTestingModule({
      imports: [AppModule],
    }).compile();

    app = moduleRef.createNestApplication();
    await app.init();
  });

  afterAll(async () => {
    await app.close();
  });

  describe('Order → Print Job → Fulfillment Path', () => {
    it('should complete full automation cycle', async () => {
      // This is a placeholder for a full E2E test
      // In a real scenario, this would:
      // 1. Create a Shopify order via webhook
      // 2. Verify print jobs are created
      // 3. Simulate print completion
      // 4. Verify shipping label is generated
      // 5. Verify Shopify fulfillment is created
      expect(true).toBe(true);
    });
  });

  describe('Cancellation Path', () => {
    it('should handle cancellation at any stage', async () => {
      expect(true).toBe(true);
    });
  });

  describe('Error Recovery Path', () => {
    it('should recover from transient failures', async () => {
      expect(true).toBe(true);
    });
  });
});

// Start tracking a new order flow
this.businessObservability.startFlow(orderId);

// Set Sentry context
this.businessObservability.setOrderContext({
  id: order.id,
  shopifyOrderId: order.shopifyOrderId,
  shopifyOrderNumber: order.shopifyOrderNumber,
  status: order.status,
});

// Log state transition
await this.businessObservability.logStateTransition({
  entityType: 'order',
  entityId: orderId,
  orderId,
  previousState: OrderStatus.PENDING,
  newState: OrderStatus.PROCESSING,
  trigger: 'webhook_received',
});

// Record milestone
await this.businessObservability.recordMilestone({
  orderId,
  milestone: 'print_jobs_created',
  metadata: { jobCount: 3 },
});

// Complete flow (called automatically when flow_completed milestone is recorded)
await this.businessObservability.recordMilestone({
  orderId,
  milestone: 'flow_completed',
});

{
  "message": "[STATE CHANGE] order:order-123 PENDING → PROCESSING",
  "correlationId": "abc-123",
  "entityType": "order",
  "entityId": "order-123",
  "previousState": "PENDING",
  "newState": "PROCESSING",
  "trigger": "webhook_received"
}

{
  "message": "[FLOW COMPLETED] Order order-123 automation completed successfully",
  "correlationId": "abc-123",
  "orderId": "order-123",
  "success": true,
  "totalDurationMs": 125000,
  "totalDurationMinutes": 2.1,
  "milestones": {
    "order_received": 0,
    "order_validated": 50,
    "print_jobs_created": 200,
    "all_jobs_completed": 120000,
    "shipping_label_created": 122000,
    "fulfillment_created": 125000
  }
}

import { Injectable } from '@nestjs/common';
import { HealthIndicator, HealthIndicatorResult, HealthCheckError } from '@nestjs/terminus';
import { ShopifyApiClient } from '../../shopify/shopify-api.client';

@Injectable()
export class ShopifyHealthIndicator extends HealthIndicator {
  constructor(private readonly shopifyClient: ShopifyApiClient) {
    super();
  }

  async isHealthy(key: string): Promise<HealthIndicatorResult> {
    try {
      // Make a lightweight API call to verify connectivity
      const isConnected = await this.shopifyClient.ping();

      if (isConnected) {
        return this.getStatus(key, true);
      }

      throw new HealthCheckError(
        'Shopify API check failed',
        this.getStatus(key, false, { message: 'Unable to connect to Shopify API' })
      );
    } catch (error) {
      throw new HealthCheckError(
        'Shopify API check failed',
        this.getStatus(key, false, { error: error.message })
      );
    }
  }
}

import { Injectable } from '@nestjs/common';
import { HealthIndicator, HealthIndicatorResult, HealthCheckError } from '@nestjs/terminus';
import { SimplyPrintApiClient } from '../../simplyprint/simplyprint-api.client';

@Injectable()
export class SimplyPrintHealthIndicator extends HealthIndicator {
  constructor(private readonly simplyPrintClient: SimplyPrintApiClient) {
    super();
  }

  async isHealthy(key: string): Promise<HealthIndicatorResult> {
    try {
      const isConnected = await this.simplyPrintClient.ping();

      if (isConnected) {
        return this.getStatus(key, true);
      }

      throw new HealthCheckError(
        'SimplyPrint API check failed',
        this.getStatus(key, false)
      );
    } catch (error) {
      throw new HealthCheckError(
        'SimplyPrint API check failed',
        this.getStatus(key, false, { error: error.message })
      );
    }
  }
}

import { Injectable } from '@nestjs/common';
import { HealthIndicator, HealthIndicatorResult, HealthCheckError } from '@nestjs/terminus';
import { SendcloudApiClient } from '../../sendcloud/sendcloud-api.client';

@Injectable()
export class SendcloudHealthIndicator extends HealthIndicator {
  constructor(private readonly sendcloudClient: SendcloudApiClient) {
    super();
  }

  async isHealthy(key: string): Promise<HealthIndicatorResult> {
    if (!this.sendcloudClient.isShippingEnabled()) {
      return this.getStatus(key, true, { status: 'disabled' });
    }

    try {
      // Get shipping methods as a health check
      await this.sendcloudClient.getShippingMethods();
      return this.getStatus(key, true);
    } catch (error) {
      throw new HealthCheckError(
        'Sendcloud API check failed',
        this.getStatus(key, false, { error: error.message })
      );
    }
  }
}

import { Controller, Get } from '@nestjs/common';
import { ApiTags, ApiOperation, ApiResponse } from '@nestjs/swagger';
import {
  HealthCheckService,
  HealthCheck,
  PrismaHealthIndicator,
  HealthCheckResult,
} from '@nestjs/terminus';
import { PrismaService } from '../database/prisma.service';
import { ShopifyHealthIndicator } from './indicators/shopify.indicator';
import { SimplyPrintHealthIndicator } from './indicators/simplyprint.indicator';
import { SendcloudHealthIndicator } from './indicators/sendcloud.indicator';

@ApiTags('Health')
@Controller('health')
export class HealthController {
  constructor(
    private readonly health: HealthCheckService,
    private readonly prismaHealth: PrismaHealthIndicator,
    private readonly prisma: PrismaService,
    private readonly shopifyHealth: ShopifyHealthIndicator,
    private readonly simplyPrintHealth: SimplyPrintHealthIndicator,
    private readonly sendcloudHealth: SendcloudHealthIndicator,
  ) {}

  @Get()
  @HealthCheck()
  @ApiOperation({ summary: 'Full health check with all dependencies' })
  @ApiResponse({ status: 200, description: 'System is healthy' })
  @ApiResponse({ status: 503, description: 'System is unhealthy' })
  async check(): Promise<HealthCheckResult> {
    return this.health.check([
      // Database
      () => this.prismaHealth.pingCheck('database', this.prisma),
      // External services
      () => this.shopifyHealth.isHealthy('shopify'),
      () => this.simplyPrintHealth.isHealthy('simplyprint'),
      () => this.sendcloudHealth.isHealthy('sendcloud'),
    ]);
  }

  @Get('live')
  @ApiOperation({ summary: 'Liveness probe - is the process running?' })
  @ApiResponse({ status: 200, description: 'Process is alive' })
  async liveness(): Promise<{ status: string; timestamp: string }> {
    return {
      status: 'ok',
      timestamp: new Date().toISOString(),
    };
  }

  @Get('ready')
  @HealthCheck()
  @ApiOperation({ summary: 'Readiness probe - is the service ready to accept traffic?' })
  @ApiResponse({ status: 200, description: 'Service is ready' })
  @ApiResponse({ status: 503, description: 'Service is not ready' })
  async readiness(): Promise<HealthCheckResult> {
    return this.health.check([
      () => this.prismaHealth.pingCheck('database', this.prisma),
    ]);
  }

  @Get('dependencies')
  @HealthCheck()
  @ApiOperation({ summary: 'Check all external service dependencies' })
  @ApiResponse({ status: 200, description: 'All dependencies healthy' })
  @ApiResponse({ status: 503, description: 'One or more dependencies unhealthy' })
  async dependencies(): Promise<HealthCheckResult> {
    return this.health.check([
      () => this.shopifyHealth.isHealthy('shopify'),
      () => this.simplyPrintHealth.isHealthy('simplyprint'),
      () => this.sendcloudHealth.isHealthy('sendcloud'),
    ]);
  }
}

# Alerting Rules for Forma3D.Connect
# Configure in your monitoring system (e.g., Prometheus AlertManager, Datadog, etc.)

groups:
  - name: forma3d-connect
    rules:
      # High error rate
      - alert: HighErrorRate
        expr: rate(http_requests_total{status=~"5.."}[5m]) / rate(http_requests_total[5m]) > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "High error rate detected"
          description: "Error rate is above 5% for the last 5 minutes"
          runbook_url: "docs/04-development/runbook.md#high-error-rate"

      # API latency
      - alert: HighAPILatency
        expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) > 2
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High API latency"
          description: "95th percentile latency is above 2 seconds"
          runbook_url: "docs/04-development/runbook.md#high-latency"

      # Database connection issues
      - alert: DatabaseConnectionFailed
        expr: pg_up == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Database connection failed"
          description: "Cannot connect to PostgreSQL database"
          runbook_url: "docs/04-development/runbook.md#database-connection"

      # External service failures
      - alert: ShopifyAPIDown
        expr: shopify_api_up == 0
        for: 5m
        labels:
          severity: high
        annotations:
          summary: "Shopify API unreachable"
          description: "Cannot connect to Shopify API for 5 minutes"
          runbook_url: "docs/04-development/runbook.md#shopify-down"

      - alert: SimplyPrintAPIDown
        expr: simplyprint_api_up == 0
        for: 5m
        labels:
          severity: high
        annotations:
          summary: "SimplyPrint API unreachable"
          description: "Cannot connect to SimplyPrint API for 5 minutes"
          runbook_url: "docs/04-development/runbook.md#simplyprint-down"

      # Order processing stuck
      - alert: OrdersStuckInProcessing
        expr: count(orders_status{status="processing", age_minutes > 60}) > 5
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Orders stuck in processing"
          description: "Multiple orders have been in processing state for over 60 minutes"
          runbook_url: "docs/04-development/runbook.md#stuck-orders"

      # Retry queue growing
      - alert: RetryQueueBacklog
        expr: retry_queue_size > 50
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Retry queue backlog growing"
          description: "More than 50 items in retry queue for over 15 minutes"
          runbook_url: "docs/04-development/runbook.md#retry-queue-backlog"

# Forma3D.Connect Operations Runbook

## Overview

This runbook provides procedures for operating and troubleshooting Forma3D.Connect in production.

## Table of Contents

1. [Service Architecture](#service-architecture)
2. [Health Checks](#health-checks)
3. [Common Issues and Resolutions](#common-issues)
4. [Incident Response](#incident-response)
5. [Maintenance Procedures](#maintenance-procedures)

---

## Service Architecture

### Components

| Component | URL | Purpose |
|-----------|-----|---------|
| API | `https://connect-api.forma3d.be` | Backend NestJS application |
| Web | `https://connect.forma3d.be` | React dashboard |
| Database | PostgreSQL (managed) | Data persistence |
| Traefik | Internal | Reverse proxy with TLS |

### External Dependencies

| Service | Purpose | Documentation |
|---------|---------|---------------|
| Shopify | E-commerce platform | [Shopify API Docs](https://shopify.dev/docs/api) |
| SimplyPrint | 3D print management | [SimplyPrint API](https://simplyprint.io/docs) |
| Sendcloud | Shipping labels | [Sendcloud API](https://api.sendcloud.dev) |
| Sentry | Error monitoring | [Sentry Dashboard](https://sentry.io) |

---

## Health Checks

### Endpoints

```bash
# Full health check
curl https://connect-api.forma3d.be/health

# Liveness probe
curl https://connect-api.forma3d.be/health/live

# Readiness probe
curl https://connect-api.forma3d.be/health/ready

# External dependencies
curl https://connect-api.forma3d.be/health/dependencies

{
  "status": "ok",
  "info": {
    "database": { "status": "up" },
    "shopify": { "status": "up" },
    "simplyprint": { "status": "up" },
    "sendcloud": { "status": "up" }
  }
}

{
  "status": "error",
  "error": {
    "shopify": {
      "status": "down",
      "error": "Connection timeout"
    }
  }
}

## Incident Report

**Date:** YYYY-MM-DD
**Severity:** P1/P2/P3/P4
**Duration:** HH:MM - HH:MM
**Impact:** [Description of user impact]

### Timeline
- HH:MM - Issue detected
- HH:MM - Investigation started
- HH:MM - Root cause identified
- HH:MM - Fix deployed
- HH:MM - Issue resolved

### Root Cause
[Description of what caused the issue]

### Resolution
[What was done to fix it]

### Prevention
[What will be done to prevent recurrence]

# Pull latest changes
git pull origin main

# Build and push images
docker build -t forma3d-api:latest apps/api
docker push registry.digitalocean.com/forma3d/api:latest

# Deploy with zero downtime
docker-compose up -d --no-deps api

# Run migrations
pnpm prisma migrate deploy

# Rollback (if needed)
pnpm prisma migrate resolve --rolled-back MIGRATION_NAME

# Clear old logs
docker system prune --volumes

pg_dump $DATABASE_URL > backup_$(date +%Y%m%d).sql

# traefik.yml - ACME auto-renewal configuration
certificatesResolvers:
  letsencrypt:
    acme:
      email: admin@forma3d.be          # Expiry notifications sent here
      storage: /letsencrypt/acme.json  # Persisted in Docker volume
      httpChallenge:
        entryPoint: web                # Port 80 for challenge

# Check certificate expiry date
echo | openssl s_client -connect staging-connect-api.forma3d.be:443 2>/dev/null | openssl x509 -noout -dates

# Check Traefik logs for renewal activity
docker logs forma3d-traefik 2>&1 | grep -i "acme\|certificate\|renew"

# Verify ACME storage file exists
docker exec forma3d-traefik ls -la /letsencrypt/acme.json

pnpm install

cp .env.example .env

# Database
DATABASE_URL=postgresql://user:password@localhost:5432/forma3d_connect

# Shopify
SHOPIFY_SHOP_DOMAIN=your-shop.myshopify.com
SHOPIFY_API_KEY=your-api-key
SHOPIFY_API_SECRET=your-api-secret
SHOPIFY_ACCESS_TOKEN=your-access-token
SHOPIFY_WEBHOOK_SECRET=your-webhook-secret

# SimplyPrint
SIMPLYPRINT_API_URL=https://api.simplyprint.io/v1
SIMPLYPRINT_API_KEY=your-api-key

# Sendcloud (optional)
SENDCLOUD_PUBLIC_KEY=your-public-key
SENDCLOUD_SECRET_KEY=your-secret-key
SHIPPING_ENABLED=true

# Application
API_KEY=your-admin-api-key
NODE_ENV=development
PORT=3000

docker-compose up -d postgres

pnpm prisma migrate dev

# Start all services
pnpm dev

# Or start individually
pnpm nx serve api    # Backend on http://localhost:3000
pnpm nx serve web    # Frontend on http://localhost:4200

# Check API health
curl http://localhost:3000/health

# Access Swagger docs
open http://localhost:3000/api/docs

# Access dashboard
open http://localhost:4200

docker-compose up -d

# Run all tests
pnpm test

# Run with coverage
pnpm test:coverage

# Run E2E tests
pnpm e2e

PORT=3001 pnpm nx serve api

pnpm prisma generate

pnpm prisma migrate reset

#### 2. Troubleshooting Guide

Create `docs/04-development/troubleshooting.md`:

```markdown
# Troubleshooting Guide

## Common Issues

### Build Errors

#### "Cannot find module '@forma3d/...'"

**Cause:** Library not built or missing from node_modules

**Solution:**
```bash
pnpm install
pnpm nx run-many --target=build --all

pnpm prisma generate  # Regenerate Prisma types
pnpm nx reset         # Clear Nx cache
pnpm install          # Reinstall dependencies

# Development
pnpm nx serve api --verbose

# Docker
docker logs forma3d-api -f --tail 100

# Staging
ssh staging 'docker logs forma3d-api'

DEBUG=forma3d:*
LOG_LEVEL=debug

DEBUG=prisma:query

import { ThrottlerModule } from '@nestjs/throttler';

@Module({
  imports: [
    // ... existing imports
    ThrottlerModule.forRoot([
      {
        name: 'short',
        ttl: 1000,     // 1 second
        limit: 10,      // 10 requests per second
      },
      {
        name: 'medium',
        ttl: 10000,    // 10 seconds
        limit: 50,      // 50 requests per 10 seconds
      },
      {
        name: 'long',
        ttl: 60000,    // 1 minute
        limit: 200,     // 200 requests per minute
      },
    ]),
  ],
  providers: [
    {
      provide: APP_GUARD,
      useClass: CustomThrottlerGuard,
    },
  ],
})
export class AppModule {}

import { Injectable, NestMiddleware } from '@nestjs/common';
import { Request, Response, NextFunction } from 'express';
import helmet from 'helmet';

@Injectable()
export class SecurityHeadersMiddleware implements NestMiddleware {
  private readonly helmet = helmet({
    contentSecurityPolicy: {
      directives: {
        defaultSrc: ["'self'"],
        styleSrc: ["'self'", "'unsafe-inline'"],
        scriptSrc: ["'self'"],
        imgSrc: ["'self'", 'data:', 'https:'],
        connectSrc: ["'self'", 'https://api.sentry.io'],
        frameSrc: ["'none'"],
        objectSrc: ["'none'"],
      },
    },
    crossOriginEmbedderPolicy: false, // Required for Swagger UI
    crossOriginOpenerPolicy: { policy: 'same-origin-allow-popups' },
    crossOriginResourcePolicy: { policy: 'cross-origin' },
    hsts: {
      maxAge: 31536000,
      includeSubDomains: true,
      preload: true,
    },
    noSniff: true,
    referrerPolicy: { policy: 'strict-origin-when-cross-origin' },
    xssFilter: true,
  });

  use(req: Request, res: Response, next: NextFunction) {
    this.helmet(req, res, next);
  }
}

name: Security Scan

on:
  push:
    branches: [main, develop]
  pull_request:
    branches: [main]
  schedule:
    - cron: '0 0 * * 1' # Weekly on Monday

jobs:
  dependency-scan:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'

      - name: Install pnpm
        uses: pnpm/action-setup@v2
        with:
          version: 8

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Run npm audit
        run: pnpm audit --audit-level=high
        continue-on-error: true

      - name: Run Snyk scan
        uses: snyk/actions/node@master
        continue-on-error: true
        env:
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

      - name: Upload Snyk report
        uses: github/codeql-action/upload-sarif@v2
        if: always()
        with:
          sarif_file: snyk.sarif
        continue-on-error: true

  code-scan:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Initialize CodeQL
        uses: github/codeql-action/init@v2
        with:
          languages: javascript, typescript

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v2

# Security Audit Checklist

## Authentication & Authorization

- [ ] API key authentication implemented for admin endpoints
- [ ] API keys are stored hashed in database (if applicable)
- [ ] Rate limiting applied to authentication endpoints
- [ ] Failed authentication attempts are logged

## Webhook Security

- [ ] Shopify webhooks use HMAC verification
- [ ] SimplyPrint webhooks use token verification
- [ ] Sendcloud webhooks verified (if applicable)
- [ ] Webhook idempotency implemented

## Data Protection

- [ ] Sensitive data is not logged (passwords, tokens, etc.)
- [ ] Database credentials are not in code
- [ ] API keys are stored in environment variables
- [ ] HTTPS enforced in production

## Input Validation

- [ ] All DTOs have validation decorators
- [ ] JSON payloads are validated at boundaries
- [ ] SQL injection prevented via Prisma
- [ ] XSS prevented via proper output encoding

## Dependencies

- [ ] No critical vulnerabilities in dependencies
- [ ] Dependencies are up to date
- [ ] Lock file is committed

## Infrastructure

- [ ] TLS certificates valid and auto-renewed
- [ ] Security headers configured
- [ ] CORS properly configured
- [ ] Firewall rules reviewed

# Run all tests with coverage
pnpm test:coverage

# Expected output: > 80% coverage

# Run load tests
pnpm load-test:staging

# Expected output: All thresholds passed

# Check health endpoints
curl https://staging-connect-api.forma3d.be/health
curl https://staging-connect-api.forma3d.be/health/dependencies

# Run security scan
pnpm audit

# Expected output: 0 high/critical vulnerabilities