Disaster Recovery Research Report¶

┌─────────────────────────────────────────────────────────────────────────┐
│                          EXTERNAL SERVICES                               │
├──────────────────┬──────────────────┬──────────────────┬────────────────┤
│    Shopify       │   SimplyPrint    │    SendCloud     │     SMTP       │
│  (E-commerce)    │   (Printing)     │   (Shipping)     │   (Email)      │
└────────┬─────────┴────────┬─────────┴────────┬─────────┴────────┬───────┘
         │                  │                  │                  │
         ▼                  ▼                  ▼                  ▼
┌─────────────────────────────────────────────────────────────────────────┐
│                           NestJS API                                     │
│  ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ ┌──────────────────┐│
│  │  Webhooks    │ │   Orders     │ │  Print Jobs  │ │   Fulfillment    ││
│  │  Handlers    │ │   Service    │ │   Service    │ │   Service        ││
│  └──────────────┘ └──────────────┘ └──────────────┘ └──────────────────┘│
│  ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ ┌──────────────────┐│
│  │ Retry Queue  │ │   Health     │ │   Sentry     │ │   Notifications  ││
│  │  Processor   │ │   Checks     │ │ Integration  │ │   (Email/Push)   ││
│  └──────────────┘ └──────────────┘ └──────────────┘ └──────────────────┘│
└─────────────────────────────────────────────────────────────────────────┘
                                    │
                                    ▼
┌─────────────────────────────────────────────────────────────────────────┐
│                          PostgreSQL Database                             │
│  ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌────────────────┐ │
│  │  Orders  │ │PrintJobs │ │ Shipments│ │AuditLogs │ │ RetryQueue     │ │
│  └──────────┘ └──────────┘ └──────────┘ └──────────┘ └────────────────┘ │
└─────────────────────────────────────────────────────────────────────────┘

     │ Catastrophic (5) │  R02  │       │ R01,R07,R08,R10 │       │       │
     │   Major (4)      │       │ R04,R11,R15 │ R03,R05,R09 │       │       │
     │   Moderate (3)   │       │ R06,R12 │ R13,R14 │       │       │
     │   Minor (2)      │       │       │       │       │       │
     │   Minimal (1)    │       │       │       │       │       │
     └──────────────────┴───────┴───────┴───────┴───────┴───────┘
                           Rare   Unlikely  Possible  Likely  Frequent
                            (1)     (2)       (3)      (4)      (5)

# PostgreSQL High Availability Setup
primary:
  - Streaming replication to standby
  - WAL archiving to object storage
  - Automated backups every 6 hours

standby:
  - Hot standby for read queries
  - Automatic promotion via Patroni/pgpool

backup_retention:
  daily: 7 days
  weekly: 4 weeks
  monthly: 12 months

apiVersion: apps/v1
kind: Deployment
spec:
  replicas: 3
  template:
    spec:
      containers:
        - name: api
          resources:
            requests:
              memory: "512Mi"
              cpu: "250m"
            limits:
              memory: "1Gi"
              cpu: "1000m"
          livenessProbe:
            httpGet:
              path: /health/live
              port: 3000
            initialDelaySeconds: 30
            periodSeconds: 10
          readinessProbe:
            httpGet:
              path: /health/ready
              port: 3000
            initialDelaySeconds: 5
            periodSeconds: 5

## Security Incident Response Checklist

### Phase 1: Contain (First 30 minutes)
- [ ] Isolate affected systems (network level)
- [ ] Revoke compromised credentials immediately
- [ ] Enable maintenance mode if needed
- [ ] Preserve logs and evidence (DO NOT delete)

### Phase 2: Assess (1-4 hours)
- [ ] Identify attack vector and timeline
- [ ] Determine scope of data exposure
- [ ] Check for persistence mechanisms (backdoors)
- [ ] Review audit logs for suspicious activity

### Phase 3: Remediate (4-24 hours)
- [ ] Rotate all secrets and API keys
- [ ] Patch vulnerabilities
- [ ] Reset user passwords if needed
- [ ] Review and update access controls

### Phase 4: Recover (24-72 hours)
- [ ] Restore from clean backup if needed
- [ ] Re-enable services gradually
- [ ] Enhanced monitoring period
- [ ] Notify affected parties (GDPR: 72 hours)

# Emergency credential rotation script
#!/bin/bash

# 1. Database password
echo "Rotating database credentials..."
# Update in secret manager, then rolling restart

# 2. Shopify credentials
echo "Rotating Shopify API credentials..."
# Generate new keys in Shopify Partner Dashboard
# Update SHOPIFY_API_KEY, SHOPIFY_API_SECRET

# 3. SimplyPrint API key
echo "Rotating SimplyPrint credentials..."
# Generate new API key in SimplyPrint dashboard
# Update SIMPLYPRINT_API_KEY

# 4. SendCloud credentials
echo "Rotating SendCloud credentials..."
# Generate new API keys in SendCloud dashboard
# Update SENDCLOUD_PUBLIC_KEY, SENDCLOUD_SECRET_KEY

# 5. Internal secrets
echo "Rotating internal secrets..."
# SESSION_SECRET, INTERNAL_API_KEY, SHOPIFY_TOKEN_ENCRYPTION_KEY

# 6. Rolling restart
echo "Performing rolling restart..."
kubectl rollout restart deployment/api

// Add Shopify-specific circuit breaker
@Injectable()
export class ShopifyCircuitBreaker {
  private failures = 0;
  private isOpen = false;
  private lastFailure: Date;

  async execute<T>(operation: () => Promise<T>): Promise<T> {
    if (this.isOpen && this.shouldAttemptReset()) {
      this.isOpen = false;
    }

    if (this.isOpen) {
      throw new ServiceUnavailableException('Shopify circuit breaker is open');
    }

    try {
      const result = await operation();
      this.failures = 0;
      return result;
    } catch (error) {
      this.failures++;
      this.lastFailure = new Date();
      if (this.failures >= 5) {
        this.isOpen = true;
      }
      throw error;
    }
  }
}

{
  "backup": {
    "geoRedundantBackup": "Enabled",
    "backupRetentionDays": 35,
    "earliestRestoreDate": "2026-01-01T00:00:00Z"
  },
  "storage": {
    "storageSizeGB": 100,
    "autoGrow": "Enabled"
  }
}

#!/bin/bash
# backup-database.sh

DATE=$(date +%Y%m%d_%H%M%S)
BACKUP_FILE="forma3d_backup_${DATE}.sql.gz"

# Create backup
pg_dump $DATABASE_URL | gzip > /tmp/$BACKUP_FILE

# Upload to cloud storage
az storage blob upload \
  --container-name backups \
  --file /tmp/$BACKUP_FILE \
  --name "database/$BACKUP_FILE"

# Cleanup local file
rm /tmp/$BACKUP_FILE

# Verify backup
az storage blob exists \
  --container-name backups \
  --name "database/$BACKUP_FILE"

# 1. Identify target recovery time
TARGET_TIME="2026-02-05T14:30:00Z"

# 2. Create new database from backup (Azure)
az postgres server restore \
  --resource-group forma3d-rg \
  --name forma3d-db-recovered \
  --source-server forma3d-db \
  --restore-point-in-time $TARGET_TIME

# 3. Verify data integrity
psql $RECOVERED_DATABASE_URL -c "SELECT COUNT(*) FROM orders;"

# 4. Update connection string and restart
kubectl set env deployment/api DATABASE_URL=$RECOVERED_DATABASE_URL

# 5. Run integrity checks
npm run db:integrity-check

# 1. Download latest backup
az storage blob download \
  --container-name backups \
  --name "database/forma3d_backup_latest.sql.gz" \
  --file /tmp/restore.sql.gz

# 2. Create new database
createdb -h $DB_HOST forma3d_restored

# 3. Restore backup
gunzip -c /tmp/restore.sql.gz | psql -h $DB_HOST forma3d_restored

# 4. Run migrations to ensure schema is current
DATABASE_URL="..." npx prisma migrate deploy

# 5. Switch over
# Update Kubernetes secrets and restart

graph TD
    A[Incident Detected] --> B{Severity Assessment}
    B -->|SEV-1/2| C[Page On-Call]
    B -->|SEV-3/4| D[Create Ticket]
    C --> E[Acknowledge]
    E --> F[Investigate]
    F --> G{Root Cause Found?}
    G -->|No| H[Escalate]
    G -->|Yes| I[Implement Fix]
    I --> J[Verify Resolution]
    J --> K[Update Status Page]
    K --> L[Create Postmortem]
    D --> F
    H --> F

Primary On-Call (5 min)
    ↓
Secondary On-Call (10 min)
    ↓
Engineering Lead (15 min)
    ↓
CTO (30 min for SEV-1)

## Incident Response Checklist

### Initial Response (0-5 min)
- [ ] Acknowledge alert
- [ ] Open incident channel (#incident-YYYYMMDD-HHMM)
- [ ] Initial severity assessment
- [ ] Update status page to "Investigating"

### Investigation (5-30 min)
- [ ] Check health endpoints: /health, /health/dependencies
- [ ] Review Sentry for recent errors
- [ ] Check database connectivity
- [ ] Review recent deployments
- [ ] Check external service status pages

### Mitigation (varies)
- [ ] Document attempted fixes
- [ ] Rollback if deployment-related
- [ ] Fail over to backup systems if needed
- [ ] Update status page with progress

### Resolution
- [ ] Confirm service restored
- [ ] Update status page to "Resolved"
- [ ] Document timeline and actions
- [ ] Schedule postmortem (SEV-1/2)

# Postmortem: [Incident Title]

**Date**: YYYY-MM-DD
**Duration**: X hours Y minutes
**Severity**: SEV-X
**Author**: [Name]
**Status**: [Draft | Final]

## Summary

[2-3 sentence summary of what happened and impact]

## Impact

- **Affected Services**: [List services]
- **Customer Impact**: [Description]
- **Orders Affected**: [Number]
- **Duration of Impact**: [Time]

## Timeline (All times in UTC)

| Time | Event |
|------|-------|
| HH:MM | First alert triggered |
| HH:MM | On-call acknowledged |
| HH:MM | Root cause identified |
| HH:MM | Fix deployed |
| HH:MM | Service fully restored |

## Root Cause

[Detailed explanation of what caused the incident]

## Detection

**How was the incident detected?**
- [ ] Automated alerting
- [ ] Customer report
- [ ] Internal observation
- [ ] External service notification

**Detection gap**: [If applicable, why wasn't this detected sooner?]

## Resolution

[Step-by-step description of how the incident was resolved]

## Lessons Learned

### What Went Well
- [Positive item 1]
- [Positive item 2]

### What Went Wrong
- [Issue 1]
- [Issue 2]

### Where We Got Lucky
- [Lucky circumstance that limited impact]

## Action Items

| Action | Owner | Priority | Due Date | Status |
|--------|-------|----------|----------|--------|
| [Action 1] | [Owner] | P1 | YYYY-MM-DD | [ ] Open |
| [Action 2] | [Owner] | P2 | YYYY-MM-DD | [ ] Open |

## Appendix

### Supporting Data
- [Links to dashboards, logs, etc.]

### Related Incidents
- [Links to previous related incidents]

# docker-compose.statuspage.yml
version: "3.8"

services:
  statping:
    image: adamboutcher/statping-ng:latest
    container_name: statping
    restart: always
    ports:
      - "8080:8080"
    volumes:
      - statping_data:/app
    environment:
      - DB_CONN=postgres
      - DB_HOST=postgres
      - DB_PORT=5432
      - DB_DATABASE=statping
      - DB_USER=statping
      - DB_PASS=${STATPING_DB_PASS}
      - NAME=Forma 3D Connect Status
      - DESCRIPTION=Real-time status of Forma 3D Connect services
    depends_on:
      - postgres

  postgres:
    image: postgres:15-alpine
    container_name: statping-db
    restart: always
    volumes:
      - postgres_data:/var/lib/postgresql/data
    environment:
      - POSTGRES_DB=statping
      - POSTGRES_USER=statping
      - POSTGRES_PASSWORD=${STATPING_DB_PASS}

volumes:
  statping_data:
  postgres_data:

# prometheus.yml
scrape_configs:
  - job_name: 'statping'
    bearer_token: '${STATPING_API_SECRET}'
    static_configs:
      - targets: ['statping:8080']

## Investigating: [Service] Performance Degradation
We are currently investigating reports of slow response times on [service].
Updates will be provided every 30 minutes.

## Identified: [Service] Outage
We have identified the cause of the [service] outage. Our team is working on a fix.
Estimated time to resolution: [X hours]

## Monitoring: [Service] Restored
[Service] has been restored. We are monitoring for stability.
A postmortem will be published within 48 hours.

## Resolved: [Service] Incident
The incident affecting [service] has been fully resolved.
[Link to postmortem when available]

# alerting-rules.yml
groups:
  - name: forma3d-critical
    rules:
      - alert: DatabaseDown
        expr: pg_up == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "PostgreSQL database is down"
          description: "Database has been unreachable for more than 1 minute"

      - alert: HighErrorRate
        expr: sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m])) > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "High HTTP error rate detected"
          description: "Error rate is above 5% for 5 minutes"

      - alert: RetryQueueBacklog
        expr: retry_queue_pending_count > 100
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Retry queue backlog growing"
          description: "More than 100 items pending in retry queue"

      - alert: IntegrationUnhealthy
        expr: health_dependency_status{service=~"shopify|simplyprint|sendcloud"} == 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "External integration unhealthy"
          description: "{{ $labels.service }} has been unhealthy for 5 minutes"

// libs/observability/src/alerting/slack-notifier.ts
import { WebClient } from '@slack/web-api';

@Injectable()
export class SlackNotifier {
  private client: WebClient;

  constructor(@Inject(CONFIG) private config: Config) {
    this.client = new WebClient(config.SLACK_BOT_TOKEN);
  }

  async sendAlert(alert: Alert): Promise<void> {
    const channel = this.getChannel(alert.severity);

    await this.client.chat.postMessage({
      channel,
      blocks: [
        {
          type: 'header',
          text: {
            type: 'plain_text',
            text: `🚨 ${alert.title}`,
          },
        },
        {
          type: 'section',
          fields: [
            { type: 'mrkdwn', text: `*Severity:*\n${alert.severity}` },
            { type: 'mrkdwn', text: `*Service:*\n${alert.service}` },
          ],
        },
        {
          type: 'section',
          text: { type: 'mrkdwn', text: alert.description },
        },
        {
          type: 'actions',
          elements: [
            {
              type: 'button',
              text: { type: 'plain_text', text: 'View in Sentry' },
              url: alert.sentryUrl,
            },
            {
              type: 'button',
              text: { type: 'plain_text', text: 'Runbook' },
              url: `https://docs.forma3d.com/runbooks/${alert.type}`,
            },
          ],
        },
      ],
    });
  }

  private getChannel(severity: string): string {
    switch (severity) {
      case 'critical': return '#incidents';
      case 'warning': return '#alerts';
      default: return '#monitoring';
    }
  }
}

Uptime % = ((Total Minutes - Downtime Minutes) / Total Minutes) × 100

Exclusions:
- Scheduled maintenance (with 48h notice)
- Third-party service outages (Shopify, SimplyPrint, SendCloud)
- Force majeure events
- Customer-caused issues

# SLO Configuration
slos:
  api_availability:
    target: 99.9%
    window: 30d
    indicator: probe_success{job="api-health"}

  api_latency:
    target: 99.0%
    window: 30d
    indicator: histogram_quantile(0.95, http_request_duration_seconds_bucket) < 0.5

  order_processing:
    target: 99.5%
    window: 30d
    indicator: order_processing_time_seconds < 30

99.9% uptime = 0.1% error budget = 43.8 minutes/month

Error Budget Remaining = Target Downtime - Actual Downtime

If error budget exhausted:
- Freeze non-critical deployments
- Focus on reliability improvements
- Increase testing requirements

# Forma 3D Connect Service Level Agreement

## Service Commitment

Forma 3D Connect commits to providing [TIER] level service with the following guarantees:

### Availability
- **Target**: [99.X]% monthly uptime
- **Measurement**: Based on successful responses to health check endpoints
- **Exclusions**: Scheduled maintenance, third-party outages

### Support Response
- **Critical Issues**: Response within [X] hours
- **Major Issues**: Response within [X] hours
- **Minor Issues**: Response within [X] business days

### Remedies

If monthly uptime falls below the target:

| Uptime | Service Credit |
|--------|---------------|
| < 99.X% but ≥ 99.Y% | 10% of monthly fee |
| < 99.Y% but ≥ 99.Z% | 25% of monthly fee |
| < 99.Z% | 50% of monthly fee |

Credits must be requested within 30 days of the incident.

### Exclusions

This SLA does not apply to:
- Scheduled maintenance announced 48+ hours in advance
- Third-party service disruptions (Shopify, SimplyPrint, SendCloud)
- Customer misuse or misconfiguration
- Features labeled as "Beta" or "Preview"
- Free tier accounts

gantt
    title Disaster Recovery Implementation
    dateFormat  YYYY-MM-DD
    section Phase 1: Foundation
    Document incident procedures     :a1, 2026-02-10, 5d
    Implement database backups       :a2, 2026-02-10, 10d
    section Phase 2: Visibility
    Deploy status page               :b1, 2026-02-17, 7d
    Set up Slack alerts              :b2, 2026-02-17, 5d
    Configure monitoring stack       :b3, 2026-02-20, 10d
    section Phase 3: Resilience
    Database replication             :c1, 2026-03-01, 14d
    Circuit breakers                 :c2, 2026-03-01, 7d
    Create runbooks                  :c3, 2026-03-08, 10d
    section Phase 4: Operations
    PagerDuty setup                  :d1, 2026-03-15, 7d
    DR drill                         :d2, 2026-03-22, 3d
    Publish SLAs                     :d3, 2026-03-25, 5d

## Emergency Contacts

| Role | Name | Phone | Email |
|------|------|-------|-------|
| Primary On-Call | [TBD] | [TBD] | [TBD] |
| Secondary On-Call | [TBD] | [TBD] | [TBD] |
| Engineering Lead | [TBD] | [TBD] | [TBD] |

## External Services

| Service | Status Page | Support |
|---------|-------------|---------|
| Shopify | shopifystatus.com | partners@shopify.com |
| SimplyPrint | [TBD] | support@simplyprint.io |
| SendCloud | status.sendcloud.sc | support@sendcloud.sc |

# Health check
curl https://api.forma3d.com/health | jq

# Database connection test
psql $DATABASE_URL -c "SELECT 1"

# View recent errors in Sentry
# (via Sentry dashboard)

# Check retry queue
curl -H "Authorization: Bearer $API_KEY" \
  https://api.forma3d.com/retry-queue/stats

# Force retry queue processing
curl -X POST -H "Authorization: Bearer $API_KEY" \
  https://api.forma3d.com/retry-queue/process

# Kubernetes pod restart
kubectl rollout restart deployment/api -n forma3d

# View recent logs
kubectl logs -l app=api -n forma3d --tail=100 -f

# Database backup
pg_dump $DATABASE_URL | gzip > backup_$(date +%Y%m%d).sql.gz