Forma3D.Connect — Staging Deployment Guide¶

# On the staging server during initial setup
SEED_ADMIN_EMAIL=admin@forma3d.be \
SEED_ADMIN_PASSWORD=Admin1234! \
npx prisma db seed

# Generate VAPID keys (do this once, reuse the keys)
npx web-push generate-vapid-keys --json

{
  "publicKey": "BGAS6q-nMn1lt8-foIqSoJORZlarc0RbEmxzz2qU4O-...",
  "privateKey": "99Rho7TKzIlIIIkeNgJ-BTvqD9t5bjSHJkKwsiXDxAY"
}

variables:
  - group: forma3d-staging
  # ... other variables

# From your local machine (use the private key from /Users/jeankedotcom/Library/CloudStorage/OneDrive-Personal/DevGem/projects/forma-3d-connect/sec/droplet/)
ssh -i /Users/jeankedotcom/Library/CloudStorage/OneDrive-Personal/DevGem/projects/forma-3d-connect/sec/droplet/azure-devops root@167.172.45.47

# Update packages
apt update && apt upgrade -y

# Install Docker
curl -fsSL https://get.docker.com | sh

# Verify installation
docker --version
# Expected: Docker version 24.x or later

# Verify Docker Compose plugin
docker compose version
# Expected: Docker Compose version v2.x

# Create Docker config directory
mkdir -p ~/.docker

# Copy the docker-config.json content (from /Users/jeankedotcom/Library/CloudStorage/OneDrive-Personal/DevGem/projects/forma-3d-connect/sec/container-registry/docker-config.json)
# Either upload the file via scp or create it manually:
cat > ~/.docker/config.json << 'EOF'
{
  "auths": {
    "registry.digitalocean.com": {
      "auth": "YOUR_BASE64_AUTH_TOKEN"
    }
  }
}
EOF

# Verify login works
docker pull registry.digitalocean.com/forma-3d/forma3d-connect-api:latest || echo "Image not yet available"

# Create deployment directory
mkdir -p /opt/forma3d/certs

# Navigate to directory
cd /opt/forma3d

# From your local machine (project root)
scp -i /Users/jeankedotcom/Library/CloudStorage/OneDrive-Personal/DevGem/projects/forma-3d-connect/sec/droplet/azure-devops \
  deployment/staging/docker-compose.yml \
  deployment/staging/traefik.yml \
  root@167.172.45.47:/opt/forma3d/

scp -i /Users/jeankedotcom/Library/CloudStorage/OneDrive-Personal/DevGem/projects/forma-3d-connect/sec/droplet/azure-devops \
  deployment/staging/deploy.sh \
  root@167.172.45.47:/opt/forma3d/

# Upload the PostgreSQL CA certificate
# (Get this from DigitalOcean Managed Database dashboard)
scp -i /Users/jeankedotcom/Library/CloudStorage/OneDrive-Personal/DevGem/projects/forma-3d-connect/sec/droplet/azure-devops \
  /path/to/do-postgresql-ca-certificate-2.crt \
  root@167.172.45.47:/opt/forma3d/certs/

# On the droplet
cd /opt/forma3d

# Create docker-compose.yml (copy content from deployment/staging/docker-compose.yml)
nano docker-compose.yml

# Create traefik.yml (copy content from deployment/staging/traefik.yml)
nano traefik.yml

# Create deploy.sh (copy content from deployment/staging/deploy.sh)
nano deploy.sh
chmod +x deploy.sh

# =============================================================================
# Forma3D.Connect Staging Environment
# Auto-generated by Azure DevOps Pipeline
# =============================================================================

# Docker Registry
REGISTRY_URL=registry.digitalocean.com/forma3d
GATEWAY_IMAGE_TAG=20260215143709
ORDER_SERVICE_IMAGE_TAG=20260215143709
PRINT_SERVICE_IMAGE_TAG=20260215143709
SHIPPING_SERVICE_IMAGE_TAG=20260215143709
GRIDFLOCK_SERVICE_IMAGE_TAG=20260215143709
SLICER_IMAGE_TAG=20260215143709
WEB_IMAGE_TAG=20260215143709

# Redis (BullMQ + Sessions + Socket.IO)
REDIS_URL=redis://redis:6379

# Application URLs
API_URL=https://staging-connect-api.forma3d.be
WEB_URL=https://staging-connect.forma3d.be

# Database
DATABASE_URL=postgresql://...

# Shopify Integration
SHOPIFY_SHOP_DOMAIN=your-shop.myshopify.com
SHOPIFY_API_KEY=...
SHOPIFY_API_SECRET=...
SHOPIFY_ACCESS_TOKEN=...
SHOPIFY_WEBHOOK_SECRET=...  # Must equal SHOPIFY_API_SECRET (API-registered webhooks use the app client secret for HMAC)
SHOPIFY_API_VERSION=2026-01

# SimplyPrint Integration (Phase 2)
SIMPLYPRINT_API_URL=https://api.simplyprint.io
SIMPLYPRINT_API_KEY=your-api-key
SIMPLYPRINT_COMPANY_ID=S123456
SIMPLYPRINT_WEBHOOK_SECRET=your-webhook-secret
SIMPLYPRINT_POLLING_ENABLED=false
SIMPLYPRINT_RECONCILIATION_ENABLED=true

# Observability
SENTRY_DSN=https://...
SENTRY_TRACES_SAMPLE_RATE=1.0
SENTRY_PROFILES_SAMPLE_RATE=1.0
# Optional:
# SENTRY_DEBUG=true
# SENTRY_ENVIRONMENT=staging

# API Security
INTERNAL_API_KEY=your-api-key-here

# Storefront Origins (comma-separated, '.domain' = suffix match)
STOREFRONT_ALLOWED_ORIGINS=.myshopify.com

# Shipping Integration (SendCloud) - Phase 5
SHIPPING_ENABLED=true
SENDCLOUD_PUBLIC_KEY=your-public-key
SENDCLOUD_SECRET_KEY=your-secret-key
SENDCLOUD_WEBHOOK_SECRET=your-webhook-signature-key
SENDCLOUD_RECONCILIATION_ENABLED=true
DEFAULT_SHIPPING_METHOD_ID=8
DEFAULT_SENDER_ADDRESS_ID=your-sender-address-id

# Shopify Backfill (Phase 5n - Resilience)
SHOPIFY_BACKFILL_ENABLED=true
SHOPIFY_BACKFILL_BATCH_SIZE=50

# Rate Limiting (optional - defaults shown)
# RATE_LIMIT_TTL_SECONDS=60
# RATE_LIMIT_DEFAULT=100
# RATE_LIMIT_WEBHOOK=50

# Database Pool (optional - defaults shown)
# DATABASE_POOL_SIZE=3
# DATABASE_CONNECT_TIMEOUT_SECONDS=5
# DATABASE_POOL_TIMEOUT_SECONDS=10

# Push Notifications (Phase 7 - PWA)
VAPID_PUBLIC_KEY=BGAS6q-nMn1lt8-foIqSoJORZlarc0RbEmxzz2qU4O-...
VAPID_PRIVATE_KEY=99Rho7TKzIlIIIkeNgJ-BTvqD9t5bjSHJkKwsiXDxAY
VAPID_SUBJECT=mailto:jan.wielemans@devgem.be

# On the droplet - only for manual testing
cd /opt/forma3d
touch .env
chmod 600 .env
# The pipeline will overwrite this on first deployment

# On the droplet
ls -la /opt/forma3d/

/opt/forma3d/
├── .env                                # Environment variables (600 permissions)
├── certs/
│   └── do-postgresql-ca-certificate-2.crt  # PostgreSQL CA cert
├── deploy.sh                           # Deployment script (executable)
├── docker-compose.yml                  # Docker Compose stack
├── traefik.yml                         # Traefik configuration
├── otel-collector-config.yaml          # OpenTelemetry Collector config
├── clickhouse-config.xml               # ClickHouse server config (S3 backups)
├── clickhouse-users.xml                # ClickHouse user config
├── grafana/
│   └── provisioning/
│       └── datasources/
│           └── clickhouse.yaml         # Grafana ClickHouse datasource
└── scripts/
    └── backup-clickhouse-logs.sh       # Daily ClickHouse backup script

# On the droplet
cat >> ~/.ssh/authorized_keys << 'EOF'
ssh-rsa AAAAB3... (content of azure-devops.pub)
EOF

# Set correct permissions
chmod 700 ~/.ssh
chmod 600 ~/.ssh/authorized_keys

# On the droplet
ufw allow 22/tcp   # SSH
ufw allow 80/tcp   # HTTP (for Let's Encrypt challenge)
ufw allow 443/tcp  # HTTPS
ufw enable
ufw status

# On the droplet
cd /opt/forma3d

# Set the image tag (use 'latest' for first deployment)
export IMAGE_TAG=latest

# Pull images (will fail if not yet pushed - that's OK)
docker compose pull

# Start the stack
docker compose up -d

# Check status
docker compose ps

# View logs
docker compose logs -f

# On the droplet - watch containers
watch docker compose ps

# View real-time logs
docker compose logs -f

# Check specific service
docker compose logs api
docker compose logs web
docker compose logs traefik

# On the droplet
docker compose ps

NAME                    IMAGE                                                                  STATUS
forma3d-traefik         traefik:v3.0                                                           Up
forma3d-redis           redis:7-alpine                                                         Up (healthy)
forma3d-gateway         registry.digitalocean.com/forma-3d/forma3d-connect-gateway:xxx         Up (healthy)
forma3d-order-service   registry.digitalocean.com/forma-3d/forma3d-connect-order-service:xxx   Up (healthy)
forma3d-print-service   registry.digitalocean.com/forma-3d/forma3d-connect-print-service:xxx   Up (healthy)
forma3d-shipping-service registry.digitalocean.com/forma-3d/forma3d-connect-shipping-service:xxx Up (healthy)
forma3d-gridflock-service registry.digitalocean.com/forma-3d/forma3d-connect-gridflock-service:xxx Up (healthy)
forma3d-slicer          registry.digitalocean.com/forma-3d/forma3d-connect-slicer:xxx          Up (healthy)
forma3d-web             registry.digitalocean.com/forma-3d/forma3d-connect-web:xxx             Up (healthy)
forma3d-otel-collector  otel/opentelemetry-collector-contrib:latest                             Up
forma3d-clickhouse      clickhouse/clickhouse-server:24-alpine                                  Up (healthy)
forma3d-grafana         grafana/grafana-oss:latest                                              Up
forma3d-uptime-kuma     louislam/uptime-kuma:1                                                 Up
forma3d-dozzle          amir20/dozzle:latest                                                   Up

# From any machine

# API health check (includes database status and build info)
curl https://staging-connect-api.forma3d.be/health
# Expected: {
#   "status": "ok",
#   "database": "connected",
#   "timestamp": "...",
#   "version": "0.0.1",
#   "build": {
#     "number": "20260111143948",
#     "date": "2026-01-11T14:39:48Z",
#     "commit": "abc123..."
#   },
#   "uptime": 3600
# }

# API liveness probe (simple check)
curl https://staging-connect-api.forma3d.be/health/live
# Expected: {"status":"ok"}

# API readiness probe (checks database)
curl https://staging-connect-api.forma3d.be/health/ready
# Expected: {"status":"ok","database":"connected"}

# Web health check (includes build info)
curl https://staging-connect.forma3d.be/health
# Expected: {"status":"ok","build":{"number":"...","date":"...","commit":"..."}}

# Web liveness probe
curl https://staging-connect.forma3d.be/health/live
# Expected: {"status":"ok"}

curl https://staging-connect-api.forma3d.be/api/docs
# Expected: Swagger UI HTML

# Check certificate
echo | openssl s_client -connect staging-connect-api.forma3d.be:443 2>/dev/null | openssl x509 -noout -dates
# Expected: notAfter=... (valid date in the future)

# On the droplet
docker compose exec api npx prisma db execute --stdin <<< "SELECT 1"
# Expected: No errors

pool_size_per_service = (max_connections - reserved - headroom) / num_services

# Check current connection usage
docker run --rm --network forma3d-network postgres:15-alpine \
  psql "$DATABASE_URL" -c "SELECT usename, application_name, state, count(*) FROM pg_stat_activity GROUP BY 1,2,3 ORDER BY 4 DESC;"

# Restart services to reclaim leaked connections
cd /opt/forma3d && docker compose restart gateway order-service print-service shipping-service gridflock-service

# Check logs
docker compose logs api
docker compose logs web

# Common issues:
# - DATABASE_URL incorrect → check .env file
# - Certificate not found → check /opt/forma3d/certs/
# - Port conflict → check `docker ps` for conflicting containers

# ❌ Wrong - cannot access Build outputs from another stage
- stage: DeployStaging
  dependsOn: SomeOtherStage
  condition: eq(dependencies.Build.outputs['DetectAffected.affected.webAffected'], 'true')

# ✅ Correct
- stage: DeployStaging
  dependsOn: Build
  condition: and(succeeded('Build'), eq(dependencies.Build.outputs['DetectAffected.affected.webAffected'], 'true'))

# Check API health endpoint directly
docker compose exec api wget -qO- http://localhost:3000/health
# Returns full health info with database status and build info

# Check API liveness (simpler check)
docker compose exec api wget -qO- http://localhost:3000/health/live

# Check Web health endpoint
docker compose exec web wget -qO- http://localhost:80/health
# Returns build info

# Check Web liveness
docker compose exec web wget -qO- http://localhost:80/health/live

# Common issues:
# - Database not connected (check health.database field)
# - Missing environment variables
# - Application startup error (check logs)

# Check disk usage
df -h /

# Clean up Docker resources
docker system prune -af --volumes

# If images are in use, remove old dangling images
docker image prune -f
docker images -q -f 'dangling=true' | xargs -r docker rmi -f

# Restart containers after cleanup
docker compose restart

# Create or update /etc/docker/daemon.json
cat > /etc/docker/daemon.json << 'EOF'
{
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "10m",
    "max-file": "3"
  }
}
EOF

# Restart Docker daemon
systemctl restart docker

# Recreate containers to apply new log settings
cd /opt/forma3d
docker compose up -d --force-recreate

# Verify log rotation is applied
docker inspect forma3d-api --format='{{.HostConfig.LogConfig.Config}}'
# Expected: map[max-file:3 max-size:10m]

crontab -l | grep backup-clickhouse
# Expected: 0 3 * * * /opt/forma3d/scripts/backup-clickhouse-logs.sh >> /var/log/clickhouse-backup.log 2>&1

# Recent backup log entries
tail -20 /var/log/clickhouse-backup.log

# ClickHouse internal backup log
docker exec forma3d-clickhouse clickhouse-client --query "SELECT * FROM system.backup_log ORDER BY event_time DESC LIMIT 10"

# Data per day (to verify exports cover all days)
docker exec forma3d-clickhouse clickhouse-client --query "SELECT toDate(Timestamp) as day, count() FROM otel.otel_logs GROUP BY day ORDER BY day"

# On the server — export a specific day
source /opt/forma3d/.env
DATE="2026-03-01"           # the day to export
NEXT_DAY="2026-03-02"       # day after
docker exec forma3d-clickhouse clickhouse-client --query "
  INSERT INTO FUNCTION s3(
    'https://${DO_SPACES_BUCKET}.${DO_SPACES_REGION}.digitaloceanspaces.com/${DO_SPACES_LOG_PREFIX}/exports/${DATE}.jsonl.gz',
    '${DO_SPACES_KEY}', '${DO_SPACES_SECRET}',
    'JSONEachRow',
    'Timestamp DateTime64(9), SeverityText String, ServiceName String, Body String, TraceId String, SpanId String, LogAttributes Map(String,String), ResourceAttributes Map(String,String)',
    'gzip')
  SELECT Timestamp, SeverityText, ServiceName, Body, TraceId, SpanId, LogAttributes, ResourceAttributes
  FROM otel.otel_logs
  WHERE Timestamp >= toDateTime64('${DATE} 00:00:00', 9)
    AND Timestamp < toDateTime64('${NEXT_DAY} 00:00:00', 9)
"

# Restore a full backup
docker exec forma3d-clickhouse clickhouse-client --query \
  "RESTORE TABLE otel.otel_logs FROM Disk('s3_backups', 'full/full_2026-03-09/')"

# All services
docker compose logs -f

# Specific service (last 100 lines)
docker compose logs --tail=100 api

# Since timestamp
docker compose logs --since="2024-01-10T10:00:00" api

# Restart all
docker compose restart

# Restart specific service
docker compose restart api

# Pull latest images (all)
docker compose pull

# Or pull specific service
docker compose pull api
docker compose pull web

# Recreate containers with new images
docker compose up -d

# Or restart specific service
docker compose up -d --no-deps api
docker compose up -d --no-deps web

# Clean up old images
docker image prune -f

# For API-only deployment
docker compose pull api
docker compose run --rm api npx prisma migrate deploy
docker compose up -d --no-deps api

# For Web-only deployment
docker compose pull web
docker compose up -d --no-deps web

# For breaking migrations (stop first)
docker compose stop api
docker compose pull api
docker compose run --rm api npx prisma migrate deploy
docker compose up -d --no-deps api

# List available image tags
docker images | grep forma3d

# Update .env or export to use specific tag
export IMAGE_TAG=20240110120000

# Redeploy
docker compose up -d

# Check current running image
docker inspect forma3d-api --format='{{.Config.Labels}}'

# Check image build info
docker inspect registry.digitalocean.com/forma-3d/forma3d-connect-api:latest --format='{{json .Config.Labels}}' | jq

# Run migrations manually
docker compose run --rm api npx prisma migrate deploy

# Check migration status
docker compose run --rm api npx prisma migrate status

# Open Prisma Studio (requires port forwarding)
docker compose exec api npx prisma studio

# Start
cd /opt/forma3d/deployment/staging
docker compose -f docker-compose.pgadmin.yml up -d

# Stop
docker compose -f docker-compose.pgadmin.yml down

# Status
docker compose -f docker-compose.pgadmin.yml ps

postgresql://USERNAME:PASSWORD@HOST:PORT/DATABASE?sslmode=require&sslrootcert=...

postgresql://doadmin:<PASSWORD>@db-postgresql-ams3-forma3d-staging-do-user-1196005-0.m.db.ondigitalocean.com:25060/defaultdb?sslmode=require

# Reset pgAdmin (removes all saved connections!)
docker compose -f docker-compose.pgadmin.yml down
docker volume rm pgadmin-data
docker compose -f docker-compose.pgadmin.yml up -d

# SSH to droplet
ssh -i /Users/jeankedotcom/Library/CloudStorage/OneDrive-Personal/DevGem/projects/forma-3d-connect/sec/droplet/azure-devops root@167.172.45.47

# View running containers
docker compose ps

# View logs
docker compose logs -f

# Redeploy
cd /opt/forma3d && ./deploy.sh <image_tag>

# Manual deployment
docker compose pull && docker compose up -d