AI Prompt: Forma3D.Connect — Phase 3: Fulfillment Loop ⏳¶

Order → Print Job → Print Complete → Fulfillment → Customer Notified

model Order {
  id                   String        @id @default(uuid())
  shopifyOrderId       String        @unique
  shopifyOrderNumber   String
  status               OrderStatus   @default(PENDING)
  customerName         String
  customerEmail        String?
  shippingAddress      Json
  totalPrice           Decimal       @db.Decimal(10, 2)
  currency             String        @default("EUR")
  shopifyFulfillmentId String?
  trackingNumber       String?
  trackingUrl          String?
  createdAt            DateTime      @default(now())
  updatedAt            DateTime      @updatedAt
  completedAt          DateTime?
  lineItems            LineItem[]
}

enum OrderStatus {
  PENDING
  PROCESSING
  PARTIALLY_COMPLETED
  COMPLETED
  FAILED
  CANCELLED
}

model PrintJob {
  id                String        @id @default(uuid())
  simplyPrintJobId  String?       @unique
  lineItemId        String        @unique
  lineItem          LineItem      @relation(fields: [lineItemId], references: [id])
  status            PrintJobStatus @default(PENDING)
  printerId         String?
  startedAt         DateTime?
  completedAt       DateTime?
  errorMessage      String?
  retryCount        Int           @default(0)
  createdAt         DateTime      @default(now())
  updatedAt         DateTime      @updatedAt
}

enum PrintJobStatus {
  PENDING
  QUEUED
  ASSIGNED
  PRINTING
  COMPLETED
  FAILED
  CANCELLED
}

                                 ┌─────────────────────────────────┐
                                 │                                 │
                                 ▼                                 │
┌──────────────┐     ┌──────────────────┐     ┌──────────────┐    │
│  All Print   │────▶│   Fulfillment    │────▶│   Shopify    │    │
│ Jobs Done    │     │    Service       │     │ Fulfillment  │    │
└──────────────┘     └──────────────────┘     └──────────────┘    │
                              │                                    │
                              │ (on failure)                       │
                              ▼                                    │
                     ┌──────────────────┐                          │
                     │   Retry Queue    │──────────────────────────┘
                     │  (Exponential    │
                     │   Backoff)       │
                     └──────────────────┘
                              │
                              │ (after max retries)
                              ▼
                     ┌──────────────────┐
                     │   Notification   │
                     │    Service       │
                     └──────────────────┘

┌──────────────────┐     ┌──────────────────┐     ┌──────────────────┐
│  Shopify Order   │────▶│   Cancellation   │────▶│  SimplyPrint     │
│  Cancelled       │     │    Handler       │     │  Cancel Jobs     │
└──────────────────┘     └──────────────────┘     └──────────────────┘
                                  │
                                  │ (if already printing)
                                  ▼
                         ┌──────────────────┐
                         │  Flag for Review │
                         │  + Notify        │
                         └──────────────────┘

apps/api/src/
├── fulfillment/
│   ├── fulfillment.module.ts             # Module definition
│   ├── fulfillment.service.ts            # Fulfillment business logic
│   ├── fulfillment.controller.ts         # REST endpoints (manual triggers)
│   ├── dto/
│   │   ├── fulfillment.dto.ts            # Response DTOs
│   │   └── create-fulfillment.dto.ts     # Manual fulfillment DTO
│   ├── events/
│   │   └── fulfillment.events.ts         # Fulfillment event definitions
│   └── __tests__/
│       ├── fulfillment.service.spec.ts
│       └── fulfillment.controller.spec.ts
│
├── cancellation/
│   ├── cancellation.module.ts            # Module definition
│   ├── cancellation.service.ts           # Cancellation business logic
│   ├── cancellation.controller.ts        # Manual cancellation endpoint
│   └── __tests__/
│       └── cancellation.service.spec.ts
│
├── retry-queue/
│   ├── retry-queue.module.ts             # Module definition
│   ├── retry-queue.service.ts            # Retry queue processing
│   ├── retry-queue.processor.ts          # Scheduled job processor
│   ├── dto/
│   │   └── retry-job.dto.ts              # Retry job DTO
│   └── __tests__/
│       └── retry-queue.service.spec.ts
│
├── notifications/
│   ├── notifications.module.ts           # Module definition
│   ├── notifications.service.ts          # Notification dispatch
│   ├── email.service.ts                  # Email sending via nodemailer
│   ├── templates/
│   │   ├── error-notification.hbs        # Error email template
│   │   └── daily-summary.hbs             # Daily summary template
│   ├── dto/
│   │   └── notification.dto.ts           # Notification DTOs
│   └── __tests__/
│       └── notifications.service.spec.ts
│
├── orchestration/
│   └── orchestration.service.ts          # UPDATE: Add fulfillment trigger

prisma/
└── schema.prisma                          # UPDATE: Add RetryQueue model

prisma/migrations/
└── YYYYMMDD_add_retry_queue/              # Migration for retry queue table

model RetryQueue {
  id            String      @id @default(uuid())
  jobType       RetryJobType
  payload       Json
  attempts      Int         @default(0)
  maxAttempts   Int         @default(5)
  nextAttemptAt DateTime
  lastError     String?
  status        RetryStatus @default(PENDING)
  createdAt     DateTime    @default(now())
  updatedAt     DateTime    @updatedAt
}

enum RetryJobType {
  FULFILLMENT
  PRINT_JOB_CREATION
  CANCELLATION
  NOTIFICATION
}

enum RetryStatus {
  PENDING
  PROCESSING
  COMPLETED
  FAILED
}

pnpm prisma migrate dev --name add_retry_queue

import { Order } from '@prisma/client';

export const FULFILLMENT_EVENTS = {
  CREATED: 'fulfillment.created',
  FAILED: 'fulfillment.failed',
  RETRYING: 'fulfillment.retrying',
} as const;

export class FulfillmentCreatedEvent {
  constructor(
    public readonly order: Order,
    public readonly fulfillmentId: string
  ) {}
}

export class FulfillmentFailedEvent {
  constructor(
    public readonly order: Order,
    public readonly error: string,
    public readonly willRetry: boolean
  ) {}
}

import { Injectable, Logger, NotFoundException } from '@nestjs/common';
import { OnEvent } from '@nestjs/event-emitter';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { Order, OrderStatus } from '@prisma/client';
import * as Sentry from '@sentry/nestjs';
import { ShopifyApiClient } from '../shopify/shopify-api.client';
import { OrdersRepository } from '../orders/orders.repository';
import { EventLogService } from '../event-log/event-log.service';
import { RetryQueueService } from '../retry-queue/retry-queue.service';
import { ORDER_EVENTS, OrderReadyForFulfillmentEvent } from '../orders/events/order.events';
import {
  FULFILLMENT_EVENTS,
  FulfillmentCreatedEvent,
  FulfillmentFailedEvent,
} from './events/fulfillment.events';

interface FulfillmentInput {
  orderId: string;
  notify_customer?: boolean;
  trackingInfo?: {
    number?: string;
    url?: string;
    company?: string;
  };
}

@Injectable()
export class FulfillmentService {
  private readonly logger = new Logger(FulfillmentService.name);

  constructor(
    private readonly shopifyClient: ShopifyApiClient,
    private readonly ordersRepository: OrdersRepository,
    private readonly eventLogService: EventLogService,
    private readonly retryQueueService: RetryQueueService,
    private readonly eventEmitter: EventEmitter2
  ) {}

  /**
   * Listen for order.ready-for-fulfillment events from orchestration
   */
  @OnEvent(ORDER_EVENTS.READY_FOR_FULFILLMENT)
  async handleOrderReadyForFulfillment(event: OrderReadyForFulfillmentEvent): Promise<void> {
    this.logger.log(`Order ready for fulfillment: ${event.order.id}`);
    await this.createFulfillment({ orderId: event.order.id, notify_customer: true });
  }

  /**
   * Create fulfillment in Shopify
   */
  async createFulfillment(input: FulfillmentInput): Promise<void> {
    const order = await this.ordersRepository.findById(input.orderId);
    if (!order) {
      throw new NotFoundException(`Order not found: ${input.orderId}`);
    }

    // Skip if already fulfilled
    if (order.shopifyFulfillmentId) {
      this.logger.warn(`Order ${order.id} already fulfilled`);
      return;
    }

    // Skip if not completed
    if (order.status !== OrderStatus.COMPLETED) {
      this.logger.warn(`Order ${order.id} not in COMPLETED status, skipping fulfillment`);
      return;
    }

    try {
      this.logger.log(`Creating fulfillment for order ${order.shopifyOrderNumber}`);

      // Build fulfillment payload
      const fulfillmentPayload = {
        line_items: order.lineItems.map((item) => ({
          id: parseInt(item.shopifyLineItemId, 10),
          quantity: item.quantity,
        })),
        notify_customer: input.notify_customer ?? true,
        tracking_info: input.trackingInfo,
      };

      // Create fulfillment in Shopify
      const fulfillmentResponse = await this.shopifyClient.createFulfillment(
        order.shopifyOrderId,
        fulfillmentPayload
      );

      // Update local order record
      await this.ordersRepository.updateFulfillment(order.id, {
        shopifyFulfillmentId: String(fulfillmentResponse.fulfillment.id),
        trackingNumber: fulfillmentResponse.fulfillment.tracking_number,
        trackingUrl: fulfillmentResponse.fulfillment.tracking_url,
      });

      // Log success
      await this.eventLogService.log({
        orderId: order.id,
        eventType: 'ORDER_FULFILLED',
        severity: 'INFO',
        message: `Order fulfilled in Shopify`,
        metadata: {
          fulfillmentId: fulfillmentResponse.fulfillment.id,
          notifyCustomer: input.notify_customer,
        },
      });

      // Emit success event
      this.eventEmitter.emit(
        FULFILLMENT_EVENTS.CREATED,
        new FulfillmentCreatedEvent(order, String(fulfillmentResponse.fulfillment.id))
      );

      this.logger.log(`Fulfillment created for order ${order.shopifyOrderNumber}`);
    } catch (error) {
      await this.handleFulfillmentError(order, error);
    }
  }

  /**
   * Force fulfill an order (manual override)
   */
  async forceFulfill(orderId: string): Promise<void> {
    const order = await this.ordersRepository.findById(orderId);
    if (!order) {
      throw new NotFoundException(`Order not found: ${orderId}`);
    }

    await this.eventLogService.log({
      orderId,
      eventType: 'ORDER_FORCE_FULFILLED',
      severity: 'WARNING',
      message: 'Order manually force-fulfilled by operator',
    });

    // Update order status to completed if not already
    if (order.status !== OrderStatus.COMPLETED) {
      await this.ordersRepository.updateStatus(orderId, OrderStatus.COMPLETED);
    }

    // Create fulfillment
    await this.createFulfillment({ orderId, notify_customer: true });
  }

  /**
   * Handle fulfillment creation errors
   */
  private async handleFulfillmentError(order: Order, error: unknown): Promise<void> {
    const errorMessage = error instanceof Error ? error.message : 'Unknown error';

    this.logger.error(`Fulfillment failed for order ${order.id}: ${errorMessage}`);

    // Check if error is retryable
    const isRetryable = this.isRetryableError(error);

    if (isRetryable) {
      // Add to retry queue
      await this.retryQueueService.enqueue({
        jobType: 'FULFILLMENT',
        payload: { orderId: order.id },
        maxAttempts: 5,
      });

      await this.eventLogService.log({
        orderId: order.id,
        eventType: 'FULFILLMENT_RETRY_SCHEDULED',
        severity: 'WARNING',
        message: `Fulfillment failed, scheduled for retry: ${errorMessage}`,
        metadata: { error: errorMessage },
      });

      this.eventEmitter.emit(
        FULFILLMENT_EVENTS.FAILED,
        new FulfillmentFailedEvent(order, errorMessage, true)
      );
    } else {
      // Permanent failure - alert operator
      await this.eventLogService.log({
        orderId: order.id,
        eventType: 'FULFILLMENT_FAILED_PERMANENT',
        severity: 'ERROR',
        message: `Fulfillment permanently failed: ${errorMessage}`,
        metadata: { error: errorMessage, requiresAttention: true },
      });

      // Update order status to failed
      await this.ordersRepository.updateStatus(order.id, OrderStatus.FAILED);

      this.eventEmitter.emit(
        FULFILLMENT_EVENTS.FAILED,
        new FulfillmentFailedEvent(order, errorMessage, false)
      );

      Sentry.captureException(error, {
        tags: { service: 'fulfillment', action: 'create' },
        extra: { orderId: order.id, shopifyOrderId: order.shopifyOrderId },
      });
    }
  }

  /**
   * Determine if error is retryable
   */
  private isRetryableError(error: unknown): boolean {
    if (error instanceof Error) {
      const message = error.message.toLowerCase();
      const retryablePatterns = [
        'timeout',
        'rate limit',
        'service unavailable',
        '503',
        '429',
        'econnreset',
        'econnrefused',
        'etimedout',
      ];
      return retryablePatterns.some((pattern) => message.includes(pattern));
    }
    return false;
  }
}

import { Controller, Post, Param, HttpCode, HttpStatus, Logger } from '@nestjs/common';
import { ApiTags, ApiOperation, ApiResponse, ApiParam } from '@nestjs/swagger';
import { FulfillmentService } from './fulfillment.service';

@ApiTags('Fulfillment')
@Controller('api/v1/fulfillments')
export class FulfillmentController {
  private readonly logger = new Logger(FulfillmentController.name);

  constructor(private readonly fulfillmentService: FulfillmentService) {}

  @Post('order/:orderId')
  @HttpCode(HttpStatus.OK)
  @ApiOperation({ summary: 'Create fulfillment for an order' })
  @ApiParam({ name: 'orderId', description: 'Order ID to fulfill' })
  @ApiResponse({ status: 200, description: 'Fulfillment created successfully' })
  @ApiResponse({ status: 404, description: 'Order not found' })
  async createFulfillment(
    @Param('orderId') orderId: string
  ): Promise<{ success: boolean; message: string }> {
    await this.fulfillmentService.createFulfillment({ orderId });
    return { success: true, message: 'Fulfillment created' };
  }

  @Post('order/:orderId/force')
  @HttpCode(HttpStatus.OK)
  @ApiOperation({ summary: 'Force fulfill an order (manual override)' })
  @ApiParam({ name: 'orderId', description: 'Order ID to force fulfill' })
  @ApiResponse({ status: 200, description: 'Order force fulfilled' })
  @ApiResponse({ status: 404, description: 'Order not found' })
  async forceFulfill(
    @Param('orderId') orderId: string
  ): Promise<{ success: boolean; message: string }> {
    await this.fulfillmentService.forceFulfill(orderId);
    return { success: true, message: 'Order force fulfilled' };
  }
}

import { Module } from '@nestjs/common';
import { FulfillmentController } from './fulfillment.controller';
import { FulfillmentService } from './fulfillment.service';
import { ShopifyModule } from '../shopify/shopify.module';
import { OrdersModule } from '../orders/orders.module';
import { EventLogModule } from '../event-log/event-log.module';
import { RetryQueueModule } from '../retry-queue/retry-queue.module';

@Module({
  imports: [ShopifyModule, OrdersModule, EventLogModule, RetryQueueModule],
  controllers: [FulfillmentController],
  providers: [FulfillmentService],
  exports: [FulfillmentService],
})
export class FulfillmentModule {}

import { Injectable, Logger, NotFoundException } from '@nestjs/common';
import { OnEvent } from '@nestjs/event-emitter';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { Order, PrintJob, PrintJobStatus, OrderStatus } from '@prisma/client';
import * as Sentry from '@sentry/nestjs';
import { SimplyPrintApiClient } from '../simplyprint/simplyprint-api.client';
import { OrdersRepository } from '../orders/orders.repository';
import { PrintJobsRepository } from '../print-jobs/print-jobs.repository';
import { EventLogService } from '../event-log/event-log.service';
import { NotificationsService } from '../notifications/notifications.service';
import { ORDER_EVENTS } from '../orders/events/order.events';

interface CancellationResult {
  orderId: string;
  cancelledJobs: string[];
  flaggedJobs: string[];
  alreadyCompletedJobs: string[];
}

@Injectable()
export class CancellationService {
  private readonly logger = new Logger(CancellationService.name);

  constructor(
    private readonly simplyPrintClient: SimplyPrintApiClient,
    private readonly ordersRepository: OrdersRepository,
    private readonly printJobsRepository: PrintJobsRepository,
    private readonly eventLogService: EventLogService,
    private readonly notificationsService: NotificationsService,
    private readonly eventEmitter: EventEmitter2
  ) {}

  /**
   * Handle order cancellation from Shopify webhook
   */
  @OnEvent(ORDER_EVENTS.CANCELLED)
  async handleOrderCancelled(event: { orderId: string; reason?: string }): Promise<void> {
    this.logger.log(`Handling order cancellation: ${event.orderId}`);
    await this.cancelOrder(event.orderId, event.reason);
  }

  /**
   * Cancel an order and its associated print jobs
   */
  async cancelOrder(orderId: string, reason?: string): Promise<CancellationResult> {
    const order = await this.ordersRepository.findById(orderId);
    if (!order) {
      throw new NotFoundException(`Order not found: ${orderId}`);
    }

    const result: CancellationResult = {
      orderId,
      cancelledJobs: [],
      flaggedJobs: [],
      alreadyCompletedJobs: [],
    };

    // Get all print jobs for this order
    const printJobs = await this.printJobsRepository.findByOrderId(orderId);

    for (const job of printJobs) {
      await this.processPrintJobCancellation(job, result);
    }

    // Update order status
    await this.ordersRepository.updateStatus(orderId, OrderStatus.CANCELLED);

    // Log cancellation
    await this.eventLogService.log({
      orderId,
      eventType: 'ORDER_CANCELLED',
      severity: 'WARNING',
      message: reason || 'Order cancelled',
      metadata: {
        cancelledJobs: result.cancelledJobs.length,
        flaggedJobs: result.flaggedJobs.length,
        alreadyCompletedJobs: result.alreadyCompletedJobs.length,
      },
    });

    // If there are jobs that need attention, notify operator
    if (result.flaggedJobs.length > 0) {
      await this.notificationsService.sendOperatorAlert({
        type: 'CANCELLATION_NEEDS_REVIEW',
        orderId,
        message: `Order ${order.shopifyOrderNumber} cancelled but ${result.flaggedJobs.length} job(s) are already printing`,
        metadata: { flaggedJobs: result.flaggedJobs },
      });
    }

    this.logger.log(`Order ${orderId} cancellation processed: ${JSON.stringify(result)}`);
    return result;
  }

  /**
   * Process individual print job cancellation
   */
  private async processPrintJobCancellation(
    job: PrintJob,
    result: CancellationResult
  ): Promise<void> {
    switch (job.status) {
      case PrintJobStatus.PENDING:
      case PrintJobStatus.QUEUED:
      case PrintJobStatus.ASSIGNED:
        // Can be cancelled directly
        await this.cancelPrintJob(job);
        result.cancelledJobs.push(job.id);
        break;

      case PrintJobStatus.PRINTING:
        // Already printing - flag for operator review
        await this.flagForReview(job);
        result.flaggedJobs.push(job.id);
        break;

      case PrintJobStatus.COMPLETED:
        // Already printed - log warning
        this.logger.warn(`Print job ${job.id} already completed, cannot cancel`);
        result.alreadyCompletedJobs.push(job.id);
        break;

      case PrintJobStatus.CANCELLED:
      case PrintJobStatus.FAILED:
        // Already in terminal state
        this.logger.debug(`Print job ${job.id} already in terminal state: ${job.status}`);
        break;
    }
  }

  /**
   * Cancel a print job in SimplyPrint
   */
  private async cancelPrintJob(job: PrintJob): Promise<void> {
    try {
      if (job.simplyPrintJobId) {
        await this.simplyPrintClient.cancelJob(job.simplyPrintJobId);
        this.logger.log(`Cancelled SimplyPrint job: ${job.simplyPrintJobId}`);
      }

      await this.printJobsRepository.update(job.id, {
        status: PrintJobStatus.CANCELLED,
        completedAt: new Date(),
      });

      await this.eventLogService.log({
        printJobId: job.id,
        eventType: 'PRINT_JOB_CANCELLED',
        severity: 'INFO',
        message: 'Print job cancelled due to order cancellation',
      });
    } catch (error) {
      this.logger.error(`Failed to cancel print job ${job.id}: ${error.message}`);

      // Still mark as cancelled locally even if SimplyPrint fails
      await this.printJobsRepository.update(job.id, {
        status: PrintJobStatus.CANCELLED,
        errorMessage: `Cancellation failed: ${error.message}`,
      });

      Sentry.captureException(error, {
        tags: { service: 'cancellation', action: 'cancel-print-job' },
        extra: { printJobId: job.id, simplyPrintJobId: job.simplyPrintJobId },
      });
    }
  }

  /**
   * Flag a printing job for operator review
   */
  private async flagForReview(job: PrintJob): Promise<void> {
    await this.eventLogService.log({
      printJobId: job.id,
      eventType: 'PRINT_JOB_NEEDS_REVIEW',
      severity: 'WARNING',
      message: 'Order cancelled but print job is already printing - requires operator review',
      metadata: {
        simplyPrintJobId: job.simplyPrintJobId,
        printerId: job.printerId,
        requiresAttention: true,
      },
    });
  }
}

// In handleOrderCancelled method, emit event for cancellation service:
async handleOrderCancelled(payload: OrderWebhookPayload, webhookId: string): Promise<void> {
  const existingOrder = await this.ordersService.findByShopifyOrderId(String(payload.id));

  if (!existingOrder) {
    this.logger.warn(`Cannot cancel order ${payload.id} - not found in database`);
    return;
  }

  // Emit cancellation event for CancellationService to handle
  this.eventEmitter.emit(ORDER_EVENTS.CANCELLED, {
    orderId: existingOrder.id,
    reason: `Cancelled in Shopify at ${payload.cancelled_at}`,
  });

  this.logger.log(`Emitted cancellation event for order ${existingOrder.shopifyOrderNumber}`);
}

import { Injectable, Logger } from '@nestjs/common';
import { Prisma, RetryQueue, RetryJobType, RetryStatus } from '@prisma/client';
import { PrismaService } from '../database/prisma.service';

@Injectable()
export class RetryQueueRepository {
  private readonly logger = new Logger(RetryQueueRepository.name);

  constructor(private readonly prisma: PrismaService) {}

  async create(data: Prisma.RetryQueueCreateInput): Promise<RetryQueue> {
    return this.prisma.retryQueue.create({ data });
  }

  async findById(id: string): Promise<RetryQueue | null> {
    return this.prisma.retryQueue.findUnique({ where: { id } });
  }

  async findPendingJobs(limit = 10): Promise<RetryQueue[]> {
    return this.prisma.retryQueue.findMany({
      where: {
        status: RetryStatus.PENDING,
        nextAttemptAt: { lte: new Date() },
      },
      orderBy: { nextAttemptAt: 'asc' },
      take: limit,
    });
  }

  async update(id: string, data: Prisma.RetryQueueUpdateInput): Promise<RetryQueue> {
    return this.prisma.retryQueue.update({ where: { id }, data });
  }

  async markProcessing(id: string): Promise<RetryQueue> {
    return this.update(id, { status: RetryStatus.PROCESSING });
  }

  async markCompleted(id: string): Promise<RetryQueue> {
    return this.update(id, { status: RetryStatus.COMPLETED });
  }

  async markFailed(id: string, error: string): Promise<RetryQueue> {
    return this.update(id, {
      status: RetryStatus.FAILED,
      lastError: error,
    });
  }

  async scheduleRetry(id: string, nextAttemptAt: Date): Promise<RetryQueue> {
    const job = await this.findById(id);
    return this.update(id, {
      status: RetryStatus.PENDING,
      nextAttemptAt,
      attempts: (job?.attempts || 0) + 1,
    });
  }

  async deleteOldCompleted(olderThan: Date): Promise<number> {
    const result = await this.prisma.retryQueue.deleteMany({
      where: {
        status: RetryStatus.COMPLETED,
        updatedAt: { lt: olderThan },
      },
    });
    return result.count;
  }
}

import { Injectable, Logger } from '@nestjs/common';
import { RetryQueue, RetryJobType, RetryStatus } from '@prisma/client';
import { RetryQueueRepository } from './retry-queue.repository';
import { EventLogService } from '../event-log/event-log.service';

interface EnqueueParams {
  jobType: RetryJobType;
  payload: Record<string, unknown>;
  maxAttempts?: number;
  delayMs?: number;
}

interface RetryConfig {
  maxRetries: number;
  initialDelayMs: number;
  maxDelayMs: number;
  backoffMultiplier: number;
}

@Injectable()
export class RetryQueueService {
  private readonly logger = new Logger(RetryQueueService.name);
  private readonly config: RetryConfig = {
    maxRetries: 5,
    initialDelayMs: 1000, // 1 second
    maxDelayMs: 3600000, // 1 hour
    backoffMultiplier: 2,
  };

  constructor(
    private readonly repository: RetryQueueRepository,
    private readonly eventLogService: EventLogService
  ) {}

  /**
   * Add a job to the retry queue
   */
  async enqueue(params: EnqueueParams): Promise<RetryQueue> {
    const nextAttemptAt = new Date(Date.now() + (params.delayMs || this.config.initialDelayMs));

    const job = await this.repository.create({
      jobType: params.jobType,
      payload: params.payload,
      maxAttempts: params.maxAttempts || this.config.maxRetries,
      nextAttemptAt,
    });

    this.logger.log(`Enqueued retry job: ${job.id} (${params.jobType})`);
    return job;
  }

  /**
   * Get pending jobs ready for processing
   */
  async getPendingJobs(limit = 10): Promise<RetryQueue[]> {
    return this.repository.findPendingJobs(limit);
  }

  /**
   * Mark job as processing
   */
  async startProcessing(jobId: string): Promise<RetryQueue> {
    return this.repository.markProcessing(jobId);
  }

  /**
   * Mark job as completed
   */
  async completeJob(jobId: string): Promise<RetryQueue> {
    return this.repository.markCompleted(jobId);
  }

  /**
   * Handle job failure - either reschedule or mark as failed
   */
  async handleFailure(jobId: string, error: string): Promise<RetryQueue> {
    const job = await this.repository.findById(jobId);
    if (!job) {
      throw new Error(`Retry job not found: ${jobId}`);
    }

    const nextAttempt = job.attempts + 1;

    if (nextAttempt >= job.maxAttempts) {
      // Maximum retries exceeded
      this.logger.error(`Retry job ${jobId} exceeded max attempts (${job.maxAttempts})`);

      await this.eventLogService.log({
        eventType: 'RETRY_EXHAUSTED',
        severity: 'ERROR',
        message: `Retry job exhausted after ${job.maxAttempts} attempts`,
        metadata: {
          jobId,
          jobType: job.jobType,
          payload: job.payload,
          lastError: error,
          requiresAttention: true,
        },
      });

      return this.repository.markFailed(jobId, error);
    }

    // Calculate next attempt time with exponential backoff and jitter
    const delay = this.calculateDelay(nextAttempt);
    const nextAttemptAt = new Date(Date.now() + delay);

    this.logger.log(`Rescheduling retry job ${jobId} for ${nextAttemptAt.toISOString()}`);

    return this.repository.scheduleRetry(jobId, nextAttemptAt);
  }

  /**
   * Calculate delay with exponential backoff and jitter
   */
  private calculateDelay(attempt: number): number {
    // Exponential backoff: initialDelay * (multiplier ^ attempt)
    let delay = this.config.initialDelayMs * Math.pow(this.config.backoffMultiplier, attempt - 1);

    // Cap at max delay
    delay = Math.min(delay, this.config.maxDelayMs);

    // Add jitter (±10%)
    const jitter = delay * 0.1 * (Math.random() * 2 - 1);
    delay = Math.round(delay + jitter);

    return delay;
  }

  /**
   * Cleanup old completed jobs
   */
  async cleanupOldJobs(daysToKeep = 7): Promise<number> {
    const cutoffDate = new Date(Date.now() - daysToKeep * 24 * 60 * 60 * 1000);
    return this.repository.deleteOldCompleted(cutoffDate);
  }
}

import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
import { RetryQueue, RetryJobType } from '@prisma/client';
import * as Sentry from '@sentry/nestjs';
import { RetryQueueService } from './retry-queue.service';
import { FulfillmentService } from '../fulfillment/fulfillment.service';
import { PrintJobsService } from '../print-jobs/print-jobs.service';
import { NotificationsService } from '../notifications/notifications.service';
import { EventLogService } from '../event-log/event-log.service';

@Injectable()
export class RetryQueueProcessor implements OnModuleInit {
  private readonly logger = new Logger(RetryQueueProcessor.name);
  private isProcessing = false;

  constructor(
    private readonly retryQueueService: RetryQueueService,
    private readonly fulfillmentService: FulfillmentService,
    private readonly printJobsService: PrintJobsService,
    private readonly notificationsService: NotificationsService,
    private readonly eventLogService: EventLogService
  ) {}

  async onModuleInit(): Promise<void> {
    this.logger.log('Retry queue processor initialized');
  }

  /**
   * Process pending retry jobs every 30 seconds
   */
  @Cron(CronExpression.EVERY_30_SECONDS)
  async processQueue(): Promise<void> {
    // Prevent concurrent processing
    if (this.isProcessing) {
      return;
    }

    this.isProcessing = true;

    try {
      const pendingJobs = await this.retryQueueService.getPendingJobs(10);

      if (pendingJobs.length === 0) {
        return;
      }

      this.logger.debug(`Processing ${pendingJobs.length} retry jobs`);

      for (const job of pendingJobs) {
        await this.processJob(job);
      }
    } catch (error) {
      this.logger.error(`Error processing retry queue: ${error.message}`);
      Sentry.captureException(error, {
        tags: { service: 'retry-queue', action: 'process-queue' },
      });
    } finally {
      this.isProcessing = false;
    }
  }

  /**
   * Process a single retry job
   */
  private async processJob(job: RetryQueue): Promise<void> {
    this.logger.log(`Processing retry job ${job.id} (${job.jobType})`);

    try {
      await this.retryQueueService.startProcessing(job.id);

      switch (job.jobType) {
        case 'FULFILLMENT':
          await this.processsFulfillmentRetry(job);
          break;

        case 'PRINT_JOB_CREATION':
          await this.processPrintJobRetry(job);
          break;

        case 'NOTIFICATION':
          await this.processNotificationRetry(job);
          break;

        default:
          this.logger.warn(`Unknown retry job type: ${job.jobType}`);
      }

      await this.retryQueueService.completeJob(job.id);
      this.logger.log(`Retry job ${job.id} completed successfully`);
    } catch (error) {
      const errorMessage = error instanceof Error ? error.message : 'Unknown error';
      this.logger.error(`Retry job ${job.id} failed: ${errorMessage}`);
      await this.retryQueueService.handleFailure(job.id, errorMessage);
    }
  }

  /**
   * Retry fulfillment creation
   */
  private async processsFulfillmentRetry(job: RetryQueue): Promise<void> {
    const payload = job.payload as { orderId: string };
    await this.fulfillmentService.createFulfillment({ orderId: payload.orderId });
  }

  /**
   * Retry print job creation
   */
  private async processPrintJobRetry(job: RetryQueue): Promise<void> {
    const payload = job.payload as { lineItemId: string; orderId: string };
    // Implementation depends on PrintJobsService interface
    // await this.printJobsService.createPrintJobForLineItem(...)
    throw new Error('Print job retry not yet implemented');
  }

  /**
   * Retry notification sending
   */
  private async processNotificationRetry(job: RetryQueue): Promise<void> {
    const payload = job.payload as {
      type: string;
      recipients: string[];
      subject: string;
      body: string;
    };
    await this.notificationsService.sendEmail(payload);
  }

  /**
   * Cleanup old completed jobs daily
   */
  @Cron(CronExpression.EVERY_DAY_AT_3AM)
  async cleanupOldJobs(): Promise<void> {
    const deleted = await this.retryQueueService.cleanupOldJobs(7);
    this.logger.log(`Cleaned up ${deleted} old retry jobs`);
  }
}

# Email Notifications
SMTP_HOST=smtp.example.com
SMTP_PORT=587
SMTP_USER=notifications@forma3d.be
SMTP_PASS=your-smtp-password
SMTP_FROM=noreply@forma3d.be

# Notification Recipients
OPERATOR_EMAIL=operator@forma3d.be
ADMIN_EMAIL=admin@forma3d.be

# Notification Settings
NOTIFICATIONS_ENABLED=true

import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import * as nodemailer from 'nodemailer';
import * as handlebars from 'handlebars';
import * as fs from 'fs';
import * as path from 'path';

interface EmailOptions {
  to: string | string[];
  subject: string;
  template: string;
  context: Record<string, unknown>;
}

@Injectable()
export class EmailService implements OnModuleInit {
  private readonly logger = new Logger(EmailService.name);
  private transporter: nodemailer.Transporter;
  private templates: Map<string, handlebars.TemplateDelegate> = new Map();
  private isEnabled: boolean;

  constructor(private readonly configService: ConfigService) {}

  async onModuleInit(): Promise<void> {
    this.isEnabled = this.configService.get<boolean>('NOTIFICATIONS_ENABLED', false);

    if (!this.isEnabled) {
      this.logger.warn('Email notifications are disabled');
      return;
    }

    // Create transport
    this.transporter = nodemailer.createTransport({
      host: this.configService.get<string>('SMTP_HOST'),
      port: this.configService.get<number>('SMTP_PORT', 587),
      secure: false,
      auth: {
        user: this.configService.get<string>('SMTP_USER'),
        pass: this.configService.get<string>('SMTP_PASS'),
      },
    });

    // Load templates
    await this.loadTemplates();

    // Verify connection
    try {
      await this.transporter.verify();
      this.logger.log('Email service connected successfully');
    } catch (error) {
      this.logger.error(`Email service connection failed: ${error.message}`);
    }
  }

  /**
   * Load email templates
   */
  private async loadTemplates(): Promise<void> {
    const templatesDir = path.join(__dirname, 'templates');

    const templateFiles = ['error-notification', 'daily-summary'];

    for (const templateName of templateFiles) {
      const templatePath = path.join(templatesDir, `${templateName}.hbs`);
      if (fs.existsSync(templatePath)) {
        const source = fs.readFileSync(templatePath, 'utf8');
        this.templates.set(templateName, handlebars.compile(source));
      }
    }

    this.logger.log(`Loaded ${this.templates.size} email templates`);
  }

  /**
   * Send an email
   */
  async send(options: EmailOptions): Promise<boolean> {
    if (!this.isEnabled) {
      this.logger.debug('Email disabled, skipping send');
      return false;
    }

    try {
      const template = this.templates.get(options.template);
      if (!template) {
        throw new Error(`Template not found: ${options.template}`);
      }

      const html = template(options.context);

      await this.transporter.sendMail({
        from: this.configService.get<string>('SMTP_FROM'),
        to: Array.isArray(options.to) ? options.to.join(', ') : options.to,
        subject: options.subject,
        html,
      });

      this.logger.log(`Email sent: ${options.subject} to ${options.to}`);
      return true;
    } catch (error) {
      this.logger.error(`Failed to send email: ${error.message}`);
      return false;
    }
  }
}

import { Injectable, Logger } from '@nestjs/common';
import { OnEvent } from '@nestjs/event-emitter';
import { ConfigService } from '@nestjs/config';
import * as Sentry from '@sentry/nestjs';
import { EmailService } from './email.service';
import { EventLogService } from '../event-log/event-log.service';
import {
  FULFILLMENT_EVENTS,
  FulfillmentFailedEvent,
} from '../fulfillment/events/fulfillment.events';
import { PRINT_JOB_EVENTS, PrintJobFailedEvent } from '../print-jobs/events/print-job.events';

interface OperatorAlert {
  type: string;
  orderId?: string;
  message: string;
  metadata?: Record<string, unknown>;
}

interface EmailPayload {
  type: string;
  recipients: string[];
  subject: string;
  body: string;
}

@Injectable()
export class NotificationsService {
  private readonly logger = new Logger(NotificationsService.name);
  private readonly operatorEmail: string;
  private readonly adminEmail: string;

  constructor(
    private readonly configService: ConfigService,
    private readonly emailService: EmailService,
    private readonly eventLogService: EventLogService
  ) {
    this.operatorEmail = this.configService.get<string>('OPERATOR_EMAIL', '');
    this.adminEmail = this.configService.get<string>('ADMIN_EMAIL', '');
  }

  /**
   * Send alert to operator
   */
  async sendOperatorAlert(alert: OperatorAlert): Promise<void> {
    if (!this.operatorEmail) {
      this.logger.warn('Operator email not configured, skipping alert');
      return;
    }

    try {
      await this.emailService.send({
        to: this.operatorEmail,
        subject: `[Forma3D.Connect] ${alert.type}: Action Required`,
        template: 'error-notification',
        context: {
          alertType: alert.type,
          orderId: alert.orderId,
          message: alert.message,
          metadata: alert.metadata,
          timestamp: new Date().toISOString(),
          dashboardUrl: `${this.configService.get('APP_URL')}/orders/${alert.orderId}`,
        },
      });

      await this.eventLogService.log({
        orderId: alert.orderId,
        eventType: 'OPERATOR_ALERT_SENT',
        severity: 'INFO',
        message: `Operator alert sent: ${alert.type}`,
        metadata: { recipient: this.operatorEmail },
      });
    } catch (error) {
      this.logger.error(`Failed to send operator alert: ${error.message}`);
      Sentry.captureException(error, {
        tags: { service: 'notifications', action: 'operator-alert' },
      });
    }
  }

  /**
   * Send email (for retry queue)
   */
  async sendEmail(payload: EmailPayload): Promise<void> {
    await this.emailService.send({
      to: payload.recipients,
      subject: payload.subject,
      template: 'error-notification',
      context: { message: payload.body },
    });
  }

  /**
   * Listen for fulfillment failures that won't retry
   */
  @OnEvent(FULFILLMENT_EVENTS.FAILED)
  async handleFulfillmentFailed(event: FulfillmentFailedEvent): Promise<void> {
    if (!event.willRetry) {
      await this.sendOperatorAlert({
        type: 'FULFILLMENT_FAILED',
        orderId: event.order.id,
        message: `Fulfillment failed and will not be retried: ${event.error}`,
        metadata: { shopifyOrderNumber: event.order.shopifyOrderNumber },
      });
    }
  }

  /**
   * Listen for print job failures
   */
  @OnEvent(PRINT_JOB_EVENTS.FAILED)
  async handlePrintJobFailed(event: PrintJobFailedEvent): Promise<void> {
    await this.sendOperatorAlert({
      type: 'PRINT_JOB_FAILED',
      message: `Print job ${event.printJob.id} failed: ${event.errorMessage}`,
      metadata: {
        printJobId: event.printJob.id,
        simplyPrintJobId: event.printJob.simplyPrintJobId,
      },
    });
  }
}

<html>
  <head>
    <meta charset='utf-8' />
    <title>Forma3D.Connect Alert</title>
    <style>
      body {
        font-family: Arial, sans-serif;
        line-height: 1.6;
        color: #333;
      }
      .container {
        max-width: 600px;
        margin: 0 auto;
        padding: 20px;
      }
      .header {
        background: #dc3545;
        color: white;
        padding: 20px;
        border-radius: 5px 5px 0 0;
      }
      .content {
        background: #f8f9fa;
        padding: 20px;
        border: 1px solid #dee2e6;
      }
      .footer {
        background: #e9ecef;
        padding: 15px;
        border-radius: 0 0 5px 5px;
        font-size: 12px;
      }
      .button {
        display: inline-block;
        background: #007bff;
        color: white;
        padding: 10px 20px;
        text-decoration: none;
        border-radius: 5px;
      }
      .metadata {
        background: #fff;
        padding: 10px;
        border: 1px solid #dee2e6;
        margin: 10px 0;
        font-family: monospace;
        font-size: 12px;
      }
    </style>
  </head>
  <body>
    <div class='container'>
      <div class='header'>
        <h1>⚠️ {{alertType}}</h1>
      </div>
      <div class='content'>
        <p><strong>Time:</strong> {{timestamp}}</p>
        {{#if orderId}}
          <p><strong>Order ID:</strong> {{orderId}}</p>
        {{/if}}
        <p><strong>Message:</strong></p>
        <p>{{message}}</p>

        {{#if metadata}}
          <p><strong>Details:</strong></p>
          <div class='metadata'>
            <pre>{{json metadata}}</pre>
          </div>
        {{/if}}

        {{#if dashboardUrl}}
          <p>
            <a href='{{dashboardUrl}}' class='button'>View in Dashboard</a>
          </p>
        {{/if}}
      </div>
      <div class='footer'>
        <p>This is an automated message from Forma3D.Connect. Please do not reply directly.</p>
      </div>
    </div>
  </body>
</html>

import { Module } from '@nestjs/common';
import { ConfigModule } from '@nestjs/config';
import { EventEmitterModule } from '@nestjs/event-emitter';
import { ScheduleModule } from '@nestjs/schedule';
import { AppController } from './app.controller';
import { AppService } from './app.service';
import { DatabaseModule } from '../database/database.module';
import { HealthModule } from '../health/health.module';
import { ConfigurationModule } from '../config/config.module';
import { ShopifyModule } from '../shopify/shopify.module';
import { OrdersModule } from '../orders/orders.module';
import { ProductMappingsModule } from '../product-mappings/product-mappings.module';
import { EventLogModule } from '../event-log/event-log.module';
import { SimplyPrintModule } from '../simplyprint/simplyprint.module';
import { PrintJobsModule } from '../print-jobs/print-jobs.module';
import { OrchestrationModule } from '../orchestration/orchestration.module';
// Phase 3 modules
import { FulfillmentModule } from '../fulfillment/fulfillment.module';
import { CancellationModule } from '../cancellation/cancellation.module';
import { RetryQueueModule } from '../retry-queue/retry-queue.module';
import { NotificationsModule } from '../notifications/notifications.module';

@Module({
  imports: [
    ConfigModule.forRoot({
      isGlobal: true,
      envFilePath: ['.env.local', '.env'],
    }),
    EventEmitterModule.forRoot(),
    ScheduleModule.forRoot(),
    ConfigurationModule,
    DatabaseModule,
    HealthModule,
    EventLogModule,
    ShopifyModule,
    OrdersModule,
    ProductMappingsModule,
    SimplyPrintModule,
    PrintJobsModule,
    OrchestrationModule,
    // Phase 3
    FulfillmentModule,
    CancellationModule,
    RetryQueueModule,
    NotificationsModule,
  ],
  controllers: [AppController],
  providers: [AppService],
})
export class AppModule {}