graph TB
    subgraph dashboard [Dashboard]
        Upload[File Upload]
        VaultItem[VaultItem Component]
        DataTable[Vault DataTable]
    end
    
    subgraph storage [Supabase Storage]
        Bucket[(vault bucket)]
        Trigger[Storage Trigger]
    end
    
    subgraph api [API Layer]
        ProcessAPI[processDocument]
        Reprocess[reprocessDocument]
    end
    
    subgraph db [Database]
        Documents[(documents table)]
        Tags[(document_tags)]
        Embeddings[(document_tag_embeddings)]
    end
    
    subgraph worker [Worker - BullMQ]
        ProcessDoc[process-document]
        ClassifyDoc[classify-document]
        ClassifyImg[classify-image]
        EmbedTags[embed-document-tags]
    end
    
    Upload --> Bucket
    Bucket --> Trigger
    Trigger --> Documents
    Upload -->|after upload| ProcessAPI
    ProcessAPI --> ProcessDoc
    
    ProcessDoc -->|PDF/text| ClassifyDoc
    ProcessDoc -->|image| ClassifyImg
    ClassifyDoc --> EmbedTags
    ClassifyImg --> EmbedTags
    
    ClassifyDoc --> Documents
    ClassifyImg --> Documents
    EmbedTags --> Tags
    EmbedTags --> Embeddings
    
    VaultItem -->|retry| Reprocess
    DataTable -->|retry| Reprocess
    Reprocess --> ProcessDoc

stateDiagram-v2
    [*] --> pending: File uploaded
    
    pending --> completed: Classification success
    pending --> completed: Classification failed (graceful)
    pending --> failed: Hard failure (retryable)
    pending --> failed: Stale timeout (>10 min)
    
    failed --> pending: User retry
    completed --> pending: User retry (unclassified)
    
    note right of pending
        Shows skeleton UI
        Fresh: < 10 minutes
        Stale: > 10 minutes (shows retry)
    end note
    
    note right of completed
        title=null: Amber indicator
        title!=null: Normal display
    end note
    
    note right of failed
        Red indicator
        Retry button shown
    end note

sequenceDiagram
    participant User
    participant Storage as Supabase Storage
    participant DB as Database
    participant Queue as BullMQ
    participant Process as process-document
    participant Classify as classify-document/image
    participant Embed as embed-document-tags
    
    User->>Storage: Upload file
    Storage->>DB: Create document (pending)
    Storage->>Queue: Trigger process-document
    
    Queue->>Process: Execute job
    
    alt PDF/Text Document
        Process->>Process: Extract text content
        Process->>Queue: Trigger classify-document
        Queue->>Classify: Execute classification
        Classify->>Classify: AI classification (with timeout)
        
        alt AI Success
            Classify->>DB: Update title, summary, tags
            Classify->>DB: Set status = completed
            Classify->>Queue: Trigger embed-document-tags
        else AI Failure (graceful)
            Classify->>DB: Set status = completed (title=null)
            Note over Classify,DB: User can still access file
        end
        
    else Image
        Process->>Process: Convert HEIC if needed
        Process->>Queue: Trigger classify-image
        Queue->>Classify: Execute classification
        Classify->>Classify: Vision AI classification
        
        alt AI Success
            Classify->>DB: Update title, summary, content
            Classify->>DB: Set status = completed
            Classify->>Queue: Trigger embed-document-tags
        else AI Failure (graceful)
            Classify->>DB: Set status = completed (title=null)
        end
    end
    
    opt Tags exist
        Queue->>Embed: Execute embedding
        Embed->>DB: Upsert tags and embeddings
    end

// Pattern: {action}_{teamId}_{identifier}
jobId: `process-doc_${teamId}_${filePath.join("/")}`
jobId: `classify-doc_${teamId}_${fileName}`
jobId: `classify-img_${teamId}_${fileName}`
jobId: `embed-tags_${teamId}_${documentId}`

const documentsQueueConfig = {
  name: "documents",
  concurrency: 10,            // Conservative for memory + API rate limits
  lockDuration: 660_000,      // 11 minutes (> process timeout)
  stalledInterval: 720_000,   // 12 minutes (> lock duration)
  limiter: {
    max: 20,                  // 20 jobs/second max - prevents API burst
    duration: 1000,
  },
};

// Sharp memory optimization (in image-processing.ts)
sharp.cache({ memory: 256, files: 20, items: 100 }); // 256MB cache limit
sharp.concurrency(2); // Limit internal parallelism

// File size limit for HEIC
const MAX_HEIC_FILE_SIZE = 15 * 1024 * 1024; // 15MB - larger files skip AI

flowchart TD
    A[Start Processing] --> B{Content Extraction}
    B -->|Success| C{AI Classification}
    B -->|Failure| D[Complete with null values]
    
    C -->|Success| E[Complete with metadata]
    C -->|Failure| D
    
    D --> F[User can access file]
    E --> F
    
    F --> G{User satisfied?}
    G -->|Yes| H[Done]
    G -->|No| I[Click Retry]
    I --> A

// In documents.config.ts - onFailed handler
onFailed: async (job, err) => {
  // Handle unsupported file types (not a failure)
  if (err instanceof UnsupportedFileTypeError) {
    await markAsCompleted(job, filename);
    return;
  }
  
  // Only mark failed on final attempt
  if (job.attemptsMade >= job.opts.attempts) {
    await markAsFailed(job);
  }
}

sequenceDiagram
    participant User
    participant UI as VaultItem/DataTable
    participant API as reprocessDocument
    participant DB as Database
    participant Queue as BullMQ
    
    User->>UI: Click "Retry" button
    UI->>UI: Set isReprocessing = true
    UI->>API: mutate({ id })
    
    API->>DB: Get document by ID
    API->>API: Validate pathTokens exist
    API->>API: Check mimetype supported
    
    alt Unsupported mimetype
        API->>DB: Set status = completed
        API-->>UI: { skipped: true }
    else Supported
        API->>DB: Set status = pending
        API->>Queue: Trigger process-document
        API-->>UI: { success: true, jobId }
    end
    
    UI->>UI: Show skeleton (isReprocessing || isPending)
    
    Note over Queue: Job processes...
    
    Queue->>DB: Update document
    DB-->>UI: React Query invalidation
    UI->>UI: Clear isReprocessing
    UI->>UI: Show result

// VaultItem component state management
const [isReprocessing, setIsReprocessing] = useState(false);

// Clear local state when document updates
useEffect(() => {
  if (isReprocessing) {
    if (isCompleted || isFailed || isLoading) {
      setIsReprocessing(false);
    }
  }
}, [isReprocessing, isLoading, isFailed, data.processingStatus]);

// Handle mutation errors
const reprocessMutation = useMutation({
  onSuccess: () => invalidateQueries(),
  onError: () => setIsReprocessing(false), // Allow retry
});

const isStaleProcessing =
  data.processingStatus === "pending" &&
  data.createdAt &&
  Date.now() - new Date(data.createdAt).getTime() > 10 * 60 * 1000;

// Show skeleton only for fresh pending (not stale)
const isLoading = data.processingStatus === "pending" && !isStaleProcessing;

// Show retry for stale processing
const showRetry = isFailed || needsClassification || isStaleProcessing;

flowchart TD
    A[Image Uploaded] --> B{Is HEIC?}
    B -->|Yes| C[convertHeicToJpeg]
    B -->|No| D[resizeImage]
    
    C --> E[Two-stage conversion]
    E --> F{Try Sharp}
    F -->|Success| G[JPEG @ 2048px]
    F -->|Failure| H[heic-convert fallback]
    H --> I[Sharp resize]
    I --> G
    
    D --> J{Size > 2048px?}
    J -->|Yes| K[Resize to fit 2048px]
    J -->|No| L[Keep original]
    K --> M[Continue to AI]
    L --> M
    G --> M

// image-processing.ts - Centralized image utilities

// Resize any image to fit within max dimensions
export async function resizeImage(
  inputBuffer: ArrayBuffer,
  mimetype: string,
  logger: Logger,
  options?: { maxSize?: number }
): Promise<{ buffer: Buffer; mimetype: string }> {
  const maxSize = options?.maxSize ?? IMAGE_SIZES.MAX_DIMENSION; // 2048px
  
  // Skip unsupported formats
  if (!RESIZABLE_MIMETYPES.has(mimetype)) {
    return { buffer: Buffer.from(inputBuffer), mimetype };
  }
  
  // Skip if already within size limits
  const metadata = await sharp(Buffer.from(inputBuffer)).metadata();
  if (metadata.width <= maxSize && metadata.height <= maxSize) {
    return { buffer: Buffer.from(inputBuffer), mimetype };
  }
  
  // Resize maintaining aspect ratio
  const buffer = await sharp(Buffer.from(inputBuffer))
    .rotate()
    .resize({ width: maxSize, height: maxSize, fit: "inside" })
    .toBuffer();
  
  return { buffer, mimetype };
}

// HEIC conversion with resize
export async function convertHeicToJpeg(
  inputBuffer: ArrayBuffer,
  logger: Logger,
  options?: { maxSize?: number }
): Promise<HeicConversionResult> {
  const maxSize = options?.maxSize ?? IMAGE_SIZES.MAX_DIMENSION; // 2048px
  
  // Try sharp first (handles HEIF/HEIC + mislabeled files)
  try {
    const buffer = await sharp(Buffer.from(inputBuffer))
      .rotate()
      .resize({ width: maxSize, height: maxSize, fit: "inside" })
      .toFormat("jpeg")
      .toBuffer();
    return { buffer, mimetype: "image/jpeg" };
  } catch (sharpError) {
    // Fall back to heic-convert for edge cases
    // Note: heic-convert decodes to raw pixels - memory intensive!
    // 12MP photo = ~48MB raw RGBA. Quality 0.8 reduces output size.
    const decodedImage = await convert({
      buffer: new Uint8Array(inputBuffer),
      format: "JPEG",
      quality: 0.8, // Reduced from 1.0 to save memory
    });
    
    const buffer = await sharp(Buffer.from(decodedImage))
      .rotate()
      .resize({ width: maxSize, height: maxSize, fit: "inside" })
      .toFormat("jpeg")
      .toBuffer();
    return { buffer, mimetype: "image/jpeg" };
  }
}

// In process-document.ts - graceful degradation for HEIC
// If conversion fails (e.g., OOM), document completes with fallback
try {
  const { buffer: image } = await convertHeicToJpeg(buffer, logger);
  // ... upload and continue
} catch (conversionError) {
  // Complete with fallback - user can still see file and retry
  await updateDocument({ title: filename, status: "completed" });
  return;
}

// timeout.ts - Centralized timeout constants
export const TIMEOUTS = {
  DOCUMENT_PROCESSING: 600_000,  // 10 minutes - full pipeline
  AI_CLASSIFICATION: 90_000,     // 90 seconds - AI calls
  CLASSIFICATION_JOB_WAIT: 180_000, // 3 minutes - parent waiting for child
  FILE_DOWNLOAD: 60_000,         // 1 minute - storage downloads
  FILE_UPLOAD: 60_000,           // 1 minute - storage uploads
  EMBEDDING: 30_000,             // 30 seconds - embedding generation
} as const;

// Image size constants
export const IMAGE_SIZES = {
  MAX_DIMENSION: 2048,  // Optimal for vision models + OCR
} as const;

// Usage with timeout wrapper
const result = await withTimeout(
  classifier.classifyDocument({ content }),
  TIMEOUTS.AI_CLASSIFICATION,
  `Classification timed out after ${TIMEOUTS.AI_CLASSIFICATION}ms`
);

CLASSIFICATION_JOB_WAIT (180s) > AI_CLASSIFICATION (90s) + FILE_DOWNLOAD (60s)