┌─────────────────────────────────────────────────────────────────────────┐
│                         Metrics Collection                              │
├─────────────────────────────────────────────────────────────────────────┤
│                                                                         │
│  ┌──────────────┐      ┌─────────────────────┐      ┌────────────────┐  │
│  │   Sidekiq    │      │   Sidekiq Web UI    │      │   Prometheus   │  │
│  │   Workers    │─────▶│   + Prometheus      │─────▶│                │  │
│  │              │      │   Exporter          │      │                │  │
│  └──────────────┘      └─────────────────────┘      └────────────────┘  │
│                              :3000/prometheus/metrics                   │
│                                                                         │
│  ┌──────────────┐      ┌─────────────────────┐      ┌────────────────┐  │
│  │   Sidekiq    │      │   StatsD Exporter   │      │   Prometheus   │  │
│  │   Pro        │─────▶│   (DogStatsD)       │─────▶│                │  │
│  │   Middleware │      │                     │      │                │  │
│  └──────────────┘      └─────────────────────┘      └────────────────┘  │
│     (optional)           :12345/metrics                                 │
│                                                                         │
└─────────────────────────────────────────────────────────────────────────┘

# lago-sidekiqs/config.ru
require './app'
require 'sidekiq'
require 'sidekiq/web'
require 'sidekiq/prometheus/exporter'
require 'sidekiq/throttled'
require 'sidekiq/throttled/web'

Sidekiq.configure_client do |config|
  config.redis = { url: ENV['REDIS_URL'] }
end

Sidekiq::Web.use(Rack::Session::Cookie, secret: ENV['SESSION_SECRET'])

run Rack::URLMap.new('/' => Sidekiq::Web, '/prometheus/metrics' => Sidekiq::Prometheus::Exporter)

# lago-api/config/initializers/sidekiq.rb
def configure_sidekiq_pro_metrics(config)
  statsd_endpoint = ENV.fetch("LAGO_SIDEKIQ_STATSD_ENDPOINT", nil)
  if statsd_endpoint.nil?
    Rails.logger.warn "LAGO_SIDEKIQ_STATSD_ENDPOINT not set, Sidekiq Pro metrics will not be reported"
    return
  end

  statsd_host, statsd_port = statsd_endpoint.split(":")
  if statsd_host.empty? || statsd_port.nil? || statsd_port.empty?
    Rails.logger.error "LAGO_SIDEKIQ_STATSD_ENDPOINT invalid format, expected host:port"
    return
  end

  require "datadog/statsd"

  config.dogstatsd = -> {
    Datadog::Statsd.new(statsd_host, statsd_port.to_i,
      tags: ["env:#{config[:environment]}", "service:sidekiq"],
      namespace: Rails.application.name)
  }

  config.server_middleware do |chain|
    require "sidekiq/middleware/server/statsd"
    chain.add Sidekiq::Middleware::Server::Statsd
  end
end

LAGO_SIDEKIQ_STATSD_ENDPOINT=statsd-exporter:9125

rate(lago_api_jobs_failure[5m]) / rate(lago_api_jobs_count[5m])

lago_api_jobs_perform{queue="billing", quantile="0.99"}

topk(10, lago_api_jobs_perform{quantile="0.5"})

sum by (queue) (rate(lago_api_jobs_count[5m]))

# Queue latency too high (jobs waiting too long)
- alert: SidekiqQueueLatencyHigh
  expr: sidekiq_queue_latency_seconds > 300
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: "Sidekiq queue {{ $labels.name }} has high latency"
    description: "Queue {{ $labels.name }} has jobs waiting for {{ $value | humanizeDuration }}"

# Dead jobs accumulating
- alert: SidekiqDeadJobsIncreasing
  expr: increase(sidekiq_dead_jobs[1h]) > 100
  labels:
    severity: critical
  annotations:
    summary: "Sidekiq dead jobs increasing rapidly"
    description: "{{ $value }} jobs moved to dead queue in the last hour"

# No workers available
- alert: SidekiqNoWorkers
  expr: sidekiq_workers == 0
  for: 2m
  labels:
    severity: critical
  annotations:
    summary: "No Sidekiq workers available"
    description: "All Sidekiq workers are down"

# Queue backlog building up
- alert: SidekiqQueueBacklog
  expr: sidekiq_queue_enqueued_jobs > 1000
  for: 10m
  labels:
    severity: warning
  annotations:
    summary: "Sidekiq queue {{ $labels.name }} has backlog"
    description: "Queue {{ $labels.name }} has {{ $value }} jobs waiting"

# High failure rate
- alert: SidekiqHighFailureRate
  expr: |
    rate(lago_api_jobs_failure[5m])
    / rate(lago_api_jobs_count[5m]) > 0.05
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: "High job failure rate for {{ $labels.worker }}"
    description: "{{ $labels.worker }} has {{ $value | humanizePercentage }} failure rate"

# Worker process in quiet mode (shutting down)
- alert: SidekiqWorkerQuiet
  expr: sidekiq_host_processes{quiet="true"} > 0
  for: 10m
  labels:
    severity: warning
  annotations:
    summary: "Sidekiq worker {{ $labels.host }} in quiet mode"
    description: "Worker has been shutting down for over 10 minutes"

# Slow job execution
- alert: SidekiqSlowJobs
  expr: lago_api_jobs_perform{quantile="0.99"} > 30
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: "Slow job execution for {{ $labels.worker }}"
    description: "P99 execution time is {{ $value }}s"

# Retry queue has jobs
- alert: SidekiqRetryQueueNotEmpty
  expr: sidekiq_retry_jobs > 50
  for: 15m
  labels:
    severity: info
  annotations:
    summary: "Sidekiq retry queue has pending jobs"
    description: "{{ $value }} jobs waiting to be retried"