rules/alerts/infra.yml

groups:
  - name: infra.alerts
    rules:
      - alert: InstanceDown
        expr: up == 0
        for: 2m
        labels:
          severity: critical
          team: platform
        annotations:
          summary: A scrape target is down
          description: Prometheus has seen up == 0 for this target for at least 2 minutes. Other alerts on the same instance are inhibited.
          runbook_url: https://runbooks.larkspur.example/InstanceDown
      - alert: ScrapeTargetMissing
        expr: absent(up{job="storefront"})
        for: 5m
        labels:
          severity: critical
          team: platform
          job: storefront
        annotations:
          summary: The storefront scrape job has no targets
          description: absent(up{job="storefront"}) has been true for 5 minutes, so the whole storefront job is missing rather than a single instance being down.
          runbook_url: https://runbooks.larkspur.example/ScrapeTargetMissing
      - alert: NodeFilesystemFillingUp
        expr: |
          instance:node_filesystem_avail:ratio < 0.2
          and
          predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"}[6h], 4 * 3600) < 0
        for: 30m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: A filesystem is predicted to fill within 4 hours
          description: Available space is already under 20 percent and predict_linear over 6 hours reaches zero within 4 hours. tmpfs and overlay mounts are excluded.
          runbook_url: https://runbooks.larkspur.example/NodeFilesystemFillingUp
      - alert: NodeMemoryHigh
        expr: instance:node_memory_utilisation:ratio > 0.90
        for: 15m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: Host memory utilisation is above 90 percent
          description: instance:node_memory_utilisation:ratio has stayed above 0.90 for 15 minutes.
          runbook_url: https://runbooks.larkspur.example/NodeMemoryHigh
      - alert: PostgresDown
        expr: pg_up == 0
        for: 2m
        labels:
          severity: critical
          team: platform
        annotations:
          summary: Postgres exporter reports the database is down
          description: pg_up has been 0 for at least 2 minutes.
          runbook_url: https://runbooks.larkspur.example/PostgresDown
      # pg_stat_activity_connections avoids the promtool check-metrics "_count" suffix lint.
      # The catalog is the name the rules match. This is not the postgres_exporter metric name.
      - alert: PostgresConnectionsNearLimit
        expr: pg_stat_activity_connections / pg_settings_max_connections > 0.85
        for: 10m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: Postgres connections are above 85 percent of max_connections
          description: pg_stat_activity_connections divided by pg_settings_max_connections has stayed above 0.85 for 10 minutes.
          runbook_url: https://runbooks.larkspur.example/PostgresConnectionsNearLimit
      - alert: BlackboxProbeFailed
        expr: probe_success == 0
        for: 5m
        labels:
          severity: critical
          team: platform
        annotations:
          summary: An external blackbox probe is failing
          description: probe_success has been 0 for 5 minutes on a blackbox HTTP or TLS target.
          runbook_url: https://runbooks.larkspur.example/BlackboxProbeFailed
      - alert: TLSCertExpiringSoon
        expr: (probe_ssl_earliest_cert_expiry - time()) < (14 * 24 * 3600)
        for: 1h
        labels:
          severity: warning
          team: platform
        annotations:
          summary: A TLS certificate expires within 14 days
          description: probe_ssl_earliest_cert_expiry is less than 14 days ahead of the evaluation time, and that has held for 1 hour.
          runbook_url: https://runbooks.larkspur.example/TLSCertExpiringSoon