alertmanager/alertmanager.yml

# Credential fields are *_file only. Placeholder files are created under
# /tmp/observastack-secrets by make check because amtool requires the paths to exist.
# Those files are not secrets and are not committed.
global:
  resolve_timeout: 5m
  smtp_from: alertmanager@larkspur.example
  smtp_smarthost: smtp.larkspur.example:587
  smtp_auth_username: alertmanager@larkspur.example
  smtp_auth_password_file: /tmp/observastack-secrets/smtp_password
  smtp_require_tls: true

templates:
  - alertmanager/templates/*.tmpl

route:
  receiver: default
  group_by: [alertname, service, severity]
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  routes:
    - matchers:
        - alertname = Watchdog
      receiver: deadman-webhook
      group_wait: 0s
      group_interval: 1m
      repeat_interval: 1m
    # Sibling routes, not a parent/child pair. Alertmanager notifies the leaf
    # receiver only, so a child slack route would drop pagerduty-oncall.
    # continue: true on both keeps matching into the team routes below.
    - matchers:
        - severity = critical
      receiver: pagerduty-oncall
      continue: true
    - matchers:
        - severity = critical
      receiver: slack-incidents
      continue: true
    - matchers:
        - team = payments
      receiver: slack-payments
    - matchers:
        - team = orders
      receiver: email-oncall
    - matchers:
        - severity = warning
      receiver: slack-platform
      active_time_intervals:
        - business-hours
      mute_time_intervals:
        - maintenance-window
    - matchers:
        - severity = info
      receiver: "null"

inhibit_rules:
  - source_matchers:
      - alertname = InstanceDown
    target_matchers:
      - alertname != InstanceDown
    equal:
      - instance
  - source_matchers:
      - severity = critical
    target_matchers:
      - severity = warning
    equal:
      - alertname
      - service
  - source_matchers:
      - alertname = OrdersApiErrorBudgetBurnFast
    target_matchers:
      - alertname = ServiceHighErrorRate
      - service = orders-api
    equal:
      - service

time_intervals:
  - name: business-hours
    time_intervals:
      - times:
          - start_time: "09:00"
            end_time: "18:00"
        weekdays: ["monday:friday"]
        location: Asia/Shanghai
  - name: maintenance-window
    time_intervals:
      - times:
          - start_time: "02:00"
            end_time: "04:00"
        weekdays: ["sunday"]
        location: Asia/Shanghai

receivers:
  - name: default
  - name: "null"
  - name: deadman-webhook
    webhook_configs:
      - url: https://example.invalid/deadman
        send_resolved: false
  - name: pagerduty-oncall
    pagerduty_configs:
      - routing_key_file: /tmp/observastack-secrets/pagerduty_routing_key
        url: https://events.pagerduty.example/v2/enqueue
        send_resolved: true
  - name: slack-incidents
    slack_configs:
      - api_url_file: /tmp/observastack-secrets/slack_incidents_url
        channel: "#incidents"
        title: '{{ template "slack.larkspur.title" . }}'
        text: '{{ template "slack.larkspur.text" . }}'
        send_resolved: true
  - name: slack-payments
    slack_configs:
      - api_url_file: /tmp/observastack-secrets/slack_payments_url
        channel: "#payments"
        title: '{{ template "slack.larkspur.title" . }}'
        text: '{{ template "slack.larkspur.text" . }}'
        send_resolved: true
  - name: slack-platform
    slack_configs:
      - api_url_file: /tmp/observastack-secrets/slack_platform_url
        channel: "#platform"
        title: '{{ template "slack.larkspur.title" . }}'
        text: '{{ template "slack.larkspur.text" . }}'
        send_resolved: true
  - name: email-oncall
    email_configs:
      - to: oncall@larkspur.example
        from: alertmanager@larkspur.example
        smarthost: smtp.larkspur.example:587
        auth_username: alertmanager@larkspur.example
        auth_password_file: /tmp/observastack-secrets/smtp_password
        require_tls: true
        headers:
          Subject: '{{ template "email.larkspur.subject" . }}'