I have been attempting to setup alerting with dyanamic notifications with prometheus alertmanager. I spent the last two days thinking I was doing something wrong, I could not see the severity field in the incident payload. The urgency would be set correctly if I set the severity under the pagerduty_config to ‘critical’, but I never saw the severity field in the incident message payload.

I then set the severity in the pagerduty_config to ‘warning’ and the incident showed up with a ‘Low’ urgency, so clearly the pagerduty_config severity field is being considered in the code logic when generating the pagerduty incident. However, I still do not see the severity field in the incident’s message payload.

Is this expected?

For references see below

receivers:

• name: default-receiver


• name: pagerduty
  
  pagerduty_configs:
  
  
  
  • 
    

    description: ‘d{{ .Status | toUpper }}{{ if eq .Status “firing” }}:{{ .Alerts.Firing
    
    | len }}{{ end }}] {{ .CommonAnnotations.summary }}’
    
    routing_key: KEY
    
    severity: ‘info’
    
    route:
    
    group_interval: 5m
    
    group_wait: 10s
    
    receiver: default-receiver
    
    repeat_interval: 3h
    
    routes:

    
    
  
  
  • 
    

    matchers:

    
    
    
    
    • team = sre
      
      receiver: pagerduty
    
    
    
    

    # the generated incident payload
    
    {
    
      "client": "Alertmanager",
    
      "client_url": "http://prometheus-alertmanager-0:9093/#/alerts?receiver=pagerduty",
    
      "contexts": null,
    
      "description": ",FIRING:1] Spark Operator available replicas is not the same as desired replicas.",
    
      "event_type": "trigger",
    
      "incident_key": "KEY",
    
      "service_key": "KEY",
    
      "details": {
    
        "firing": "Labels:
    
     - alertname = SparkOperatorDown
    
     - app_kubernetes_io_component = metrics
    
     - app_kubernetes_io_instance = prometheus
    
     - app_kubernetes_io_managed_by = Helm
    
     - app_kubernetes_io_name = kube-state-metrics
    
     - app_kubernetes_io_part_of = kube-state-metrics
    
     - app_kubernetes_io_version = 2.8.0
    
     - cluster = CLUSTERNAME
    
     - deployment = spark-spark-operator
    
     - helm_sh_chart = kube-state-metrics-4.30.0
    
     - instance =IP:PORT
    
     - job = kubernetes-service-endpoints
    
     - namespace = spark-operator
    
     - node = NODE
    
     - service = prometheus-kube-state-metrics
    
     - severity = page
    
     - team = sre
    
    Annotations:
    
     - summary = Spark Operator available replicas is not the same as desired replicas.
    
    Source: http://prometheus-server-8577d64b5d-4hbhz:9090/graph?g0.expr=kube_deployment_status_replicas_available%7Bdeployment%3D%22spark-spark-operator%22%2Cnamespace%3D%22spark-operator%22%7D+%2F+kube_deployment_status_replicas%7Bdeployment%3D%22spark-spark-operator%22%2Cnamespace%3D%22spark-operator%22%7D+%21%3D+1&g0.tab=1
    
    ",
    
        "num_firing": "1",
    
        "num_resolved": "0",
    
        "resolved": ""
    
      }
    
    }