kube-prometheus使用prometheus-webhook-dingtalk进行告警

prometheus-webhook-dingtalk

github地址

timonwong/prometheus-webhook-dingtalk: DingTalk integration for Prometheus Alertmanager (github.com)

配置文件

apiVersion: v1
kind: ConfigMap
metadata:
  name: dingtalk-config
  namespace: monitoring
data:
  config.yml: |-
    templates:
      - /etc/prometheus-webhook-dingtalk/template.tmpl
    targets:
      webhook:
        url: https://oapi.dingtalk.com/robot/send?access_token=19cb6bce1fb7ed778a8bfa47b3ddfd1ecbd4f8233d16b9a9f23613804aa7bf6b
        secret: SEC7047130be0d4e671a1f049baa37fb667612ef6606f7b6d7de968dcc445c1a565
        mention:
          all: true #@所有人
      webhook2:
        url: https://oapi.dingtalk.com/robot/send?access_token=19cb6bce1fb7ed778a8bfa47b3ddfd1ecbd4f8233d16b9a9f23613804aa7bf6b
        secret: SEC7047130be0d4e671a1f049baa37fb667612ef6606f7b6d7de968dcc445c1a565

  template.tmpl: |-
    {{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }}
    {{ define "__alertmanagerURL" }}{{ .ExternalURL }}/#/alerts?receiver={{ .Receiver }}{{ end }}
    
    {{ define "__text_alert_list" }}{{ range . }}
    **Labels**
    {{ range .Labels.SortedPairs }}> - {{ .Name }}: {{ .Value | markdown | html }}
    {{ end }}
    **Annotations**
    {{ range .Annotations.SortedPairs }}> - {{ .Name }}: {{ .Value | markdown | html }}
    {{ end }}
    **Source:** [{{ .GeneratorURL }}]({{ .GeneratorURL }})
    {{ end }}{{ end }}
    
    {{ define "default.__text_alert_list" }}{{ range . }}
    #### \[{{ .Labels.severity | upper }}\] {{ .Annotations.summary }}
    
    **Description:** {{ .Annotations.description }}
    
    **Graph:** [📈]({{ .GeneratorURL }})
    
    **Details:**
    {{ range .Labels.SortedPairs }}{{ if and (ne (.Name) "severity") (ne (.Name) "summary") }}> - {{ .Name }}: {{ .Value | markdown | html }}
    {{ end }}{{ end }}
    {{ end }}{{ end }}
    
    {{/* Default */}}
    {{ define "default.title" }}{{ template "__subject" . }}{{ end }}
    {{ define "default.content" }}#### \[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}\] **[{{ index .GroupLabels "alertname" }}]({{ template "__alertmanagerURL" . }})**
    {{ if gt (len .Alerts.Firing) 0 -}}
    **Alerts Firing**
    {{ template "default.__text_alert_list" .Alerts.Firing }}
    {{ range .AtMobiles }}@{{ . }}{{ end }}
    {{- end }}
    {{ if gt (len .Alerts.Resolved) 0 -}}
    **Alerts Resolved**
    {{ template "default.__text_alert_list" .Alerts.Resolved }}
    {{ range .AtMobiles }}@{{ . }}{{ end }}
    {{- end }}
    {{- end }}
    
    {{/* Legacy */}}
    {{ define "legacy.title" }}{{ template "__subject" . }}{{ end }}
    {{ define "legacy.content" }}#### \[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}\] **[{{ index .GroupLabels "alertname" }}]({{ template "__alertmanagerURL" . }})**
    {{ template "__text_alert_list" .Alerts.Firing }}
    {{- end }}
    
    {{/* Following names for compatibility */}}
    {{ define "ding.link.title" }}{{ template "default.title" . }}{{ end }}
    {{ define "ding.link.content" }}{{ template "default.content" . }}{{ end }}
    

---
apiVersion: v1
kind: Service
metadata:
  name: dingtalk
  namespace: monitoring
  labels:
    app: dingtalk
  annotations:
    prometheus.io/scrape: 'false'
spec:
  selector:
    app: dingtalk
  ports:
  - name: dingtalk
    port: 8060
    protocol: TCP
    targetPort: 8060
 
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: dingtalk
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: dingtalk
  template:
    metadata:
      name: dingtalk
      labels:
        app: dingtalk
    spec:
      containers:
      - name: dingtalk
        image: timonwong/prometheus-webhook-dingtalk:latest
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 8060
        readinessProbe:
          failureThreshold: 3
          periodSeconds: 5
          initialDelaySeconds: 30
          successThreshold: 1
          tcpSocket:
            port: 8060
        livenessProbe:
          tcpSocket:
            port: 8060
          initialDelaySeconds: 30
          periodSeconds: 10
        volumeMounts:
        - name: config
          mountPath: /etc/prometheus-webhook-dingtalk
        resources:
          limits:
            cpu: 100m
            memory: 100Mi
      volumes:
      - name: config
        configMap:
          name: dingtalk-config

alertmanager-secret.yaml 配置文件

apiVersion: v1
kind: Secret
metadata:
  labels:
    app.kubernetes.io/component: alert-router
    app.kubernetes.io/instance: main
    app.kubernetes.io/name: alertmanager
    app.kubernetes.io/part-of: kube-prometheus
    app.kubernetes.io/version: 0.23.0
  name: alertmanager-main
  namespace: monitoring
stringData:
  alertmanager.yaml: |-
    "global":
      "resolve_timeout": "5m"
    "receivers":
    - "name": "Webhook"
      "webhook_configs":
      - "url": "http://dingtalk.monitoring.svc.cluster.local:8060/dingtalk/webhook/send"
    "route":
      "group_by":
      - "namespace"
      "group_wait": "30s"  #组告警等待时间，也就是告警产生后等待30s，如果有同一组告警一起发出
      "receiver": "Webhook"
      "repeat_interval": "2m" #重复告警的间隔时间，减少报警发送频率
      "routes":
      - "matchers":
        - "alertname = .*"
        "receiver": "Webhook"