Merge pull request 'icecheng/feature' (#117) from icecheng/feature into master

Reviewed-on: https://gitea.freeleaps.mathmast.com/freeleaps/freeleaps-ops/pulls/117

altermanager/alertmanager-config.yaml

@@ -3,7 +3,7 @@ kind: Secret
 type: Opaque
 metadata:
   name: altermanager-email-credentials
-  namespace: freeleaps-monitoring-system
+  namespace: freeleaps-prod
 data:
   password: cHducGNya3d0aXp5Z2RoZQ==
 ---
@@ -11,7 +11,7 @@ apiVersion: monitoring.coreos.com/v1alpha1
 kind: AlertmanagerConfig
 metadata:
   name: alertmanager-config
-  namespace: freeleaps-monitoring-system
+  namespace: freeleaps-prod
 spec:
   receivers:
     # - msteamsConfigs:
@@ -37,16 +37,21 @@ spec:
           sendResolved: true
           headers:
             - key: Subject
-              value: '🚨Freeleaps Alert: {{ .CommonAnnotations.summary }}'
+              value: '{{ if eq .Status "firing" }}🚨 Freeleaps Alert: {{ .CommonAnnotations.summary }}{{ else }}✅ Freeleaps Resolved: {{ .CommonAnnotations.summary }}{{ end }}'
           html: |-
+            <h3><strong>{{ if eq .Status "firing" }}🚨 Alert: {{ .CommonAnnotations.summary }}{{ else }}✅ Resolved: {{ .CommonAnnotations.summary }}{{ end }}</strong></h3>
+            <p><strong>📝 AlertName:</strong> {{ .CommonLabels.alertname }}</p>
+            <p><strong>🔧 Service:</strong> {{ .CommonLabels.service }}</p>
+            <p><strong>🔧 Pod:</strong> {{ .CommonLabels.pod }}({{ .CommonLabels.instance }})</p>
+            <p><strong>🏷️ Severity:</strong> {{ .CommonLabels.severity }}</p>
+            <p><strong>{{ if eq .Status "firing" }}🔴 Status:{{ else }}🟢 Status:{{ end }}</strong> {{ .Status | toUpper }}</p>
             <p>📝 Description: {{ .CommonAnnotations.description }}</p>
-            <p>{{ if eq .Status "firing" }}🔴 Status: {{ .Status | toUpper }}{{ else }}🟢 Status: {{ .Status | toUpper }}{{ end }}</p>
             <p>📖 Runbook: <a href="{{ .CommonAnnotations.runbook_url }}">{{ .CommonAnnotations.runbook_url }}</a></p>
       name: email
   route:
     groupBy:
       - severity
-    groupInterval: 30s
+    groupInterval: 5m
     receiver: email
-    groupWait: 30s
-    repeatInterval: 1m
+    groupWait: 5m
+    repeatInterval: 6h

freeleaps/helm-pkg/metrics/values.prod.yaml

@@ -94,7 +94,7 @@ metrics:
       labels:
         severity: critical
         service: metrics-service
-        namespace: freeleaps-monitoring-system
+        namespace: freeleaps-prod
       annotations:
         summary: Freeleaps Metrics service is down (instance {{ $labels.instance }})
         description: Freeleaps Metrics service has been down for more than 1 minutes.
@@ -105,7 +105,7 @@ metrics:
       labels:
         severity: warning
         service: metrics-service
-        namespace: freeleaps-monitoring-system
+        namespace: freeleaps-prod
       annotations:
         summary: High error rate in freeleaps metrics service (instance {{ $labels.instance }})
         description: Freeleaps Metrics service error rate is {{ $value }} errors per second.