Update alertmanager config

Add mailcatcher templates

charts/user-monitoring-demo/templates/demo-deploy.yaml

@@ -16,6 +16,6 @@ spec:
     spec:
       containers:
       - image: quay.io/acend/example-web-python
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         name: example-web-python
       restartPolicy: Always

charts/user-monitoring-stack/.helmignore

@@ -1,23 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*.orig
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/

charts/user-monitoring-stack/Chart.lock

@@ -1,6 +0,0 @@
-dependencies:
-- name: kube-prometheus-stack
-  repository: https://prometheus-community.github.io/helm-charts
-  version: 51.5.3
-digest: sha256:8e25b83cde5acfd903c3e45f30b9ae28201f9dd68daab1aeca709f1185608916
-generated: "2023-10-12T13:25:38.219319684+02:00"

charts/user-monitoring-stack/Chart.yaml

@@ -1,29 +0,0 @@
-apiVersion: v2
-name: user-monitoring-stack
-description: A Helm chart for Kubernetes
-
-# A chart can be either an 'application' or a 'library' chart.
-#
-# Application charts are a collection of templates that can be packaged into versioned archives
-# to be deployed.
-#
-# Library charts provide useful utilities or functions for the chart developer. They're included as
-# a dependency of application charts to inject those utilities and functions into the rendering
-# pipeline. Library charts do not define any templates and therefore cannot be deployed.
-type: application
-
-# This is the chart version. This version number should be incremented each time you make changes
-# to the chart and its templates, including the app version.
-# Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
-
-# This is the version number of the application being deployed. This version number should be
-# incremented each time you make changes to the application. Versions are not expected to
-# follow Semantic Versioning. They should reflect the version the application is using.
-# It is recommended to use it with quotes.
-appVersion: "1.16.0"
-dependencies:
-- name: kube-prometheus-stack
-  condition: promstack.enabled
-  version: "51.5.3"
-  repository: "https://prometheus-community.github.io/helm-charts"

charts/user-monitoring-stack/values.yaml

@@ -1,73 +0,0 @@
-promstack:
-  enabled: true
-
-kube-prometheus-stack:
-  prometheus:
-    enabled: true
-    prometheusSpec:
-      resources:
-        requests:
-          cpu: 100m
-          memory: 400Mi
-      scrapeInterval: 60s
-      serviceMonitorNamespaceSelector:
-        matchLabels:
-          user: <user>
-      serviceMonitorSelector: {}
-      podMonitorNamespaceSelector:
-        matchLabels:
-          user: <user>
-      podMonitorSelector: {}
-      probeNamespaceSelector:
-        matchLabels:
-          user: <user>
-      ruleSelectorNilUsesHelmValues: false
-      serviceMonitorSelectorNilUsesHelmValues: false
-      podMonitorSelectorNilUsesHelmValues: false
-      probeSelectorNilUsesHelmValues: false
-    serviceAccount:
-      create: true
-      name: "prometheus-<user>"
-  grafana:
-    enabled: false
-  alertmanager:
-    enabled: false
-  
-  kubernetesServiceMonitors:
-    enabled: false
-  
-
-
-# not relevant for the lab
-  crds:
-    enabled: false
-  kubeProxy:
-    enabled: false 
-  kubeScheduler:
-    enabled: false
-  kubeControllerManager:
-    enabled: false 
-  kubeEtcd:
-    enabled: false 
-  kubeApiServer:
-    enabled: false
-  kubelet:
-    enabled: false
-  coreDns:
-    enabled: false
-  kubeStateMetrics:
-    enabled: false
-  nodeExporter:
-    enabled: false
-  prometheusOperator:
-    enabled: false
-  kubeletService:
-    enabled: false
-  defaultRules:
-    create: false
-
-
-global:
-  rbac:
-    create: false
-    createAggregateClusterRoles: false

charts/user-monitoring/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.2.5
+version: 0.2.6
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/user-monitoring/templates/_user-am-config-mailcatcher.yaml

@@ -0,0 +1,23 @@
+{{- define "alertmanager.config-mailcatcher" }}
+---
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: AlertmanagerConfig
+metadata:
+  name: {{ .Values.user }}-mailcatcher
+  labels:
+    alertmanagerConfig: {{ .Values.user }}-alertmanager
+spec:
+  route:
+    groupBy: ['job']
+    groupWait: 30s
+    groupInterval: 5m
+    repeatInterval: 12h
+    receiver: 'mailcatcher'
+  receivers:
+    - name: 'mailcatcher'
+      emailConfigs:
+        - to: alert@localhost
+          from: prometheus-operator@localhost
+          smarthost: mailcatcher:1025
+          requireTLS: false
+{{- end }}

charts/user-monitoring/templates/_user-am-servicemonitor.yaml

@@ -0,0 +1,21 @@
+{{- define "alertmanager.servicemonitor" }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/name: mariadb
+  name: alertmanager
+spec:
+  endpoints:
+    - interval: 60s
+      port: web
+      scheme: http
+      path: /metrics
+  namespaceSelector:
+    matchNames:
+    - {{ .Values.user }}-monitoring
+  selector:
+    matchLabels:
+      operated-alertmanager: 'true'
+{{- end }}

charts/user-monitoring/templates/_user-am.yaml

@@ -4,7 +4,51 @@ apiVersion: monitoring.coreos.com/v1
 kind: Alertmanager
 metadata:
   labels:
-    app.kubernetes.io/name: {{ .Values.user }}-am
-  name: {{ .Values.user }}-am
-spec: {}
+    app.kubernetes.io/name: {{ .Values.user }}-alertmanager
+  name: {{ .Values.user }}-alertmanager
+spec:
+  alertmanagerConfigNamespaceSelector:
+    matchLabels:
+      user: {{ .Values.user }}-alertmanager
+  alertmanagerConfigSelector:
+    matchLabels:
+      alertmanagerConfig: {{ .Values.user }}-alertmanager
+  image: quay.io/prometheus/alertmanager:v0.25.0
+  replicas: 1
+  resources:
+    requests:
+      cpu: 10m
+      memory: 40Mi
+  storage:
+    volumeClaimTemplate:
+      spec:
+        resources:
+          requests:
+            storage: 100Mi
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    haproxy.org/auth-secret: basic-auth
+    haproxy.org/auth-type: basic-auth
+    haproxy.org/auth-realm: Authentication Required
+  name: {{ .Values.user }}-alertmanager
+spec:
+  ingressClassName: haproxy
+  rules:
+  - host: {{ .Values.user }}-alertmanager.training.cluster.acend.ch
+    http:
+      paths:
+      - backend:
+          service:
+            name: alertmanager-operated
+            port:
+              number: 9093
+        path: /
+        pathType: ImplementationSpecific
+  tls:
+  - hosts:
+    - {{ .Values.user }}-alertmanager.training.cluster.acend.ch
+    secretName: acend-wildcard
 {{- end }}

charts/user-monitoring/templates/_user-blackboxexporter-svc.yaml

@@ -0,0 +1,18 @@
+{{- define  "blackboxexporter.svc" }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: blackbox
+  labels:
+    app.kubernetes.io/name: blackbox-exporter
+spec:
+  ports:
+    - name: http
+      port: 9115
+      protocol: TCP
+      targetPort: 9115
+  selector:
+    app.kubernetes.io/name: blackbox-exporter
+  type: ClusterIP
+{{- end }}

charts/user-monitoring/templates/_user-grafana-datasource-secret.yaml

@@ -16,7 +16,7 @@ stringData:
         access: proxy
         editable: false
         type: prometheus
-        url: http://thanos-query.{{ .Values.user }}-monitoring.svc.cluster.local:10902
+        url: http://thanos-query:10902
         version: 1
       {{- if .Values.grafana.datasources }}
       {{- toYaml .Values.grafana.datasources | nindent 6 }} 

charts/user-monitoring/templates/_user-grafana-servicemonitor.yaml

@@ -0,0 +1,23 @@
+{{- define "grafana.servicemonitor" }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: grafana-monitor
+spec:
+  endpoints:
+    - basicAuth:
+        password:
+          name: basic-auth
+          key: {{ .Values.user }}
+        username:
+          name: basic-auth
+          key: grafana_user
+      interval: 60s
+      port: http
+      scheme: http
+      path: /metrics
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: grafana
+{{- end }}

charts/user-monitoring/templates/_user-mailcatcher-deploy.yaml

@@ -0,0 +1,28 @@
+{{- define "mailcatcher.deploy" }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: mailcatcher
+  name: mailcatcher
+  namespace: {{ .Values.user }}-monitoring
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: mailcatcher
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: mailcatcher
+    spec:
+      containers:
+      - image: sj26/mailcatcher:v0.9.0
+        name: mailcatcher
+        ports:
+        - containerPort: 1080
+          name: http
+        - containerPort: 1025
+          name: smtp
+{{- end }}

charts/user-monitoring/templates/_user-mailcatcher-ingress.yaml

@@ -0,0 +1,30 @@
+{{- define "mailcatcher.ingress" }}
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    haproxy.org/auth-secret: basic-auth
+    haproxy.org/auth-type: basic-auth
+    haproxy.org/auth-realm: Authentication Required
+  labels:
+    app.kubernetes.io/name: {{ .Values.user }}-mailcatcher
+  name: {{ .Values.user }}-mailcatcher
+spec:
+  ingressClassName: haproxy
+  rules:
+  - host: {{ .Values.user }}-mailcatcher.training.cluster.acend.ch
+    http:
+      paths:
+      - backend:
+          service:
+            name: mailcatcher
+            port:
+              number: 1080
+        path: /
+        pathType: ImplementationSpecific
+  tls:
+  - hosts:
+    - {{ .Values.user }}-mailcatcher.training.cluster.acend.ch
+    secretName: acend-wildcard
+{{- end }}

charts/user-monitoring/templates/_user-mailcatcher-svc.yaml

@@ -0,0 +1,22 @@
+{{- define "mailcatcher.svc" }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mailcatcher
+  labels:
+    app.kubernetes.io/name: mailcatcher
+spec:
+  ports:
+  - name: http
+    port: 1080
+    protocol: TCP
+    targetPort: http
+  - name: smtp
+    port: 1025
+    protocol: TCP
+    targetPort: smtp
+  selector:
+    app.kubernetes.io/name: mailcatcher
+  type: ClusterIP
+{{- end }}

charts/user-monitoring/templates/_user-prometheus-servicemonitor.yaml

@@ -0,0 +1,20 @@
+{{- define "prometheus.servicemonitor" }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/name: prometheus
+    user: {{ .Values.user }}
+  name: {{ .Values.user }}-prometheus-servicemonitor
+spec:
+  endpoints:
+  - path: /metrics
+    port: web
+  namespaceSelector:
+    matchNames:
+    - {{ .Values.user }}-monitoring
+  selector:
+    matchLabels:
+      operated-prometheus: 'true'
+{{- end }}

charts/user-monitoring/templates/_user-prometheus.yaml

@@ -7,6 +7,10 @@ metadata:
     app.kubernetes.io/name: prometheus
   name: prometheus
 spec:
+  {{- if .Values.query.enabled }}
+  thanos:
+    image: quay.io/thanos/thanos:v0.32.3
+  {{- end }}
   scrapeInterval: 60s
   serviceAccountName: prometheus-{{ .Values.user }}
   serviceMonitorNamespaceSelector:

charts/user-monitoring/templates/_user-thanos-query-deploy.yaml

@@ -25,10 +25,11 @@ spec:
         - --query.replica-label=thanos_ruler_replica
         - --endpoint=prometheus-operated.{{ .Values.user }}-monitoring.svc.cluster.local:10901
         - --endpoint=thanos-ruler-operated.{{ .Values.user }}-monitoring.svc.cluster.local:10901
-        - --endpoint=prometheus-operated.monitoring.svc.cluster.local:10901
+        - --store=dnssrv+prometheus-operated.{{ .Values.user }}-monitoring.svc.cluster.local
+        - --store=dnssrv+_web._tcp.prometheus-operated.{{ .Values.user }}-monitoring.svc.cluster.local
         - --store=dnssrv+_grpc._tcp.kube-prometheus-stack-thanos-discovery.monitoring.svc.cluster.local
         - --store.unhealthy-timeout=3d
-        image: quay.io/thanos/thanos:v0.25.1
+        image: quay.io/thanos/thanos:v0.32.3
         name: thanos-query
         ports:
         - containerPort: 10902

charts/user-monitoring/templates/_user-thanosruler.yaml

@@ -7,12 +7,26 @@ metadata:
     app.kubernetes.io/name: thanos-ruler
   name: thanos-ruler
 spec:
-  alertQueryUrl: {{ .Values.user }}-thanos-query.training.cluster.acend.ch
+  image: quay.io/thanos/thanos:v0.28.1
   evaluationInterval: 10s
   queryEndpoints:
-  - thanos-query.{{ .Values.user }}-monitoring.svc.cluster.local:10902
+  - dnssrv+_http._tcp.thanos-query:10902
   ruleSelector: {}
   ruleNamespaceSelector:
     matchLabels:
       user: {{ .Values.user }}
+  alertmanagersConfig:
+    key: alertmanager-configs.yaml
+    name: thanosruler-alertmanager-config
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: thanosruler-alertmanager-config
+stringData:
+  alertmanager-configs.yaml: |-
+    alertmanagers:
+    - static_configs:
+      - "dnssrv+_web._tcp.alertmanager-operated.{{ .Values.user }}-monitoring.svc.cluster.local"
+      api_version: v2
 {{- end }}

charts/user-monitoring/templates/monitoring.yaml

@@ -2,6 +2,18 @@
 
 {{- if .Values.alertmanager.enabled }}
   {{- template "alertmanager.alertmanager" . }}
+  {{- template "alertmanager.servicemonitor" . }}
+  {{- template "alertmanager.config-mailcatcher" . }}
+  {{- template "mailcatcher.deploy" . }}
+  {{- template "mailcatcher.svc" . }}
+  {{- template "mailcatcher.ingress" . }} 
+{{- end }}
+
+# blackboxexporter
+{{- if .Values.blackboxexporter.enabled }}
+  {{- template "blackboxexporter.cm" . }}
+  {{- template "blackboxexporter.deploy" . }}
+  {{- template "blackboxexporter.svc" . }}
 {{- end }}
 
 # grafana
@@ -12,6 +24,7 @@
   {{- template "grafana.svc" . }}
   {{- template "grafana.sa" . }}
   {{- template "grafana.deploy" . }}
+  {{- template "grafana.servicemonitor" . }}
   {{- template "grafana.ingress" . }}
 {{- end }}
 
@@ -19,6 +32,7 @@
 {{- if .Values.prometheus.enabled }}
   {{- template "prometheus.ingress" . }}
   {{- template "prometheus.prometheus" . }}
+  {{- template "prometheus.servicemonitor" . }}
   {{- template "prometheus.role" . }}
   {{- template "prometheus.rolebinding" . }}
   {{- template "prometheus.sa" . }}

charts/user-monitoring/values.yaml

@@ -1,32 +1,22 @@
 user: <user> # Replace me
-
 # prometheus
 prometheus:
   enabled: false
-
 # thanos-query
 query:
   enabled: false
-
 # grafana
 grafana:
-  datasources:
-  - name: prometheus
-    access: proxy
-    editable: false
-    type: prometheus
-    url: http://prometheus-operated.<user>-monitoring.svc.cluster.local:9090
   enabled: false
-
+# blackboxexporter
+blackboxexporter:
+  enabled: false
 # pushgateway
 pushgateway:
   enabled: false
-
 # alertmanager
 alertmanager:
   enabled: false
-
 # thanos-ruler
 ruler:
   enabled: false
-