Update alertmanager config

Add mailcatcher templates

.github/workflows/release-charts.yaml

@@ -0,0 +1,32 @@
+name: Release Charts
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  release:
+    # depending on default permission settings for your org (contents being read-only or read-write for workloads), you will have to add permissions
+    # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Configure Git
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+
+      - name: Install Helm
+        uses: azure/setup-helm@v3
+
+      - name: Run chart-releaser
+        uses: helm/chart-releaser-action@v1.5.0
+        env:
+          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

apps/user-demo.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: <user>-demo
+  namespace: argocd
+spec:
+  destination:
+    namespace: <user>
+    server: https://kubernetes.default.svc
+  project: default
+  source:
+    repoURL: 'https://gitea.training.cluster.acend.ch/<user>/prometheus-training-lab-setup'
+    path: user-demo/
+    targetRevision: main
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - Replace=true

apps/user-prom-stack.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: <user>-prom-stack
+  namespace: argocd
+spec:
+  destination:
+    namespace: <user>-monitoring
+    server: https://kubernetes.default.svc
+  project: default
+  source:
+    repoURL: 'https://gitea.training.cluster.acend.ch/<user>/prometheus-training-lab-setup'
+    path: charts/user-monitoring/
+    targetRevision: main
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - Replace=true

charts/user-monitoring-demo/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.1.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/user-monitoring-demo/templates/demo-deploy.yaml

@@ -16,6 +16,6 @@ spec:
     spec:
       containers:
       - image: quay.io/acend/example-web-python
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         name: example-web-python
       restartPolicy: Always

charts/user-monitoring-demo/values.yaml

@@ -0,0 +1 @@
+user: userX # Replace me

charts/user-monitoring/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.2.6
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/user-monitoring/templates/_user-am-config-mailcatcher.yaml

@@ -0,0 +1,23 @@
+{{- define "alertmanager.config-mailcatcher" }}
+---
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: AlertmanagerConfig
+metadata:
+  name: {{ .Values.user }}-mailcatcher
+  labels:
+    alertmanagerConfig: {{ .Values.user }}-alertmanager
+spec:
+  route:
+    groupBy: ['job']
+    groupWait: 30s
+    groupInterval: 5m
+    repeatInterval: 12h
+    receiver: 'mailcatcher'
+  receivers:
+    - name: 'mailcatcher'
+      emailConfigs:
+        - to: alert@localhost
+          from: prometheus-operator@localhost
+          smarthost: mailcatcher:1025
+          requireTLS: false
+{{- end }}

charts/user-monitoring/templates/_user-am-servicemonitor.yaml

@@ -0,0 +1,21 @@
+{{- define "alertmanager.servicemonitor" }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/name: mariadb
+  name: alertmanager
+spec:
+  endpoints:
+    - interval: 60s
+      port: web
+      scheme: http
+      path: /metrics
+  namespaceSelector:
+    matchNames:
+    - {{ .Values.user }}-monitoring
+  selector:
+    matchLabels:
+      operated-alertmanager: 'true'
+{{- end }}

charts/user-monitoring/templates/_user-am.yaml

@@ -0,0 +1,54 @@
+{{- define "alertmanager.alertmanager" }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: Alertmanager
+metadata:
+  labels:
+    app.kubernetes.io/name: {{ .Values.user }}-alertmanager
+  name: {{ .Values.user }}-alertmanager
+spec:
+  alertmanagerConfigNamespaceSelector:
+    matchLabels:
+      user: {{ .Values.user }}-alertmanager
+  alertmanagerConfigSelector:
+    matchLabels:
+      alertmanagerConfig: {{ .Values.user }}-alertmanager
+  image: quay.io/prometheus/alertmanager:v0.25.0
+  replicas: 1
+  resources:
+    requests:
+      cpu: 10m
+      memory: 40Mi
+  storage:
+    volumeClaimTemplate:
+      spec:
+        resources:
+          requests:
+            storage: 100Mi
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    haproxy.org/auth-secret: basic-auth
+    haproxy.org/auth-type: basic-auth
+    haproxy.org/auth-realm: Authentication Required
+  name: {{ .Values.user }}-alertmanager
+spec:
+  ingressClassName: haproxy
+  rules:
+  - host: {{ .Values.user }}-alertmanager.training.cluster.acend.ch
+    http:
+      paths:
+      - backend:
+          service:
+            name: alertmanager-operated
+            port:
+              number: 9093
+        path: /
+        pathType: ImplementationSpecific
+  tls:
+  - hosts:
+    - {{ .Values.user }}-alertmanager.training.cluster.acend.ch
+    secretName: acend-wildcard
+{{- end }}

charts/user-monitoring/templates/_user-blackboxexporter-cm.yaml

@@ -0,0 +1,39 @@
+{{- define  "blackboxexporter.cm" }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: blackbox-exporter-config
+  labels:
+    app.kubernetes.io/name: blackbox-exporter
+data:
+  config.yml: |
+    modules:
+      http_2xx:
+        prober: http
+        timeout: 10s
+        http:
+          preferred_ip_protocol: "ip4"
+          tls_config:
+            insecure_skip_verify: true
+      http_3xx:
+        prober: http
+        timeout: 10s
+        http:
+          preferred_ip_protocol: "ip4"
+          follow_redirects: false
+          tls_config:
+            insecure_skip_verify: true
+          valid_status_codes: [300, 301, 302, 303, 304, 305, 306, 307, 308]
+      http_4xx:
+        prober: http
+        timeout: 10s
+        http:
+          preferred_ip_protocol: "ip4"
+          follow_redirects: false
+          tls_config:
+            insecure_skip_verify: true
+          valid_status_codes: [400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 421, 422, 423, 424, 425, 426, 428, 429, 431, 451, 418, 420, 444, 449, 499]
+      tcp_connect:
+        prober: tcp
+{{- end }}

charts/user-monitoring/templates/_user-blackboxexporter-deploy.yaml

@@ -0,0 +1,43 @@
+{{- define  "blackboxexporter.deploy" }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: blackbox-exporter
+  name: blackbox-exporter
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: blackbox-exporter
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: blackbox-exporter
+    spec:
+      containers:
+        - name: blackbox-exporter
+          image: quay.io/prometheus/blackbox-exporter:v0.24.0
+          args:
+            - --config.file=/etc/blackbox_exporter/config.yml
+          ports:
+            - containerPort: 9115
+              name: http
+          volumeMounts:
+            - mountPath: /etc/blackbox_exporter/
+              name: blackbox-config
+          resources: {}
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: http
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+      volumes:
+        - name: blackbox-config
+          configMap:
+            defaultMode: 420
+            name: blackbox-exporter-config
+{{- end }}

charts/user-monitoring/templates/_user-blackboxexporter-svc.yaml

@@ -0,0 +1,18 @@
+{{- define  "blackboxexporter.svc" }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: blackbox
+  labels:
+    app.kubernetes.io/name: blackbox-exporter
+spec:
+  ports:
+    - name: http
+      port: 9115
+      protocol: TCP
+      targetPort: 9115
+  selector:
+    app.kubernetes.io/name: blackbox-exporter
+  type: ClusterIP
+{{- end }}

charts/user-monitoring/templates/_user-grafana-cm.yaml

@@ -0,0 +1,20 @@
+{{- define "grafana.dashboardprovider" }}
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: grafana-dashboardproviders
+  labels:
+    app.kubernetes.io/name: grafana
+data:
+  dashboardproviders.yaml: |
+    apiVersion: 1
+    providers:
+    - disableDeletion: false
+      editable: false
+      name: general
+      options:
+        path: /var/lib/grafana/dashboards/general
+      orgId: 1
+      type: file
+{{- end }}

charts/user-monitoring/templates/_user-grafana-datasource-secret.yaml

@@ -0,0 +1,24 @@
+{{- define "grafana.datasource" }}
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: grafana-datasources
+  labels:
+    app.kubernetes.io/name: grafana
+    datasource: "true"
+stringData:
+  dashboardproviders.yaml: |
+    apiVersion: 1
+    datasources:
+      - name: thanos-querier
+        access: proxy
+        editable: false
+        type: prometheus
+        url: http://thanos-query:10902
+        version: 1
+      {{- if .Values.grafana.datasources }}
+      {{- toYaml .Values.grafana.datasources | nindent 6 }} 
+      {{- end }}
+{{- end }}

charts/user-monitoring/templates/_user-grafana-deploy.yaml

@@ -0,0 +1,116 @@
+{{- define "grafana.deploy" }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: grafana
+  name: grafana
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: grafana
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: grafana
+    spec:
+      serviceAccount: grafana
+      serviceAccountName: grafana
+      initContainers:
+        - name: grafana-sc-datasources
+          env:
+          - name: METHOD
+            value: LIST
+          - name: LABEL
+            value: datasource
+          - name: LABEL_VALUE
+            value: "true"
+          - name: FOLDER
+            value: /etc/grafana/provisioning/datasources
+          - name: RESOURCE
+            value: secret
+          image: quay.io/kiwigrid/k8s-sidecar:1.25.0
+          resources:
+            limits:
+              cpu: 200m
+              memory: 128Mi
+            requests:
+              cpu: 50m
+              memory: 32Mi
+          volumeMounts:
+          - mountPath: /etc/grafana/provisioning/datasources
+            name: datasource-volume
+      containers:
+        - name: grafana-sc-dashboard
+          env:
+            - name: LABEL
+              value: dashboard
+            - name: LABEL_VALUE
+              value: "true"
+            - name: FOLDER
+              value: /var/lib/grafana/dashboards/general
+            - name: RESOURCE
+              value: configmap
+          image: quay.io/kiwigrid/k8s-sidecar:1.25.0
+          resources:
+            limits:
+              cpu: 1
+              memory: 128Mi
+            requests:
+              cpu: 50m
+              memory: 32Mi
+          volumeMounts:
+            - mountPath: /var/lib/grafana/dashboards/general
+              name: dashboard-volume
+        - name: grafana
+          env:
+            - name: GF_ANALYTICS_REPORTING_ENABLED
+              value: "false"
+            - name: TZ
+              value: "Europe/Zurich"
+          image: grafana/grafana:10.0.3
+          livenessProbe:
+            failureThreshold: 10
+            httpGet:
+              path: /api/health
+              port: 3000
+              scheme: HTTP
+            initialDelaySeconds: 60
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 30
+          ports:
+            - containerPort: 3000
+              name: grafana-http
+              protocol: TCP
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /api/health
+              port: 3000
+              scheme: HTTP
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          volumeMounts:
+            - mountPath: /etc/grafana/provisioning/dashboards/
+              name: grafana-dashboardproviders
+            - mountPath: /var/lib/grafana/data
+              name: grafana-volume
+            - mountPath: /var/lib/grafana/dashboards/general
+              name: dashboard-volume
+            - mountPath: /etc/grafana/provisioning/datasources/
+              name: datasource-volume
+      volumes:
+        - name: grafana-dashboardproviders
+          configMap:
+            defaultMode: 420
+            name: grafana-dashboardproviders
+        - emptyDir: {}
+          name: grafana-volume
+        - emptyDir: {}
+          name: dashboard-volume
+        - emptyDir: {}
+          name: datasource-volume
+{{- end }}

charts/user-monitoring/templates/_user-grafana-ingress.yaml

@@ -0,0 +1,28 @@
+{{- define "grafana.ingress" }}
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    haproxy.org/auth-secret: basic-auth
+    haproxy.org/auth-type: basic-auth
+    haproxy.org/auth-realm: Authentication Required
+  name: grafana
+spec:
+  ingressClassName: haproxy
+  rules:
+  - host: {{ .Values.user }}-grafana.training.cluster.acend.ch
+    http:
+      paths:
+      - backend:
+          service:
+            name: grafana
+            port:
+              number: 3000
+        path: /
+        pathType: ImplementationSpecific
+  tls:
+  - hosts:
+    - {{ .Values.user }}-grafana.training.cluster.acend.ch
+    secretName: acend-wildcard
+{{- end }}

charts/user-monitoring/templates/_user-grafana-rolebinding.yaml

@@ -0,0 +1,16 @@
+{{- define "grafana.rolebinding" }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/name: grafana
+  name: grafana
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-prometheus-stack-grafana-clusterrole
+subjects:
+- kind: ServiceAccount
+  name: grafana
+{{- end }}

charts/user-monitoring/templates/_user-grafana-sa.yaml

@@ -0,0 +1,9 @@
+{{- define "grafana.sa" }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/name: grafana
+  name: grafana
+{{- end }}

charts/user-monitoring/templates/_user-grafana-servicemonitor.yaml

@@ -0,0 +1,23 @@
+{{- define "grafana.servicemonitor" }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: grafana-monitor
+spec:
+  endpoints:
+    - basicAuth:
+        password:
+          name: basic-auth
+          key: {{ .Values.user }}
+        username:
+          name: basic-auth
+          key: grafana_user
+      interval: 60s
+      port: http
+      scheme: http
+      path: /metrics
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: grafana
+{{- end }}

charts/user-monitoring/templates/_user-grafana-svc.yaml

@@ -0,0 +1,18 @@
+{{- define "grafana.svc" }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana
+  labels:
+    app.kubernetes.io/name: grafana
+spec:
+  ports:
+    - name: http
+      port: 3000
+      protocol: TCP
+      targetPort: 3000
+  selector:
+    app.kubernetes.io/name: grafana
+  type: ClusterIP
+{{- end }}

charts/user-monitoring/templates/_user-mailcatcher-deploy.yaml

@@ -0,0 +1,28 @@
+{{- define "mailcatcher.deploy" }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: mailcatcher
+  name: mailcatcher
+  namespace: {{ .Values.user }}-monitoring
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: mailcatcher
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: mailcatcher
+    spec:
+      containers:
+      - image: sj26/mailcatcher:v0.9.0
+        name: mailcatcher
+        ports:
+        - containerPort: 1080
+          name: http
+        - containerPort: 1025
+          name: smtp
+{{- end }}

charts/user-monitoring/templates/_user-mailcatcher-ingress.yaml

@@ -0,0 +1,30 @@
+{{- define "mailcatcher.ingress" }}
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    haproxy.org/auth-secret: basic-auth
+    haproxy.org/auth-type: basic-auth
+    haproxy.org/auth-realm: Authentication Required
+  labels:
+    app.kubernetes.io/name: {{ .Values.user }}-mailcatcher
+  name: {{ .Values.user }}-mailcatcher
+spec:
+  ingressClassName: haproxy
+  rules:
+  - host: {{ .Values.user }}-mailcatcher.training.cluster.acend.ch
+    http:
+      paths:
+      - backend:
+          service:
+            name: mailcatcher
+            port:
+              number: 1080
+        path: /
+        pathType: ImplementationSpecific
+  tls:
+  - hosts:
+    - {{ .Values.user }}-mailcatcher.training.cluster.acend.ch
+    secretName: acend-wildcard
+{{- end }}

charts/user-monitoring/templates/_user-mailcatcher-svc.yaml

@@ -0,0 +1,22 @@
+{{- define "mailcatcher.svc" }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mailcatcher
+  labels:
+    app.kubernetes.io/name: mailcatcher
+spec:
+  ports:
+  - name: http
+    port: 1080
+    protocol: TCP
+    targetPort: http
+  - name: smtp
+    port: 1025
+    protocol: TCP
+    targetPort: smtp
+  selector:
+    app.kubernetes.io/name: mailcatcher
+  type: ClusterIP
+{{- end }}

charts/user-monitoring/templates/_user-prom-ingress.yaml

@@ -1,3 +1,5 @@
+{{- define "prometheus.ingress" }}
+---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -5,6 +7,8 @@ metadata:
     haproxy.org/auth-secret: basic-auth
     haproxy.org/auth-type: basic-auth
     haproxy.org/auth-realm: Authentication Required
+  labels:
+    app.kubernetes.io/name: {{ .Values.user }}-prometheus
   name: {{ .Values.user }}-prometheus
 spec:
   ingressClassName: haproxy
@@ -23,3 +27,4 @@ spec:
   - hosts:
     - {{ .Values.user }}-prometheus.training.cluster.acend.ch
     secretName: acend-wildcard
+{{- end }}

charts/user-monitoring/templates/_user-prom-role.yaml

@@ -1,6 +1,10 @@
+{{- define "prometheus.role" }}
+---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
+  labels:
+    app.kubernetes.io/name: {{ .Values.user }}-prometheus
   name: prometheus-{{ .Values.user }}
 rules:
 - apiGroups: [""]
@@ -18,3 +22,4 @@ rules:
   resources:
   - ingresses
   verbs: ["get", "list", "watch"]
+{{- end }}

charts/user-monitoring/templates/_user-prom-rolebinding.yaml

@@ -1,6 +1,10 @@
+{{- define "prometheus.rolebinding" }}
+---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
+  labels:
+    app.kubernetes.io/name: prometheus
   name: prometheus
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -9,3 +13,4 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: prometheus-{{ .Values.user }}
+{{- end }}

charts/user-monitoring/templates/_user-prom-sa.yaml

@@ -0,0 +1,9 @@
+{{- define "prometheus.sa" }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/name: prometheus-{{ .Values.user }}
+  name: prometheus-{{ .Values.user }}
+{{- end }}

charts/user-monitoring/templates/_user-prometheus-servicemonitor.yaml

@@ -0,0 +1,20 @@
+{{- define "prometheus.servicemonitor" }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/name: prometheus
+    user: {{ .Values.user }}
+  name: {{ .Values.user }}-prometheus-servicemonitor
+spec:
+  endpoints:
+  - path: /metrics
+    port: web
+  namespaceSelector:
+    matchNames:
+    - {{ .Values.user }}-monitoring
+  selector:
+    matchLabels:
+      operated-prometheus: 'true'
+{{- end }}

charts/user-monitoring/templates/_user-prometheus.yaml

@@ -0,0 +1,34 @@
+{{- define "prometheus.prometheus" }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: Prometheus
+metadata:
+  labels:
+    app.kubernetes.io/name: prometheus
+  name: prometheus
+spec:
+  {{- if .Values.query.enabled }}
+  thanos:
+    image: quay.io/thanos/thanos:v0.32.3
+  {{- end }}
+  scrapeInterval: 60s
+  serviceAccountName: prometheus-{{ .Values.user }}
+  serviceMonitorNamespaceSelector:
+    matchLabels:
+      user: {{ .Values.user }}
+  serviceMonitorSelector: {}
+  podMonitorNamespaceSelector:
+    matchLabels:
+      user: {{ .Values.user }}
+  podMonitorSelector: {}
+  probeNamespaceSelector:
+    matchLabels:
+      user: {{ .Values.user }}
+  probeSelector: {}
+  resources:
+    requests:
+      memory: 400Mi
+  enableAdminAPI: true
+  externalLabels:
+    monitoring: {{ .Values.user }}
+{{- end }}

charts/user-monitoring/templates/_user-pushgateway-deploy.yaml

@@ -0,0 +1,33 @@
+{{- define "pushgateway.deploy" }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: pushgateway
+  name: pushgateway
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: pushgateway
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: pushgateway
+    spec:
+      containers:
+        - name: pushgateway
+          image: quay.io/prometheus/pushgateway:v1.6.0
+          ports:
+            - containerPort: 9091
+              name: web
+          resources: {}
+          livenessProbe:
+            httpGet:
+              path: /-/healthy
+              port: web
+          readinessProbe:
+            httpGet:
+              path: /-/ready
+              port: web
+{{- end }}

charts/user-monitoring/templates/_user-pushgateway-servicemonitor.yaml

@@ -0,0 +1,18 @@
+{{- define "pushgateway.servicemonitor" }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/name: pushgateway
+  name: pushgateway
+spec:
+  endpoints:
+    - interval: 30s
+      port: http
+      honorLabels: true
+  namespaceSelector: {}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: pushgateway
+{{- end }}

charts/user-monitoring/templates/_user-pushgateway-svc.yaml

@@ -0,0 +1,18 @@
+{{- define "pushgateway.svc" }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: pushgateway
+  labels:
+    app.kubernetes.io/name: pushgateway
+spec:
+  ports:
+    - name: http
+      port: 9091
+      protocol: TCP
+      targetPort: 9091
+  selector:
+    app.kubernetes.io/name: pushgateway
+  type: ClusterIP
+{{- end }}

charts/user-monitoring/templates/_user-thanos-query-deploy.yaml

@@ -0,0 +1,39 @@
+{{- define "query.deploy" }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: thanos-query
+  name: thanos-query
+  namespace: {{ .Values.user }}-monitoring
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: thanos-query
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: thanos-query
+    spec:
+      containers:
+      - args:
+        - query
+        - --log.level=debug
+        - --query.replica-label=prometheus_replica
+        - --query.replica-label=thanos_ruler_replica
+        - --endpoint=prometheus-operated.{{ .Values.user }}-monitoring.svc.cluster.local:10901
+        - --endpoint=thanos-ruler-operated.{{ .Values.user }}-monitoring.svc.cluster.local:10901
+        - --store=dnssrv+prometheus-operated.{{ .Values.user }}-monitoring.svc.cluster.local
+        - --store=dnssrv+_web._tcp.prometheus-operated.{{ .Values.user }}-monitoring.svc.cluster.local
+        - --store=dnssrv+_grpc._tcp.kube-prometheus-stack-thanos-discovery.monitoring.svc.cluster.local
+        - --store.unhealthy-timeout=3d
+        image: quay.io/thanos/thanos:v0.32.3
+        name: thanos-query
+        ports:
+        - containerPort: 10902
+          name: http
+        - containerPort: 10901
+          name: grpc
+{{- end }}

charts/user-monitoring/templates/_user-thanos-query-ingress.yaml

@@ -0,0 +1,30 @@
+{{- define "query.ingress" }}
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    haproxy.org/auth-secret: basic-auth
+    haproxy.org/auth-type: basic-auth
+    haproxy.org/auth-realm: Authentication Required
+  labels:
+    app.kubernetes.io/name: {{ .Values.user }}-thanos
+  name: {{ .Values.user }}-thanos
+spec:
+  ingressClassName: haproxy
+  rules:
+  - host: {{ .Values.user }}-thanos-query.training.cluster.acend.ch
+    http:
+      paths:
+      - backend:
+          service:
+            name: thanos-query
+            port:
+              number: 10902
+        path: /
+        pathType: ImplementationSpecific
+  tls:
+  - hosts:
+    - {{ .Values.user }}-thanos-query.training.cluster.acend.ch
+    secretName: acend-wildcard
+{{- end }}

charts/user-monitoring/templates/_user-thanos-query-svc.yaml

@@ -0,0 +1,18 @@
+{{- define "query.svc" }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: thanos-query
+  labels:
+    app.kubernetes.io/name: thanos-query
+spec:
+  ports:
+  - name: http
+    port: 10902
+    protocol: TCP
+    targetPort: http
+  selector:
+    app.kubernetes.io/name: thanos-query
+  type: ClusterIP
+{{- end }}

charts/user-monitoring/templates/_user-thanosruler-ingress.yaml

@@ -0,0 +1,28 @@
+{{- define "ruler.ingress" }}
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    haproxy.org/auth-secret: basic-auth
+    haproxy.org/auth-type: basic-auth
+    haproxy.org/auth-realm: Authentication Required
+  name: {{ .Values.user }}-thanos-ruler
+spec:
+  ingressClassName: haproxy
+  rules:
+  - host: {{ .Values.user }}-thanos-ruler.training.cluster.acend.ch
+    http:
+      paths:
+      - backend:
+          service:
+            name: thanos-ruler-operated
+            port:
+              number: 10902
+        path: /
+        pathType: ImplementationSpecific
+  tls:
+  - hosts:
+    - {{ .Values.user }}-thanos-ruler.training.cluster.acend.ch
+    secretName: acend-wildcard
+{{- end }}

charts/user-monitoring/templates/_user-thanosruler.yaml

@@ -0,0 +1,32 @@
+{{- define "ruler.ruler" }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ThanosRuler
+metadata:
+  labels:
+    app.kubernetes.io/name: thanos-ruler
+  name: thanos-ruler
+spec:
+  image: quay.io/thanos/thanos:v0.28.1
+  evaluationInterval: 10s
+  queryEndpoints:
+  - dnssrv+_http._tcp.thanos-query:10902
+  ruleSelector: {}
+  ruleNamespaceSelector:
+    matchLabels:
+      user: {{ .Values.user }}
+  alertmanagersConfig:
+    key: alertmanager-configs.yaml
+    name: thanosruler-alertmanager-config
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: thanosruler-alertmanager-config
+stringData:
+  alertmanager-configs.yaml: |-
+    alertmanagers:
+    - static_configs:
+      - "dnssrv+_web._tcp.alertmanager-operated.{{ .Values.user }}-monitoring.svc.cluster.local"
+      api_version: v2
+{{- end }}

charts/user-monitoring/templates/monitoring.yaml

@@ -0,0 +1,58 @@
+# alertmanager
+
+{{- if .Values.alertmanager.enabled }}
+  {{- template "alertmanager.alertmanager" . }}
+  {{- template "alertmanager.servicemonitor" . }}
+  {{- template "alertmanager.config-mailcatcher" . }}
+  {{- template "mailcatcher.deploy" . }}
+  {{- template "mailcatcher.svc" . }}
+  {{- template "mailcatcher.ingress" . }} 
+{{- end }}
+
+# blackboxexporter
+{{- if .Values.blackboxexporter.enabled }}
+  {{- template "blackboxexporter.cm" . }}
+  {{- template "blackboxexporter.deploy" . }}
+  {{- template "blackboxexporter.svc" . }}
+{{- end }}
+
+# grafana
+{{- if .Values.grafana.enabled }}
+  {{- template "grafana.datasource" . }}
+  {{- template "grafana.dashboardprovider" . }}
+  {{- template "grafana.rolebinding" . }}
+  {{- template "grafana.svc" . }}
+  {{- template "grafana.sa" . }}
+  {{- template "grafana.deploy" . }}
+  {{- template "grafana.servicemonitor" . }}
+  {{- template "grafana.ingress" . }}
+{{- end }}
+
+# prometheus
+{{- if .Values.prometheus.enabled }}
+  {{- template "prometheus.ingress" . }}
+  {{- template "prometheus.prometheus" . }}
+  {{- template "prometheus.servicemonitor" . }}
+  {{- template "prometheus.role" . }}
+  {{- template "prometheus.rolebinding" . }}
+  {{- template "prometheus.sa" . }}
+{{- end }}
+
+{{- if .Values.pushgateway.enabled }}
+  {{- template "pushgateway.deploy" . }}
+  {{- template "pushgateway.servicemonitor" . }}
+  {{- template "pushgateway.svc" . }}
+{{- end }}
+
+# thanos-ruler
+{{- if .Values.ruler.enabled }}
+  {{- template "ruler.ruler" . }}
+  {{- template "ruler.ingress" . }}
+{{- end }}
+
+# thanos-query
+{{- if .Values.query.enabled }}
+  {{- template "query.deploy" . }}
+  {{- template "query.ingress" . }}
+  {{- template "query.svc" . }}
+{{- end }}

charts/user-monitoring/templates/user-basic-auth-secret.yaml

@@ -2,6 +2,9 @@ apiVersion: v1
 data:
   admin: JDEkSzNRZUlsOHAkZVZZb1h2RERNWk40b3RJT2tBc3AwLw== # $(openssl passwd -1 admin)
   {{ .Values.user }}: JDEkSzNRZUlsOHAkZVZZb1h2RERNWk40b3RJT2tBc3AwLw== # $(openssl passwd -1 admin)
+  grafana_user: {{ .Values.user | b64enc }}
 kind: Secret
 metadata:
+  labels:
+    app.kubernetes.io/name: basic-auth
   name: basic-auth

charts/user-monitoring/values.yaml

@@ -0,0 +1,22 @@
+user: <user> # Replace me
+# prometheus
+prometheus:
+  enabled: false
+# thanos-query
+query:
+  enabled: false
+# grafana
+grafana:
+  enabled: false
+# blackboxexporter
+blackboxexporter:
+  enabled: false
+# pushgateway
+pushgateway:
+  enabled: false
+# alertmanager
+alertmanager:
+  enabled: false
+# thanos-ruler
+ruler:
+  enabled: false

index.yaml

@@ -1,159 +0,0 @@
-apiVersion: v1
-entries:
-  user-monitoring:
-  - apiVersion: v2
-    appVersion: 1.16.0
-    created: "2023-10-11T10:31:18.211680801Z"
-    description: Acend helm chart for user-monitoring namespace in prometheus-basics
-      training
-    digest: 653edb52c867fc63534550453d41fa6eb4acbf7e50f2f94d553ef35770608682
-    name: user-monitoring
-    type: application
-    urls:
-    - https://github.com/acend/prometheus-training-lab-setup/releases/download/user-monitoring-0.2.5/user-monitoring-0.2.5.tgz
-    version: 0.2.5
-  - apiVersion: v2
-    appVersion: 1.16.0
-    created: "2023-10-10T12:14:33.841977384Z"
-    description: Acend helm chart for user-monitoring namespace in prometheus-basics
-      training
-    digest: e9c8921209cd4967751bfbecb5306207fd13ca279d6ebd17f2c8dc3d01ead246
-    name: user-monitoring
-    type: application
-    urls:
-    - https://github.com/acend/prometheus-training-lab-setup/releases/download/user-monitoring-0.2.4/user-monitoring-0.2.4.tgz
-    version: 0.2.4
-  - apiVersion: v2
-    appVersion: 1.16.0
-    created: "2023-10-05T11:51:33.421052727Z"
-    description: Acend helm chart for user-monitoring namespace in prometheus-basics
-      training
-    digest: a4043d6b93ee4aaef394e9e66ab806b11ec8cf1665b534b85d7ee1584fac2eb9
-    name: user-monitoring
-    type: application
-    urls:
-    - https://github.com/acend/prometheus-training-lab-setup/releases/download/user-monitoring-0.2.3/user-monitoring-0.2.3.tgz
-    version: 0.2.3
-  - apiVersion: v2
-    appVersion: 1.16.0
-    created: "2023-08-10T14:08:08.237621347Z"
-    description: Acend helm chart for user-monitoring namespace in prometheus-basics
-      training
-    digest: 811a3ebdf1886d438310f4baba9d83a63d3b774149bb921010272b6a7ff87000
-    name: user-monitoring
-    type: application
-    urls:
-    - https://github.com/acend/prometheus-training-lab-setup/releases/download/user-monitoring-0.2.2/user-monitoring-0.2.2.tgz
-    version: 0.2.2
-  - apiVersion: v2
-    appVersion: 1.16.0
-    created: "2023-08-10T13:29:23.701351307Z"
-    description: Acend helm chart for user-monitoring namespace in prometheus-basics
-      training
-    digest: 9000d00432face8c8ec8f5d81d35e1b857ad7abb4459b25136a47b13320edea9
-    name: user-monitoring
-    type: application
-    urls:
-    - https://github.com/acend/prometheus-training-lab-setup/releases/download/user-monitoring-0.2.1/user-monitoring-0.2.1.tgz
-    version: 0.2.1
-  - apiVersion: v2
-    appVersion: 1.16.0
-    created: "2023-08-10T12:52:25.233417672Z"
-    description: Acend helm chart for user-monitoring namespace in prometheus-basics
-      training
-    digest: b692cc3be47716c33446bc829ff633725b7772e93b4cfd1529159aad69aed17a
-    name: user-monitoring
-    type: application
-    urls:
-    - https://github.com/acend/prometheus-training-lab-setup/releases/download/user-monitoring-0.2.0/user-monitoring-0.2.0.tgz
-    version: 0.2.0
-  - apiVersion: v2
-    appVersion: 1.16.0
-    created: "2023-08-10T11:47:27.251949946Z"
-    description: Acend helm chart for user-monitoring namespace in prometheus-basics
-      training
-    digest: e6ab40f23dede3ce432396cf5f094e95c9796f1f747679fae3af84835aa1c2b9
-    name: user-monitoring
-    type: application
-    urls:
-    - https://github.com/acend/prometheus-training-lab-setup/releases/download/user-monitoring-0.1.5/user-monitoring-0.1.5.tgz
-    version: 0.1.5
-  - apiVersion: v2
-    appVersion: 1.16.0
-    created: "2023-08-10T11:38:39.424718245Z"
-    description: Acend helm chart for user-monitoring namespace in prometheus-basics
-      training
-    digest: a9c733d0670e94a5230d679d4944208032a890cc2f06bd696f53c3c53e4e05ca
-    name: user-monitoring
-    type: application
-    urls:
-    - https://github.com/acend/prometheus-training-lab-setup/releases/download/user-monitoring-0.1.4/user-monitoring-0.1.4.tgz
-    version: 0.1.4
-  - apiVersion: v2
-    appVersion: 1.16.0
-    created: "2023-08-10T08:30:26.414647424Z"
-    description: Acend helm chart for user-monitoring namespace in prometheus-basics
-      training
-    digest: 0ad23a4686389ba739321f83ff5fe2e3d4a00403b623557e429892cf1dc16aa9
-    name: user-monitoring
-    type: application
-    urls:
-    - https://github.com/acend/prometheus-training-lab-setup/releases/download/user-monitoring-0.1.3/user-monitoring-0.1.3.tgz
-    version: 0.1.3
-  - apiVersion: v2
-    appVersion: 1.16.0
-    created: "2023-08-10T08:25:11.692388913Z"
-    description: Acend helm chart for user-monitoring namespace in prometheus-basics
-      training
-    digest: 1431107944815c981912977fa510b57b8b7459fc5bd36a4f3cff03e1862a0ebc
-    name: user-monitoring
-    type: application
-    urls:
-    - https://github.com/acend/prometheus-training-lab-setup/releases/download/user-monitoring-0.1.2/user-monitoring-0.1.2.tgz
-    version: 0.1.2
-  - apiVersion: v2
-    appVersion: 1.16.0
-    created: "2023-08-09T13:21:29.318894826Z"
-    description: Acend helm chart for user-monitoring namespace in prometheus-basics
-      training
-    digest: 873fdf9b72bf5a423d424ec34f8f2f1d28a44422d8b96480ca873a9fe244311c
-    name: user-monitoring
-    type: application
-    urls:
-    - https://github.com/acend/prometheus-training-lab-setup/releases/download/user-monitoring-0.1.1/user-monitoring-0.1.1.tgz
-    version: 0.1.1
-  - apiVersion: v2
-    appVersion: 1.16.0
-    created: "2023-08-09T12:51:03.727644492Z"
-    description: Acend helm chart for user-monitoring namespace in prometheus-basics
-      training
-    digest: e029cf3ce9eb4cf28258223e3a95723852c2c72e89e9c26e7c7342c0645c06fa
-    name: user-monitoring
-    type: application
-    urls:
-    - https://github.com/acend/prometheus-training-lab-setup/releases/download/user-monitoring-0.1.0/user-monitoring-0.1.0.tgz
-    version: 0.1.0
-  user-monitoring-demo:
-  - apiVersion: v2
-    appVersion: 1.16.0
-    created: "2023-10-05T11:51:33.69338868Z"
-    description: Demo chart for user spaced monitoring stack in prometheus basics
-      training.
-    digest: 2d175d6f134994e79a84607039b2c2211a33ce1169082bcc6046e3770936944c
-    name: user-monitoring-demo
-    type: application
-    urls:
-    - https://github.com/acend/prometheus-training-lab-setup/releases/download/user-monitoring-demo-0.1.1/user-monitoring-demo-0.1.1.tgz
-    version: 0.1.1
-  - apiVersion: v2
-    appVersion: 1.16.0
-    created: "2023-08-09T12:51:03.914020183Z"
-    description: Demo chart for user spaced monitoring stack in prometheus basics
-      training.
-    digest: 5e327930bbe335c7f9616c255616edd101e1d5f277c836474976590248ff06ef
-    name: user-monitoring-demo
-    type: application
-    urls:
-    - https://github.com/acend/prometheus-training-lab-setup/releases/download/user-monitoring-demo-0.1.0/user-monitoring-demo-0.1.0.tgz
-    version: 0.1.0
-generated: "2023-10-11T10:31:18.211875514Z"

user-demo/README.md

@@ -0,0 +1,3 @@
+# Readme
+
+This is the location for the user demo app.

user-monitoring-demo/values.yaml

@@ -1 +0,0 @@
-user: user1

user-monitoring/templates/user-prom-role.yaml

@@ -1,20 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: prometheus-{{ .Values.user }}
-rules:
-- apiGroups: [""]
-  resources:
-  - services
-  - endpoints
-  - pods
-  verbs: ["get", "list", "watch"]
-- apiGroups: [""]
-  resources:
-  - configmaps
-  verbs: ["get"]
-- apiGroups:
-  - networking.k8s.io
-  resources:
-  - ingresses
-  verbs: ["get", "list", "watch"]

user-monitoring/templates/user-prom-sa.yaml

@@ -1,5 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: prometheus-{{ .Values.user }}
-

user-monitoring/templates/user-prometheus.yaml

@@ -1,22 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: Prometheus
-metadata:
-  name: prometheus
-spec:
-  serviceAccountName: prometheus-{{ .Values.user }}
-  serviceMonitorNamespaceSelector:
-    matchLabels:
-      user: {{ .Values.user }}
-  serviceMonitorSelector:
-    matchLabels:
-      user: {{ .Values.user }}
-  resources:
-    requests:
-      memory: 400Mi
-  enableAdminAPI: false
-  storage:
-    volumeClaimTemplate:
-      spec:
-        resources:
-          requests:
-            storage: 2Gi

user-monitoring/values.yaml

@@ -1 +0,0 @@
-user: user1