Merge pull request #2782 from fluxcd/monitoring-refactoring

Refactor Flux Prometheus monitoring stack

manifests/monitoring/grafana/datasources.yaml

@@ -1,18 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: grafana-datasources
-  namespace: flux-system
-data:
-  datasources.yaml: |-
-    apiVersion: 1
-    deleteDatasources:
-      - name: prometheus
-    datasources:
-    - name: prometheus
-      type: prometheus
-      access: proxy
-      url: http://prometheus:9090
-      isDefault: true
-      editable: true
-      version: 1

manifests/monitoring/grafana/deployment.yaml

@@ -1,60 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: grafana
-  labels:
-    app: grafana
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: grafana
-  template:
-    metadata:
-      labels:
-        app: grafana
-      annotations:
-        prometheus.io/scrape: 'false'
-    spec:
-      containers:
-        - name: grafana
-          image: "grafana/grafana:7.5.4"
-          imagePullPolicy: IfNotPresent
-          ports:
-            - name: http
-              containerPort: 3000
-              protocol: TCP
-          env:
-            - name: GF_PATHS_PROVISIONING
-              value: /etc/grafana/provisioning/
-            - name: GF_AUTH_BASIC_ENABLED
-              value: "false"
-            - name: GF_AUTH_ANONYMOUS_ENABLED
-              value: "true"
-            - name: GF_AUTH_ANONYMOUS_ORG_ROLE
-              value: Admin
-            - name: GF_USERS_DEFAULT_THEME
-              value: "light"
-          volumeMounts:
-            - name: grafana
-              mountPath: /var/lib/grafana
-            - name: dashboards
-              mountPath: /etc/grafana/dashboards
-            - name: datasources
-              mountPath: /etc/grafana/provisioning/datasources
-            - name: providers
-              mountPath: /etc/grafana/provisioning/dashboards
-          resources:
-            {}
-      volumes:
-        - name: grafana
-          emptyDir: {}
-        - name: dashboards
-          configMap:
-            name: grafana-dashboards
-        - name: providers
-          configMap:
-            name: grafana-providers
-        - name: datasources
-          configMap:
-            name: grafana-datasources

manifests/monitoring/grafana/kustomization.yaml

@@ -1,14 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: flux-system
-resources:
-  - service.yaml
-  - deployment.yaml
-  - providers.yaml
-  - datasources.yaml
-configMapGenerator:
-  - name: grafana-dashboards
-    files:
-      - dashboards/control-plane.json
-      - dashboards/cluster.json
-

manifests/monitoring/grafana/providers.yaml

@@ -1,17 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: grafana-providers
-  namespace: flux-system
-data:
-  providers.yaml: |+
-    apiVersion: 1
-    providers:
-    - name: 'default'
-      orgId: 1
-      folder: ''
-      type: file
-      disableDeletion: false
-      editable: true
-      options:
-        path: /etc/grafana/dashboards

manifests/monitoring/grafana/service.yaml

@@ -1,16 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: grafana
-  namespace: flux-system
-  labels:
-    app: grafana
-spec:
-  type: ClusterIP
-  ports:
-    - port: 3000
-      targetPort: http
-      protocol: TCP
-      name: http
-  selector:
-    app: grafana

manifests/monitoring/kube-prometheus-stack/namespace.yaml

@@ -2,3 +2,5 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: monitoring
+  labels:
+    app.kubernetes.io/component: monitoring

manifests/monitoring/kube-prometheus-stack/release.yaml

@@ -6,25 +6,28 @@ spec:
   interval: 5m
   chart:
     spec:
-      version: 34.7.0
+      version: "35.x"
       chart: kube-prometheus-stack
       sourceRef:
         kind: HelmRepository
         name: prometheus-community
-      interval: 1m
+      interval: 60m
   install:
     crds: Create
   upgrade:
     crds: CreateReplace
+  # https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml
   values:
     alertmanager:
       enabled: false
-    grafana:
-      sidecar:
-        dashboards:
-          searchNamespace: ALL
     prometheus:
       prometheusSpec:
+        retention: 24h
+        resources:
+          requests:
+            cpu: 200m
+            memory: 200Mi
+        podMonitorNamespaceSelector: {}
         podMonitorSelector:
           matchLabels:
-            app.kubernetes.io/part-of: flux
+            app.kubernetes.io/component: monitoring

manifests/monitoring/kube-prometheus-stack/repository.yaml

@@ -1,7 +1,7 @@
-apiVersion: source.toolkit.fluxcd.io/v1beta1
+apiVersion: source.toolkit.fluxcd.io/v1beta2
 kind: HelmRepository
 metadata:
   name: prometheus-community
 spec:
-  interval: 1m
+  interval: 120m
   url: https://prometheus-community.github.io/helm-charts

manifests/monitoring/kustomization.yaml

@@ -1,6 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: flux-system
-resources:
-- prometheus
-- grafana

manifests/monitoring/monitoring-config/dashboards/cluster.json

@@ -9,6 +9,23 @@
         "iconColor": "rgba(0, 211, 255, 1)",
         "name": "Annotations & Alerts",
         "type": "dashboard"
+      },
+      {
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "iconColor": "red",
+        "name": "flux events",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [
+            "flux"
+          ],
+          "type": "tags"
+        }
       }
     ]
   },

manifests/monitoring/monitoring-config/dashboards/control-plane.json

@@ -15,6 +15,23 @@
           "type": "dashboard"
         },
         "type": "dashboard"
+      },
+      {
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "iconColor": "red",
+        "name": "flux events",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [
+            "flux"
+          ],
+          "type": "tags"
+        }
       }
     ]
   },

manifests/monitoring/monitoring-config/kustomization.yaml

@@ -6,8 +6,10 @@ resources:
 configMapGenerator:
   - name: flux-grafana-dashboards
     files:
-      - ../grafana/dashboards/control-plane.json
+      - dashboards/control-plane.json
-      - ../grafana/dashboards/cluster.json
+      - dashboards/cluster.json
     options:
       labels:
         grafana_dashboard: "1"
+        app.kubernetes.io/part-of: flux
+        app.kubernetes.io/component: monitoring

manifests/monitoring/monitoring-config/podmonitor.yaml

@@ -5,6 +5,7 @@ metadata:
   namespace: flux-system
   labels:
     app.kubernetes.io/part-of: flux
+    app.kubernetes.io/component: monitoring
 spec:
   namespaceSelector:
     matchNames:

manifests/monitoring/prometheus/account.yaml

@@ -1,5 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: prometheus
-  namespace: flux-system

manifests/monitoring/prometheus/deployment.yaml

@@ -1,52 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: prometheus
-  namespace: flux-system
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: prometheus
-  template:
-    metadata:
-      labels:
-        app: prometheus
-      annotations:
-        appmesh.k8s.aws/sidecarInjectorWebhook: disabled
-        sidecar.istio.io/inject: "false"
-    spec:
-      serviceAccountName: prometheus
-      containers:
-          - name: prometheus
-            image: prom/prometheus:v2.26.0
-            imagePullPolicy: IfNotPresent
-            args:
-              - '--storage.tsdb.retention=2h'
-              - '--config.file=/etc/prometheus/prometheus.yml'
-            ports:
-              - containerPort: 9090
-                name: http
-            livenessProbe:
-              httpGet:
-                path: /-/healthy
-                port: 9090
-            readinessProbe:
-              httpGet:
-                path: /-/ready
-                port: 9090
-            resources:
-              requests:
-                cpu: 10m
-                memory: 128Mi
-            volumeMounts:
-              - name: config-volume
-                mountPath: /etc/prometheus
-              - name: data-volume
-                mountPath: /prometheus/data
-      volumes:
-        - name: config-volume
-          configMap:
-            name: prometheus
-        - name: data-volume
-          emptyDir: {}

manifests/monitoring/prometheus/kustomization.yaml

@@ -1,12 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: flux-system
-resources:
-  - account.yaml
-  - rbac.yaml
-  - service.yaml
-  - deployment.yaml
-configMapGenerator:
-  - name: prometheus
-    files:
-      - prometheus.yml

manifests/monitoring/prometheus/prometheus.yml

@@ -1,52 +0,0 @@
-global:
-  scrape_interval: 10s
-scrape_configs:
-
-# Kubernetes API
-- job_name: kubernetes-apiserver
-  kubernetes_sd_configs:
-  - role: endpoints
-    namespaces:
-      names:
-      - default
-  scheme: https
-  tls_config:
-    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-    insecure_skip_verify: true
-  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-  relabel_configs:
-  - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
-    action: keep
-    regex: kubernetes;https
-
-# Kubernetes pods
-- job_name: kubernetes-pods
-  kubernetes_sd_configs:
-  - role: pod
-  relabel_configs:
-  - action: keep
-    regex: true
-    source_labels:
-    - __meta_kubernetes_pod_annotation_prometheus_io_scrape
-  - action: replace
-    regex: (.+)
-    source_labels:
-    - __meta_kubernetes_pod_annotation_prometheus_io_path
-    target_label: __metrics_path__
-  - action: replace
-    regex: ([^:]+)(?::\d+)?;(\d+)
-    replacement: $1:$2
-    source_labels:
-    - __address__
-    - __meta_kubernetes_pod_annotation_prometheus_io_port
-    target_label: __address__
-  - action: labelmap
-    regex: __meta_kubernetes_pod_label_(.+)
-  - action: replace
-    source_labels:
-    - __meta_kubernetes_namespace
-    target_label: namespace
-  - action: replace
-    source_labels:
-    - __meta_kubernetes_pod_name
-    target_label: pod

manifests/monitoring/prometheus/rbac.yaml

@@ -1,32 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: prometheus-flux-system
-rules:
-  - apiGroups: [""]
-    resources:
-      - nodes
-      - services
-      - endpoints
-      - pods
-      - nodes/proxy
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources:
-      - configmaps
-    verbs: ["get"]
-  - nonResourceURLs: ["/metrics"]
-    verbs: ["get"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: prometheus-flux-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: prometheus-flux-system
-subjects:
-- kind: ServiceAccount
-  name: prometheus
-  namespace: flux-system

manifests/monitoring/prometheus/service.yaml

@@ -1,12 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: prometheus
-  namespace: flux-system
-spec:
-  selector:
-    app: prometheus
-  ports:
-    - name: http
-      protocol: TCP
-      port: 9090