Merge pull request #323 from fluxcd/monitoring

docs: Install the monitoring stack with gotk

docs/guides/installation.md

@@ -345,44 +345,66 @@ Then you can register Helm repositories and create Helm releases:
 
 ```sh
 gotk create source helm stable \
---interval=1h \
---url=https://kubernetes-charts.storage.googleapis.com
+  --interval=1h \
+  --url=https://kubernetes-charts.storage.googleapis.com
 
 gotk create helmrelease sealed-secrets \
---interval=1h \
---release-name=sealed-secrets \
---target-namespace=gotk-system \
---source=HelmRepository/stable \
---chart=sealed-secrets \
---chart-version="1.10.x"
+  --interval=1h \
+  --release-name=sealed-secrets \
+  --target-namespace=gotk-system \
+  --source=HelmRepository/stable \
+  --chart=sealed-secrets \
+  --chart-version="1.10.x"
 ```
 
 ## Monitoring with Prometheus and Grafana
 
-The GitOps Toolkit comes with an optional monitoring stack.
-You can install the stack in the `gotk-system` namespace with:
+The GitOps Toolkit comes with a monitoring stack composed of:
 
-```yaml
-kustomize build github.com/fluxcd/toolkit/manifests/monitoring | kubectl apply -f-
+* **Prometheus** server - collects metrics from the toolkit controllers and stores them for 2h
+* **Grafana** dashboards - displays the control plane resource usage and reconciliation stats
+
+To install the monitoring stack with `gotk`, first register the toolkit Git repository on your cluster:
+
+```sh
+gotk create source git monitoring \
+  --interval=30m \
+  --url=https://github.com/fluxcd/toolkit \
+  --branch=main
 ```
 
-The monitoring stack is composed of:
+Then apply the [manifests/monitoring](https://github.com/fluxcd/toolkit/tree/main/manifests/monitoring)
+kustomization:
 
-* Prometheus server - collects metrics from the toolkit controllers and stores them for 2h
-* Grafana dashboards - displays the control plane resource usage and reconciliation stats
+```sh
+gotk create kustomization monitoring \
+  --interval=1h \
+  --prune=true \
+  --source=monitoring \
+  --path="./manifests/monitoring" \
+  --health-check="Deployment/prometheus.gotk-system" \
+  --health-check="Deployment/grafana.gotk-system"
+```
+
+You can access Grafana using port forwarding:
+
+```sh
+kubectl -n gotk-system port-forward svc/grafana 3000:3000
+```
+
+Navigate to [http://localhost:3000/d/gitops-toolkit-control-plane](http://localhost:3000/d/gitops-toolkit-control-plane/gitops-toolkit-control-plane)
+for the control plane dashboards:
 
 ![](../_files/cp-dashboard-p1.png)
 
 ![](../_files/cp-dashboard-p2.png)
 
 If you wish to use your own Prometheus and Grafana instances, then you can import the dashboards from
-[GitHub](https://github.com/fluxcd/toolkit/tree/master/manifests/monitoring/grafana/dashboards).
+[GitHub](https://github.com/fluxcd/toolkit/tree/main/manifests/monitoring/grafana/dashboards).
 
 !!! hint
     Note that the toolkit controllers expose the `/metrics` endpoint on port `8080`.
     When using Prometheus Operator you should create `PodMonitor` objects to configure scraping.
-    When Prometheus is running outside of the `gotk-system` namespace, you have to create a network policy
-    that allows traffic on port `8080` from the namespace where Prometheus is deployed.
 
 ## Uninstall
 

manifests/monitoring/grafana/dashboards/control-plane.json

@@ -19,7 +19,7 @@
   "links": [],
   "panels": [
     {
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -79,7 +79,7 @@
       "type": "stat"
     },
     {
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -143,7 +143,7 @@
       "type": "stat"
     },
     {
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -204,7 +204,7 @@
       "type": "gauge"
     },
     {
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -269,7 +269,7 @@
     },
     {
       "collapsed": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -286,7 +286,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -397,7 +397,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "decimals": null,
       "description": "",
       "fieldConfig": {
@@ -504,7 +504,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "fieldConfig": {
         "defaults": {
           "custom": {}
@@ -599,7 +599,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "fieldConfig": {
         "defaults": {
           "custom": {}
@@ -693,7 +693,7 @@
     },
     {
       "collapsed": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -710,7 +710,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "fieldConfig": {
         "defaults": {
           "custom": {}
@@ -807,7 +807,7 @@
       "bars": true,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "decimals": 2,
       "description": "",
       "fieldConfig": {
@@ -913,7 +913,7 @@
       "bars": true,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "decimals": 2,
       "description": "",
       "fieldConfig": {
@@ -1016,7 +1016,7 @@
     },
     {
       "collapsed": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -1033,7 +1033,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "fieldConfig": {
         "defaults": {
           "custom": {}
@@ -1144,7 +1144,7 @@
       "bars": true,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "decimals": 2,
       "description": "",
       "fieldConfig": {
@@ -1250,7 +1250,7 @@
       "bars": true,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "decimals": 2,
       "description": "",
       "fieldConfig": {
@@ -1360,6 +1360,24 @@
   ],
   "templating": {
     "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "Prometheus",
+          "value": "Prometheus"
+        },
+        "hide": 2,
+        "includeAll": false,
+        "label": null,
+        "multi": false,
+        "name": "DS_PROMETHEUS",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
       {
         "allValue": null,
         "current": {
@@ -1367,7 +1385,7 @@
           "text": "gotk-system",
           "value": "gotk-system"
         },
-        "datasource": "prometheus",
+        "datasource": "${DS_PROMETHEUS}",
         "definition": "workqueue_work_duration_seconds_count",
         "hide": 0,
         "includeAll": false,

manifests/monitoring/grafana/deployment.yaml

@@ -18,7 +18,7 @@ spec:
     spec:
       containers:
         - name: grafana
-          image: "grafana/grafana:7.1.1"
+          image: "grafana/grafana:7.2.1"
           imagePullPolicy: IfNotPresent
           ports:
             - name: http
@@ -33,8 +33,8 @@ spec:
               value: "true"
             - name: GF_AUTH_ANONYMOUS_ORG_ROLE
               value: Admin
-            - name: GF_DEFAULT_THEME
-              value: "Light"
+            - name: GF_USERS_DEFAULT_THEME
+              value: "light"
           volumeMounts:
             - name: grafana
               mountPath: /var/lib/grafana

manifests/monitoring/prometheus/deployment.yaml

@@ -19,7 +19,7 @@ spec:
       serviceAccountName: prometheus
       containers:
           - name: prometheus
-            image: prom/prometheus:v2.20.0
+            image: prom/prometheus:v2.21.0
             imagePullPolicy: IfNotPresent
             args:
               - '--storage.tsdb.retention=2h'

manifests/policies/allow-scraping.yaml

@@ -0,0 +1,14 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-scraping
+spec:
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector: {}
+      ports:
+        - protocol: TCP
+          port: 8080
+  podSelector: {}

manifests/policies/allow-webhooks.yaml

@@ -0,0 +1,13 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-webhooks
+spec:
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector: {}
+  podSelector:
+    matchLabels:
+      app: notification-controller

manifests/policies/deny-ingress.yaml

@@ -3,9 +3,9 @@ kind: NetworkPolicy
 metadata:
   name: deny-ingress
 spec:
-  podSelector: {}
   policyTypes:
     - Ingress
   ingress:
   - from:
     - podSelector: {}
+  podSelector: {}

manifests/policies/kustomization.yaml

@@ -2,3 +2,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - deny-ingress.yaml
+  - allow-scraping.yaml
+  - allow-webhooks.yaml