diff --git a/docs/guides/installation.md b/docs/guides/installation.md index b03d8173..5bc27718 100644 --- a/docs/guides/installation.md +++ b/docs/guides/installation.md @@ -345,44 +345,66 @@ Then you can register Helm repositories and create Helm releases: ```sh gotk create source helm stable \ ---interval=1h \ ---url=https://kubernetes-charts.storage.googleapis.com + --interval=1h \ + --url=https://kubernetes-charts.storage.googleapis.com gotk create helmrelease sealed-secrets \ ---interval=1h \ ---release-name=sealed-secrets \ ---target-namespace=gotk-system \ ---source=HelmRepository/stable \ ---chart=sealed-secrets \ ---chart-version="1.10.x" + --interval=1h \ + --release-name=sealed-secrets \ + --target-namespace=gotk-system \ + --source=HelmRepository/stable \ + --chart=sealed-secrets \ + --chart-version="1.10.x" ``` ## Monitoring with Prometheus and Grafana -The GitOps Toolkit comes with an optional monitoring stack. -You can install the stack in the `gotk-system` namespace with: +The GitOps Toolkit comes with a monitoring stack composed of: -```yaml -kustomize build github.com/fluxcd/toolkit/manifests/monitoring | kubectl apply -f- +* **Prometheus** server - collects metrics from the toolkit controllers and stores them for 2h +* **Grafana** dashboards - displays the control plane resource usage and reconciliation stats + +To install the monitoring stack with `gotk`, first register the toolkit Git repository on your cluster: + +```sh +gotk create source git monitoring \ + --interval=30m \ + --url=https://github.com/fluxcd/toolkit \ + --branch=main ``` -The monitoring stack is composed of: +Then apply the [manifests/monitoring](https://github.com/fluxcd/toolkit/tree/main/manifests/monitoring) +kustomization: -* Prometheus server - collects metrics from the toolkit controllers and stores them for 2h -* Grafana dashboards - displays the control plane resource usage and reconciliation stats +```sh +gotk create kustomization monitoring \ + --interval=1h \ + --prune=true \ + --source=monitoring \ + --path="./manifests/monitoring" \ + --health-check="Deployment/prometheus.gotk-system" \ + --health-check="Deployment/grafana.gotk-system" +``` + +You can access Grafana using port forwarding: + +```sh +kubectl -n gotk-system port-forward svc/grafana 3000:3000 +``` + +Navigate to [http://localhost:3000/d/gitops-toolkit-control-plane](http://localhost:3000/d/gitops-toolkit-control-plane/gitops-toolkit-control-plane) +for the control plane dashboards: ![](../_files/cp-dashboard-p1.png) ![](../_files/cp-dashboard-p2.png) If you wish to use your own Prometheus and Grafana instances, then you can import the dashboards from -[GitHub](https://github.com/fluxcd/toolkit/tree/master/manifests/monitoring/grafana/dashboards). +[GitHub](https://github.com/fluxcd/toolkit/tree/main/manifests/monitoring/grafana/dashboards). !!! hint Note that the toolkit controllers expose the `/metrics` endpoint on port `8080`. When using Prometheus Operator you should create `PodMonitor` objects to configure scraping. - When Prometheus is running outside of the `gotk-system` namespace, you have to create a network policy - that allows traffic on port `8080` from the namespace where Prometheus is deployed. ## Uninstall diff --git a/manifests/monitoring/grafana/dashboards/control-plane.json b/manifests/monitoring/grafana/dashboards/control-plane.json index 6c830357..9535a70c 100644 --- a/manifests/monitoring/grafana/dashboards/control-plane.json +++ b/manifests/monitoring/grafana/dashboards/control-plane.json @@ -19,7 +19,7 @@ "links": [], "panels": [ { - "datasource": null, + "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { @@ -79,7 +79,7 @@ "type": "stat" }, { - "datasource": null, + "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { @@ -143,7 +143,7 @@ "type": "stat" }, { - "datasource": null, + "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { @@ -204,7 +204,7 @@ "type": "gauge" }, { - "datasource": null, + "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { @@ -269,7 +269,7 @@ }, { "collapsed": false, - "datasource": null, + "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, @@ -286,7 +286,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { @@ -397,7 +397,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": "${DS_PROMETHEUS}", "decimals": null, "description": "", "fieldConfig": { @@ -504,7 +504,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "custom": {} @@ -599,7 +599,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "custom": {} @@ -693,7 +693,7 @@ }, { "collapsed": false, - "datasource": null, + "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, @@ -710,7 +710,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "custom": {} @@ -807,7 +807,7 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "description": "", "fieldConfig": { @@ -913,7 +913,7 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "description": "", "fieldConfig": { @@ -1016,7 +1016,7 @@ }, { "collapsed": false, - "datasource": null, + "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, @@ -1033,7 +1033,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "custom": {} @@ -1144,7 +1144,7 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "description": "", "fieldConfig": { @@ -1250,7 +1250,7 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "description": "", "fieldConfig": { @@ -1360,6 +1360,24 @@ ], "templating": { "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { "allValue": null, "current": { @@ -1367,7 +1385,7 @@ "text": "gotk-system", "value": "gotk-system" }, - "datasource": "prometheus", + "datasource": "${DS_PROMETHEUS}", "definition": "workqueue_work_duration_seconds_count", "hide": 0, "includeAll": false, diff --git a/manifests/monitoring/grafana/deployment.yaml b/manifests/monitoring/grafana/deployment.yaml index 19c349a3..99d028ba 100644 --- a/manifests/monitoring/grafana/deployment.yaml +++ b/manifests/monitoring/grafana/deployment.yaml @@ -18,7 +18,7 @@ spec: spec: containers: - name: grafana - image: "grafana/grafana:7.1.1" + image: "grafana/grafana:7.2.1" imagePullPolicy: IfNotPresent ports: - name: http @@ -33,8 +33,8 @@ spec: value: "true" - name: GF_AUTH_ANONYMOUS_ORG_ROLE value: Admin - - name: GF_DEFAULT_THEME - value: "Light" + - name: GF_USERS_DEFAULT_THEME + value: "light" volumeMounts: - name: grafana mountPath: /var/lib/grafana diff --git a/manifests/monitoring/prometheus/deployment.yaml b/manifests/monitoring/prometheus/deployment.yaml index cc3596c0..34e70551 100644 --- a/manifests/monitoring/prometheus/deployment.yaml +++ b/manifests/monitoring/prometheus/deployment.yaml @@ -19,7 +19,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: prom/prometheus:v2.20.0 + image: prom/prometheus:v2.21.0 imagePullPolicy: IfNotPresent args: - '--storage.tsdb.retention=2h' diff --git a/manifests/policies/allow-scraping.yaml b/manifests/policies/allow-scraping.yaml new file mode 100644 index 00000000..0daf25a1 --- /dev/null +++ b/manifests/policies/allow-scraping.yaml @@ -0,0 +1,14 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-scraping +spec: + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: {} + ports: + - protocol: TCP + port: 8080 + podSelector: {} diff --git a/manifests/policies/allow-webhooks.yaml b/manifests/policies/allow-webhooks.yaml new file mode 100644 index 00000000..4f0622d4 --- /dev/null +++ b/manifests/policies/allow-webhooks.yaml @@ -0,0 +1,13 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-webhooks +spec: + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: {} + podSelector: + matchLabels: + app: notification-controller diff --git a/manifests/policies/deny-ingress.yaml b/manifests/policies/deny-ingress.yaml index 92550512..5577032b 100644 --- a/manifests/policies/deny-ingress.yaml +++ b/manifests/policies/deny-ingress.yaml @@ -3,9 +3,9 @@ kind: NetworkPolicy metadata: name: deny-ingress spec: - podSelector: {} policyTypes: - Ingress ingress: - from: - podSelector: {} + podSelector: {} diff --git a/manifests/policies/kustomization.yaml b/manifests/policies/kustomization.yaml index f535811d..6884c916 100644 --- a/manifests/policies/kustomization.yaml +++ b/manifests/policies/kustomization.yaml @@ -2,3 +2,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - deny-ingress.yaml + - allow-scraping.yaml + - allow-webhooks.yaml