Merge pull request #1300 from fluxcd/kustomize-bootstrap

Allow pre-bootstrap customisation of Flux components

docs/guides/installation.md

@@ -337,16 +337,23 @@ please see [fluxcd/terraform-provider-flux](https://github.com/fluxcd/terraform-
 
 ## Customize Flux manifests
 
-You can customize the Flux components in the Git repository where you've run bootstrap with Kustomize patches.
+You can customize the Flux components before or after running bootstrap.
 
-First clone the repository locally and generate a `kustomization.yaml` file with:
+Assuming you want to customise the Flux controllers before they get deployed on the cluster,
+first you'll need to create a Git repository and clone it locally.
+
+Create the file structure required by bootstrap with:
 
 ```sh
-cd ./clusters/production && kustomize create --autodetect
+mkdir -p clusters/my-cluster/flux-system
+touch clusters/my-cluster/flux-system/gotk-components.yaml \
+    clusters/my-cluster/flux-system/gotk-patches.yaml \
+    clusters/my-cluster/flux-system/gotk-sync.yaml \
+    clusters/my-cluster/flux-system/kustomization.yaml
 ```
 
-Assuming you want to add custom annotations and labels to the Flux controllers in `clusters/production`.
+Assuming you want to add custom annotations and labels to the Flux controllers,
-Create a Kustomize patch and set the metadata for source-controller and kustomize-controller pods:
+edit `clusters/my-cluster/gotk-patches.yaml` and set the metadata for source-controller and kustomize-controller pods:
 
 ```yaml
 apiVersion: apps/v1
@@ -376,26 +383,37 @@ spec:
         custom: label
 ```
 
-Save the above file as `flux-system-patch.yaml` inside the `clusters/production` dir.
+Edit `clusters/my-cluster/kustomization.yaml` and set the resources and patches:
-
-Edit `clusters/production/kustomization.yaml` and add the patch:
 
 ```yaml
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - flux-system
+  - gotk-components.yaml
+  - gotk-sync.yaml
 patchesStrategicMerge:
-  - flux-system-patch.yaml
+  - gotk-patches.yaml
 ```
 
 Push the changes to main branch:
 
 ```sh
-git add -A && git commit -m "add production metadata" && git push
+git add -A && git commit -m "add flux customisations" && git push
 ```
 
-Flux will detect the change and will update itself on the production cluster.
+Now run the bootstrap for `clusters/my-cluster`:
+
+```sh
+flux bootstrap git \
+  --url=ssh://git@<host>/<org>/<repository> \
+  --branch=main \
+  --path=clusters/my-cluster
+```
+
+When the controllers are deployed for the first time on your cluster, they will contain all
+the customisations from `gotk-patches.yaml`.
+
+You can make changes to the patches after bootstrap and Flux will apply them in-cluster on its own.
 
 ## Dev install
 

internal/bootstrap/bootstrap_plain_git.go

@@ -19,6 +19,8 @@ package bootstrap
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
+	"os"
 	"path/filepath"
 	"strings"
 	"time"
@@ -29,6 +31,7 @@ import (
 	"sigs.k8s.io/cli-utils/pkg/object"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/kustomize/api/filesys"
+	"sigs.k8s.io/kustomize/api/konfig"
 	"sigs.k8s.io/yaml"
 
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1"
@@ -152,10 +155,48 @@ func (b *PlainGitBootstrapper) ReconcileComponents(ctx context.Context, manifest
 
 	// Conditionally install manifests
 	if mustInstallManifests(ctx, b.kube, options.Namespace) {
-		b.logger.Actionf("installing components in %q namespace", options.Namespace)
+		componentsYAML := filepath.Join(b.git.Path(), manifests.Path)
-		kubectlArgs := []string{"apply", "-f", filepath.Join(b.git.Path(), manifests.Path)}
+
-		if _, err = utils.ExecKubectlCommand(ctx, utils.ModeStderrOS, b.kubeconfig, b.kubecontext, kubectlArgs...); err != nil {
+		// Apply components using any existing customisations
-			return err
+		kfile := filepath.Join(filepath.Dir(componentsYAML), konfig.DefaultKustomizationFileName())
+		if _, err := os.Stat(kfile); err == nil {
+			tmpDir, err := ioutil.TempDir("", "gotk-crds")
+			defer os.RemoveAll(tmpDir)
+
+			// Extract the CRDs from the components manifest
+			crdsYAML := filepath.Join(tmpDir, "gotk-crds.yaml")
+			if err := utils.ExtractCRDs(componentsYAML, crdsYAML); err != nil {
+				return err
+			}
+
+			// Apply the CRDs
+			b.logger.Actionf("installing toolkit.fluxcd.io CRDs")
+			kubectlArgs := []string{"apply", "-f", crdsYAML}
+			if _, err = utils.ExecKubectlCommand(ctx, utils.ModeStderrOS, b.kubeconfig, b.kubecontext, kubectlArgs...); err != nil {
+				return err
+			}
+
+			// Wait for CRDs to be established
+			b.logger.Waitingf("waiting for CRDs to be reconciled")
+			kubectlArgs = []string{"wait", "--for", "condition=established", "-f", crdsYAML}
+			if _, err = utils.ExecKubectlCommand(ctx, utils.ModeStderrOS, b.kubeconfig, b.kubecontext, kubectlArgs...); err != nil {
+				return err
+			}
+			b.logger.Successf("CRDs reconciled successfully")
+
+			// Apply the components and their patches
+			b.logger.Actionf("installing components in %q namespace", options.Namespace)
+			kubectlArgs = []string{"apply", "-k", filepath.Dir(componentsYAML)}
+			if _, err = utils.ExecKubectlCommand(ctx, utils.ModeStderrOS, b.kubeconfig, b.kubecontext, kubectlArgs...); err != nil {
+				return err
+			}
+		} else {
+			// Apply the CRDs and controllers
+			b.logger.Actionf("installing components in %q namespace", options.Namespace)
+			kubectlArgs := []string{"apply", "-f", componentsYAML}
+			if _, err = utils.ExecKubectlCommand(ctx, utils.ModeStderrOS, b.kubeconfig, b.kubecontext, kubectlArgs...); err != nil {
+				return err
+			}
 		}
 		b.logger.Successf("installed components")
 	}

internal/utils/utils.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -29,6 +30,20 @@ import (
 	"strings"
 	"text/template"
 
+	"github.com/olekukonko/tablewriter"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	apiruntime "k8s.io/apimachinery/pkg/runtime"
+	sigyaml "k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/yaml"
+
 	helmv2 "github.com/fluxcd/helm-controller/api/v2beta1"
 	imageautov1 "github.com/fluxcd/image-automation-controller/api/v1alpha2"
 	imagereflectv1 "github.com/fluxcd/image-reflector-controller/api/v1alpha2"
@@ -37,16 +52,6 @@ import (
 	"github.com/fluxcd/pkg/runtime/dependency"
 	"github.com/fluxcd/pkg/version"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
-	"github.com/olekukonko/tablewriter"
-	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
-	networkingv1 "k8s.io/api/networking/v1"
-	rbacv1 "k8s.io/api/rbac/v1"
-	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
-	apiruntime "k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/fluxcd/flux2/pkg/manifestgen/install"
 )
@@ -313,3 +318,41 @@ func CompatibleVersion(binary, target string) bool {
 	}
 	return binSv.Major() == targetSv.Major() && binSv.Minor() == targetSv.Minor()
 }
+
+func ExtractCRDs(inManifestPath, outManifestPath string) error {
+	manifests, err := ioutil.ReadFile(inManifestPath)
+	if err != nil {
+		return err
+	}
+
+	crds := ""
+	reader := sigyaml.NewYAMLOrJSONDecoder(bytes.NewReader(manifests), 2048)
+
+	for {
+		var obj unstructured.Unstructured
+		err := reader.Decode(&obj)
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return err
+		}
+
+		if obj.GetKind() == "CustomResourceDefinition" {
+			b, err := obj.MarshalJSON()
+			if err != nil {
+				return err
+			}
+			y, err := yaml.JSONToYAML(b)
+			if err != nil {
+				return err
+			}
+			crds += "---\n" + string(y)
+		}
+	}
+
+	if crds == "" {
+		return fmt.Errorf("no CRDs found in %s", inManifestPath)
+	}
+
+	return ioutil.WriteFile(outManifestPath, []byte(crds), os.ModePerm)
+}