Merge pull request #376 from phillebaba/refactor-manifest-generation

Refactor manifest generation into a separate package

cmd/gotk/bootstrap.go

@@ -24,7 +24,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -33,7 +32,6 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/yaml"
 
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
@@ -41,6 +39,7 @@ import (
 	"github.com/fluxcd/toolkit/internal/flags"
 	"github.com/fluxcd/toolkit/internal/utils"
 	"github.com/fluxcd/toolkit/pkg/install"
+	"github.com/fluxcd/toolkit/pkg/sync"
 )
 
 var bootstrapCmd = &cobra.Command{
@@ -64,10 +63,7 @@ var (
 )
 
 const (
-	bootstrapDefaultBranch         = "main"
+	bootstrapDefaultBranch = "main"
-	bootstrapInstallManifest       = "toolkit-components.yaml"
-	bootstrapSourceManifest        = "toolkit-source.yaml"
-	bootstrapKustomizationManifest = "toolkit-kustomization.yaml"
 )
 
 func init() {
@@ -103,13 +99,6 @@ func bootstrapValidate() error {
 }
 
 func generateInstallManifests(targetPath, namespace, tmpDir string, localManifests string) (string, error) {
-	manifestsDir := path.Join(tmpDir, targetPath, namespace)
-	if err := os.MkdirAll(manifestsDir, os.ModePerm); err != nil {
-		return "", fmt.Errorf("creating manifests dir failed: %w", err)
-	}
-
-	manifest := path.Join(manifestsDir, bootstrapInstallManifest)
-
 	opts := install.Options{
 		BaseURL:                localManifests,
 		Version:                bootstrapVersion,
@@ -124,22 +113,28 @@ func generateInstallManifests(targetPath, namespace, tmpDir string, localManifes
 		NotificationController: defaultNotification,
 		ManifestsFile:          fmt.Sprintf("%s.yaml", namespace),
 		Timeout:                timeout,
+		TargetPath:             targetPath,
 	}
 
 	if localManifests == "" {
 		opts.BaseURL = install.MakeDefaultOptions().BaseURL
 	}
 
-	output, err := install.Generate(opts)
+	manifestPath, content, err := install.Generate(opts)
 	if err != nil {
 		return "", fmt.Errorf("generating install manifests failed: %w", err)
 	}
 
-	if err := ioutil.WriteFile(manifest, output, os.ModePerm); err != nil {
+	filePath := path.Join(tmpDir, manifestPath)
+	if err := os.MkdirAll(path.Dir(manifestPath), os.ModePerm); err != nil {
+		return "", fmt.Errorf("creating manifest dir failed: %w", err)
+	}
+
+	if err := ioutil.WriteFile(filePath, []byte(content), os.ModePerm); err != nil {
 		return "", fmt.Errorf("generating install manifests failed: %w", err)
 	}
 
-	return manifest, nil
+	return filePath, nil
 }
 
 func applyInstallManifests(ctx context.Context, manifestPath string, components []string) error {
@@ -158,70 +153,24 @@ func applyInstallManifests(ctx context.Context, manifestPath string, components
 }
 
 func generateSyncManifests(url, branch, name, namespace, targetPath, tmpDir string, interval time.Duration) error {
-	gvk := sourcev1.GroupVersion.WithKind(sourcev1.GitRepositoryKind)
+	opts := sync.Options{
-	gitRepository := sourcev1.GitRepository{
+		Name:       name,
-		TypeMeta: metav1.TypeMeta{
+		Namespace:  namespace,
-			Kind:       gvk.Kind,
+		URL:        url,
-			APIVersion: gvk.GroupVersion().String(),
+		Branch:     branch,
-		},
+		Interval:   interval,
-		ObjectMeta: metav1.ObjectMeta{
+		TargetPath: targetPath,
-			Name:      name,
-			Namespace: namespace,
-		},
-		Spec: sourcev1.GitRepositorySpec{
-			URL: url,
-			Interval: metav1.Duration{
-				Duration: interval,
-			},
-			Reference: &sourcev1.GitRepositoryRef{
-				Branch: branch,
-			},
-			SecretRef: &corev1.LocalObjectReference{
-				Name: name,
-			},
-		},
 	}
 
-	gitData, err := yaml.Marshal(gitRepository)
+	output, err := sync.Generate(opts)
 	if err != nil {
-		return err
+		return fmt.Errorf("generating install manifests failed: %w", err)
 	}
 
-	if err := utils.WriteFile(string(gitData), filepath.Join(tmpDir, targetPath, namespace, bootstrapSourceManifest)); err != nil {
+	for _, v := range output {
-		return err
+		if err := utils.WriteFile(v["content"], filepath.Join(tmpDir, v["file_path"])); err != nil {
-	}
+			return err
-
+		}
-	gvk = kustomizev1.GroupVersion.WithKind(kustomizev1.KustomizationKind)
-	kustomization := kustomizev1.Kustomization{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       gvk.Kind,
-			APIVersion: gvk.GroupVersion().String(),
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      name,
-			Namespace: namespace,
-		},
-		Spec: kustomizev1.KustomizationSpec{
-			Interval: metav1.Duration{
-				Duration: 10 * time.Minute,
-			},
-			Path:  fmt.Sprintf("./%s", strings.TrimPrefix(targetPath, "./")),
-			Prune: true,
-			SourceRef: kustomizev1.CrossNamespaceSourceReference{
-				Kind: sourcev1.GitRepositoryKind,
-				Name: name,
-			},
-			Validation: "client",
-		},
-	}
-
-	ksData, err := yaml.Marshal(kustomization)
-	if err != nil {
-		return err
-	}
-
-	if err := utils.WriteFile(string(ksData), filepath.Join(tmpDir, targetPath, namespace, bootstrapKustomizationManifest)); err != nil {
-		return err
 	}
 
 	if err := utils.GenerateKustomizationYaml(filepath.Join(tmpDir, targetPath, namespace)); err != nil {

cmd/gotk/install.go

@@ -42,10 +42,10 @@ If a previous version is installed, then an in-place upgrade will be performed.`
   # Dry-run install for a specific version and a series of components
   gotk install --dry-run --version=v0.0.7 --components="source-controller,kustomize-controller"
 
-  # Dry-run install with manifests preview 
+  # Dry-run install with manifests preview
   gotk install --dry-run --verbose
 
-  # Write install manifests to file 
+  # Write install manifests to file
   gotk install --export > gotk-system.yaml
 `,
 	RunE: installCmdRun,
@@ -123,24 +123,23 @@ func installCmdRun(cmd *cobra.Command, args []string) error {
 		opts.BaseURL = install.MakeDefaultOptions().BaseURL
 	}
 
-	output, err := install.Generate(opts)
+	_, content, err := install.Generate(opts)
 	if err != nil {
 		return fmt.Errorf("install failed: %w", err)
 	}
 
 	manifest := path.Join(tmpDir, fmt.Sprintf("%s.yaml", namespace))
-	if err := ioutil.WriteFile(manifest, output, os.ModePerm); err != nil {
+	if err := ioutil.WriteFile(manifest, []byte(content), os.ModePerm); err != nil {
 		return fmt.Errorf("install failed: %w", err)
 	}
 
-	yaml := string(output)
 	if verbose {
-		fmt.Print(yaml)
+		fmt.Print(content)
 	} else if installExport {
 		fmt.Println("---")
 		fmt.Println("# GitOps Toolkit revision", installVersion)
 		fmt.Println("# Components:", strings.Join(installComponents, ","))
-		fmt.Print(yaml)
+		fmt.Print(content)
 		fmt.Println("---")
 		return nil
 	}

docs/cmd/gotk_install.md

@@ -20,10 +20,10 @@ gotk install [flags]
   # Dry-run install for a specific version and a series of components
   gotk install --dry-run --version=v0.0.7 --components="source-controller,kustomize-controller"
 
-  # Dry-run install with manifests preview 
+  # Dry-run install with manifests preview
   gotk install --dry-run --verbose
 
-  # Write install manifests to file 
+  # Write install manifests to file
   gotk install --export > gotk-system.yaml
 
 ```

pkg/install/install.go

@@ -28,13 +28,13 @@ import (
 // Generate returns the install manifests as a multi-doc YAML.
 // The manifests are built from a GitHub release or from a
 // Kustomize overlay if the supplied Options.BaseURL is a local path.
-func Generate(options Options) ([]byte, error) {
+func Generate(options Options) (string, string, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), options.Timeout)
 	defer cancel()
 
 	tmpDir, err := ioutil.TempDir("", options.Namespace)
 	if err != nil {
-		return nil, fmt.Errorf("temp dir error: %w", err)
+		return "", "", fmt.Errorf("temp dir error: %w", err)
 	}
 	defer os.RemoveAll(tmpDir)
 
@@ -42,26 +42,26 @@ func Generate(options Options) ([]byte, error) {
 
 	if !strings.HasPrefix(options.BaseURL, "http") {
 		if err := build(options.BaseURL, output); err != nil {
-			return nil, err
+			return "", "", err
 		}
 	} else {
 		if err := fetch(ctx, options.BaseURL, options.Version, tmpDir); err != nil {
-			return nil, err
+			return "", "", err
 		}
 
 		if err := generate(tmpDir, options); err != nil {
-			return nil, err
+			return "", "", err
 		}
 
 		if err := build(tmpDir, output); err != nil {
-			return nil, err
+			return "", "", err
 		}
 	}
 
 	content, err := ioutil.ReadFile(output)
 	if err != nil {
-		return nil, err
+		return "", "", err
 	}
 
-	return content, nil
+	return path.Join(options.TargetPath, options.Namespace, options.ManifestsFile), string(content), nil
 }

pkg/install/install_test.go

@@ -24,7 +24,7 @@ import (
 
 func TestGenerate(t *testing.T) {
 	opts := MakeDefaultOptions()
-	output, err := Generate(opts)
+	_, output, err := Generate(opts)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/install/options.go

@@ -33,6 +33,7 @@ type Options struct {
 	NotificationController string
 	ManifestsFile          string
 	Timeout                time.Duration
+	TargetPath             string
 }
 
 func MakeDefaultOptions() Options {
@@ -51,6 +52,7 @@ func MakeDefaultOptions() Options {
 		NotificationController: "notification-controller",
 		ManifestsFile:          "toolkit-components.yaml",
 		Timeout:                time.Minute,
+		TargetPath:             "",
 	}
 }
 

pkg/sync/options.go

@@ -0,0 +1,23 @@
+package sync
+
+import "time"
+
+type Options struct {
+	Interval   time.Duration
+	URL        string
+	Name       string
+	Namespace  string
+	Branch     string
+	TargetPath string
+}
+
+func MakeDefaultOptions() Options {
+	return Options{
+		Interval:   1 * time.Minute,
+		URL:        "",
+		Name:       "gotk-system",
+		Namespace:  "gotk-system",
+		Branch:     "main",
+		TargetPath: "",
+	}
+}

pkg/sync/sync.go

@@ -0,0 +1,88 @@
+package sync
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/yaml"
+
+	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1"
+	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
+)
+
+const (
+	bootstrapSourceManifest        = "toolkit-source.yaml"
+	bootstrapKustomizationManifest = "toolkit-kustomization.yaml"
+)
+
+func Generate(options Options) ([]map[string]string, error) {
+	files := []map[string]string{}
+
+	gvk := sourcev1.GroupVersion.WithKind(sourcev1.GitRepositoryKind)
+	gitRepository := sourcev1.GitRepository{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       gvk.Kind,
+			APIVersion: gvk.GroupVersion().String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      options.Name,
+			Namespace: options.Namespace,
+		},
+		Spec: sourcev1.GitRepositorySpec{
+			URL: options.URL,
+			Interval: metav1.Duration{
+				Duration: options.Interval,
+			},
+			Reference: &sourcev1.GitRepositoryRef{
+				Branch: options.Branch,
+			},
+			SecretRef: &corev1.LocalObjectReference{
+				Name: options.Name,
+			},
+		},
+	}
+
+	gitData, err := yaml.Marshal(gitRepository)
+	if err != nil {
+		return nil, err
+	}
+
+	files = append(files, map[string]string{"file_path": filepath.Join(options.TargetPath, options.Namespace, bootstrapSourceManifest), "content": string(gitData)})
+
+	gvk = kustomizev1.GroupVersion.WithKind(kustomizev1.KustomizationKind)
+	kustomization := kustomizev1.Kustomization{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       gvk.Kind,
+			APIVersion: gvk.GroupVersion().String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      options.Name,
+			Namespace: options.Namespace,
+		},
+		Spec: kustomizev1.KustomizationSpec{
+			Interval: metav1.Duration{
+				Duration: 10 * time.Minute,
+			},
+			Path:  fmt.Sprintf("./%s", strings.TrimPrefix(options.TargetPath, "./")),
+			Prune: true,
+			SourceRef: kustomizev1.CrossNamespaceSourceReference{
+				Kind: sourcev1.GitRepositoryKind,
+				Name: options.Name,
+			},
+			Validation: "client",
+		},
+	}
+
+	ksData, err := yaml.Marshal(kustomization)
+	if err != nil {
+		return nil, err
+	}
+
+	files = append(files, map[string]string{"file_path": filepath.Join(options.TargetPath, options.Namespace, bootstrapKustomizationManifest), "content": string(ksData)})
+
+	return files, nil
+}

pkg/sync/sync_test.go

@@ -0,0 +1,32 @@
+/*
+Copyright 2020 The Flux CD contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sync
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestGenerate(t *testing.T) {
+	opts := MakeDefaultOptions()
+	output, err := Generate(opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Println(output)
+}