From 4ec5b7fc6955a14189d39bc67f1b61b3602e04c8 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 5 Jun 2020 18:45:31 +0300 Subject: [PATCH 01/11] Implement GitHub repository bootstrap --- cmd/tk/bootstrap.go | 20 ++ cmd/tk/bootstrap_github.go | 364 +++++++++++++++++++++++++++++++++++++ cmd/tk/utils.go | 15 ++ go.sum | 13 ++ 4 files changed, 412 insertions(+) create mode 100644 cmd/tk/bootstrap.go create mode 100644 cmd/tk/bootstrap_github.go diff --git a/cmd/tk/bootstrap.go b/cmd/tk/bootstrap.go new file mode 100644 index 00000000..052186c6 --- /dev/null +++ b/cmd/tk/bootstrap.go @@ -0,0 +1,20 @@ +package main + +import ( + "github.com/spf13/cobra" +) + +var bootstrapCmd = &cobra.Command{ + Use: "bootstrap", + Short: "Bootstrap commands", +} + +var ( + bootstrapVersion string +) + +func init() { + bootstrapCmd.PersistentFlags().StringVar(&bootstrapVersion, "version", "master", "toolkit tag or branch") + + rootCmd.AddCommand(bootstrapCmd) +} diff --git a/cmd/tk/bootstrap_github.go b/cmd/tk/bootstrap_github.go new file mode 100644 index 00000000..ce91202b --- /dev/null +++ b/cmd/tk/bootstrap_github.go @@ -0,0 +1,364 @@ +package main + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "sigs.k8s.io/yaml" + "strings" + "time" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/google/go-github/v32/github" + "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + + kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1alpha1" + sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1" +) + +var bootstrapGitHubCmd = &cobra.Command{ + Use: "github", + Short: "Bootstrap GitHub repository", + RunE: bootstrapGitHubCmdRun, +} + +var ( + ghOwner string + ghRepository string +) + +const ghTokenName = "GITHUB_TOKEN" + +func init() { + bootstrapGitHubCmd.Flags().StringVar(&ghOwner, "owner", "", "GitHub user or organization name") + bootstrapGitHubCmd.Flags().StringVar(&ghRepository, "repository", "", "GitHub repository name") + bootstrapCmd.AddCommand(bootstrapGitHubCmd) +} + +func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { + ghToken := os.Getenv(ghTokenName) + ghURL := fmt.Sprintf("https://github.com/%s/%s", ghOwner, ghRepository) + if ghToken == "" { + return fmt.Errorf("%s environment variable not found", ghTokenName) + } + + if ghOwner == "" || ghRepository == "" { + return fmt.Errorf("owner and repository are required") + } + + tmpDir, err := ioutil.TempDir("", namespace) + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + // create GitHub repository if doesn't exists + logAction("connecting to GitHub") + if err := initGitHubRepository(ctx, ghOwner, ghRepository, ghToken); err != nil { + return err + } + + // clone repository and checkout the master branch + repo, err := checkoutGitHubRepository(ctx, tmpDir, ghURL, ghToken) + if err != nil { + return err + } + w, err := repo.Worktree() + if err != nil { + return err + } + logSuccess("repository cloned") + + // generate install manifests + logGenerate("generating manifests") + kDir := path.Join(tmpDir, ".kustomization") + if err := os.MkdirAll(kDir, os.ModePerm); err != nil { + return fmt.Errorf("generating manifests failed: %w", err) + } + if err := genInstallManifests(bootstrapVersion, namespace, components, kDir); err != nil { + return fmt.Errorf("generating manifests failed: %w", err) + } + + manifestsDir := path.Join(tmpDir, namespace) + if err := os.MkdirAll(manifestsDir, os.ModePerm); err != nil { + return fmt.Errorf("generating manifests failed: %w", err) + } + + manifest := path.Join(manifestsDir, "toolkit.yaml") + if err := buildKustomization(kDir, manifest); err != nil { + return fmt.Errorf("build kustomization failed: %w", err) + } + + os.RemoveAll(kDir) + + // stage install manifests + _, err = w.Add(fmt.Sprintf("%s/toolkit.yaml", namespace)) + if err != nil { + return err + } + + status, err := w.Status() + if err != nil { + return err + } + + isInstall := !status.IsClean() + + if isInstall { + // commit install manifests + logGenerate("pushing manifests") + _, err = w.Commit("Add bootstrap manifests", &git.CommitOptions{ + Author: &object.Signature{ + Name: "tk", + Email: "tk@@users.noreply.github.com", + When: time.Now(), + }, + }) + if err != nil { + return err + } + + // push install manifests + if err := pushGitHubRepository(ctx, repo, ghToken); err != nil { + return err + } + logSuccess("manifests pushed") + + // apply install manifests + logAction("installing components in %s namespace", namespace) + command := fmt.Sprintf("cat %s | kubectl apply -f-", manifest) + if _, err := utils.execCommand(ctx, ModeOS, command); err != nil { + return fmt.Errorf("install failed") + } + logSuccess("install completed") + + // check rollout status + logWaiting("verifying installation") + for _, deployment := range components { + command = fmt.Sprintf("kubectl -n %s rollout status deployment %s --timeout=%s", + namespace, deployment, timeout.String()) + if _, err := utils.execCommand(ctx, ModeOS, command); err != nil { + return fmt.Errorf("install failed") + } else { + logSuccess("%s ready", deployment) + } + } + } + + // create or update auth secret + if err := generateBasicAuth(ctx, namespace, namespace, "git", ghToken); err != nil { + return err + } + logSuccess("authentication configured") + + // generate and push source kustomization + if isInstall { + logAction("generating kustomization manifests") + if err := generateGitHubKustomization(ghURL, namespace, namespace, tmpDir); err != nil { + return err + } + + // stage manifests + _, err = w.Add(fmt.Sprintf("%s", namespace)) + if err != nil { + return err + } + + // commit manifests + logAction("pushing kustomization manifests") + _, err = w.Commit("Add kustomization manifests", &git.CommitOptions{ + Author: &object.Signature{ + Name: "tk", + Email: "tk@@users.noreply.github.com", + When: time.Now(), + }, + }) + if err != nil { + return err + } + + // push install manifests + if err := pushGitHubRepository(ctx, repo, ghToken); err != nil { + return err + } + logSuccess("kustomization manifests pushed") + + logAction("applying kustomization manifests") + if err := applyGitHubKustomization(ctx, namespace, namespace, tmpDir); err != nil { + return err + } + } + + logSuccess("bootstrap finished") + return nil +} + +func initGitHubRepository(ctx context.Context, owner, name, token string) error { + isPrivate := true + isAutoInit := true + auth := github.BasicAuthTransport{ + Username: "git", + Password: token, + } + + client := github.NewClient(auth.Client()) + _, _, err := client.Repositories.Create(ctx, owner, &github.Repository{ + AutoInit: &isAutoInit, + Name: &name, + Private: &isPrivate, + }) + if err != nil { + if !strings.Contains(err.Error(), "name already exists on this account") { + return fmt.Errorf("github create repository error: %w", err) + } + } else { + logSuccess("repository created") + } + return nil +} + +func checkoutGitHubRepository(ctx context.Context, path, url, token string) (*git.Repository, error) { + branch := "master" + auth := &http.BasicAuth{ + Username: "git", + Password: token, + } + repo, err := git.PlainCloneContext(ctx, path, false, &git.CloneOptions{ + URL: url, + Auth: auth, + RemoteName: git.DefaultRemoteName, + ReferenceName: plumbing.NewBranchReferenceName(branch), + SingleBranch: true, + NoCheckout: false, + Progress: nil, + Tags: git.NoTags, + }) + if err != nil { + return nil, fmt.Errorf("git clone error: %w", err) + } + + _, err = repo.Head() + if err != nil { + return nil, fmt.Errorf("git resolve HEAD error: %w", err) + } + + return repo, nil +} + +func pushGitHubRepository(ctx context.Context, repo *git.Repository, token string) error { + auth := &http.BasicAuth{ + Username: "git", + Password: token, + } + err := repo.PushContext(ctx, &git.PushOptions{ + Auth: auth, + Progress: nil, + }) + if err != nil { + return fmt.Errorf("git push error: %w", err) + } + return nil +} + +func generateGitHubKustomization(url, name, namespace, tmpDir string) error { + gvk := sourcev1.GroupVersion.WithKind("GitRepository") + gitRepository := sourcev1.GitRepository{ + TypeMeta: metav1.TypeMeta{ + Kind: gvk.Kind, + APIVersion: gvk.GroupVersion().String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: sourcev1.GitRepositorySpec{ + URL: url, + Interval: metav1.Duration{ + Duration: 1 * time.Minute, + }, + Reference: &sourcev1.GitRepositoryRef{ + Branch: "master", + }, + SecretRef: &corev1.LocalObjectReference{ + Name: name, + }, + }, + } + + gitData, err := yaml.Marshal(gitRepository) + if err != nil { + return err + } + + if err := utils.writeFile(string(gitData), filepath.Join(tmpDir, namespace, "toolkit-source.yaml")); err != nil { + return err + } + + gvk = kustomizev1.GroupVersion.WithKind("Kustomization") + emptyAPIGroup := "" + kustomization := kustomizev1.Kustomization{ + TypeMeta: metav1.TypeMeta{ + Kind: gvk.Kind, + APIVersion: gvk.GroupVersion().String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: kustomizev1.KustomizationSpec{ + Interval: metav1.Duration{ + Duration: 10 * time.Minute, + }, + Path: "./", + Prune: true, + SourceRef: corev1.TypedLocalObjectReference{ + APIGroup: &emptyAPIGroup, + Kind: "GitRepository", + Name: name, + }, + }, + } + + ksData, err := yaml.Marshal(kustomization) + if err != nil { + return err + } + + if err := utils.writeFile(string(ksData), filepath.Join(tmpDir, namespace, "toolkit-kustomization.yaml")); err != nil { + return err + } + + return nil +} + +func applyGitHubKustomization(ctx context.Context, name, namespace, tmpDir string) error { + kubeClient, err := utils.kubeClient(kubeconfig) + if err != nil { + return err + } + + command := fmt.Sprintf("kubectl apply -f %s", filepath.Join(tmpDir, namespace)) + if _, err := utils.execCommand(ctx, ModeStderrOS, command); err != nil { + return err + } + + logWaiting("waiting for kustomization sync") + if err := wait.PollImmediate(pollInterval, timeout, + isKustomizationReady(ctx, kubeClient, name, namespace)); err != nil { + return err + } + + return nil +} diff --git a/cmd/tk/utils.go b/cmd/tk/utils.go index 41ae679f..aec69352 100644 --- a/cmd/tk/utils.go +++ b/cmd/tk/utils.go @@ -112,3 +112,18 @@ func (*Utils) kubeClient(config string) (client.Client, error) { return kubeClient, nil } + +func (*Utils) writeFile(content, filename string) error { + file, err := os.Create(filename) + if err != nil { + return err + } + defer file.Close() + + _, err = io.WriteString(file, content) + if err != nil { + return err + } + + return file.Sync() +} diff --git a/go.sum b/go.sum index 5291ba85..7d9cdf05 100644 --- a/go.sum +++ b/go.sum @@ -48,13 +48,16 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -169,6 +172,7 @@ github.com/fluxcd/source-controller v0.0.1-alpha.6 h1:CpSH1mA9bRjifRwmcVS98ejY2M github.com/fluxcd/source-controller v0.0.1-alpha.6/go.mod h1:gvU+sR6yH9kbyWZAJ9NEBQJMsJ+S3yCyOdPIhM+8FUI= github.com/fluxcd/source-controller v0.0.1-beta.1 h1:1lNFrAwnrZG2qWbm9/6qgDsY4F6go+6HZ4XSCmmpAiY= github.com/fluxcd/source-controller v0.0.1-beta.1/go.mod h1:tmscNdCxEt7+Xt2g1+bI38hMPw2leYMFAaCn4UlMGuw= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -176,6 +180,7 @@ github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYis github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= @@ -186,6 +191,7 @@ github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= github.com/go-git/go-billy/v5 v5.0.0 h1:7NQHvd9FVid8VL4qVUMm8XifBK+2xCoZ2lSk0agRrHM= github.com/go-git/go-billy/v5 v5.0.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.0.1 h1:q+IFMfLx200Q3scvt2hN79JsEzy4AmBTp/pqnefH+Bc= github.com/go-git/go-git-fixtures/v4 v4.0.1/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw= github.com/go-git/go-git/v5 v5.0.0 h1:k5RWPm4iJwYtfWoxIJy4wJX9ON7ihPeZZYC1fLYDnpg= github.com/go-git/go-git/v5 v5.0.0/go.mod h1:oYD8y9kWsGINPFJoLdaScGCN6dlKg23blmClfZwtUVA= @@ -313,6 +319,10 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github/v32 v32.0.0 h1:q74KVb22spUq0U5HqZ9VCYqQz8YRuOtL/39ZnfwO+NM= +github.com/google/go-github/v32 v32.0.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= @@ -657,6 +667,8 @@ golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM= +golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -793,6 +805,7 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= From 59ae2f327c879251d7c6701351b7abd641684e14 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sat, 6 Jun 2020 13:31:15 +0300 Subject: [PATCH 02/11] Implement bootstrap upgrade --- cmd/tk/bootstrap_github.go | 188 +++++++++++++++++++++++-------------- 1 file changed, 115 insertions(+), 73 deletions(-) diff --git a/cmd/tk/bootstrap_github.go b/cmd/tk/bootstrap_github.go index ce91202b..1231de44 100644 --- a/cmd/tk/bootstrap_github.go +++ b/cmd/tk/bootstrap_github.go @@ -19,7 +19,9 @@ import ( "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1alpha1" sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1" @@ -28,12 +30,25 @@ import ( var bootstrapGitHubCmd = &cobra.Command{ Use: "github", Short: "Bootstrap GitHub repository", - RunE: bootstrapGitHubCmdRun, + Long: ` +The bootstrap command creates the GitHub repository if it doesn't exists and +commits the toolkit components manifests to the master branch. +Then it configure the target cluster to synchronize with the repository. +If the toolkit components are present on the cluster, +the bootstrap command will perform an upgrade if needed.`, + Example: ` # Create a GitHub personal access token and export it as an env var + export GITHUB_TOKEN= + + # Run bootstrap + tk bootstrap github --owner= --repository= +`, + RunE: bootstrapGitHubCmdRun, } var ( ghOwner string ghRepository string + ghInterval time.Duration ) const ghTokenName = "GITHUB_TOKEN" @@ -41,6 +56,7 @@ const ghTokenName = "GITHUB_TOKEN" func init() { bootstrapGitHubCmd.Flags().StringVar(&ghOwner, "owner", "", "GitHub user or organization name") bootstrapGitHubCmd.Flags().StringVar(&ghRepository, "repository", "", "GitHub repository name") + bootstrapGitHubCmd.Flags().DurationVar(&ghInterval, "interval", time.Minute, "sync interval") bootstrapCmd.AddCommand(bootstrapGitHubCmd) } @@ -55,6 +71,11 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { return fmt.Errorf("owner and repository are required") } + kubeClient, err := utils.kubeClient(kubeconfig) + if err != nil { + return err + } + tmpDir, err := ioutil.TempDir("", namespace) if err != nil { return err @@ -75,70 +96,37 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { if err != nil { return err } - w, err := repo.Worktree() - if err != nil { - return err - } logSuccess("repository cloned") // generate install manifests logGenerate("generating manifests") - kDir := path.Join(tmpDir, ".kustomization") - if err := os.MkdirAll(kDir, os.ModePerm); err != nil { - return fmt.Errorf("generating manifests failed: %w", err) + manifest, err := generateGitHubInstall(namespace, tmpDir) + if err != nil { + return err } - if err := genInstallManifests(bootstrapVersion, namespace, components, kDir); err != nil { - return fmt.Errorf("generating manifests failed: %w", err) - } - - manifestsDir := path.Join(tmpDir, namespace) - if err := os.MkdirAll(manifestsDir, os.ModePerm); err != nil { - return fmt.Errorf("generating manifests failed: %w", err) - } - - manifest := path.Join(manifestsDir, "toolkit.yaml") - if err := buildKustomization(kDir, manifest); err != nil { - return fmt.Errorf("build kustomization failed: %w", err) - } - - os.RemoveAll(kDir) // stage install manifests - _, err = w.Add(fmt.Sprintf("%s/toolkit.yaml", namespace)) + changed, err := commitGitHubManifests(repo, namespace) if err != nil { return err } - status, err := w.Status() - if err != nil { - return err - } - - isInstall := !status.IsClean() - - if isInstall { - // commit install manifests - logGenerate("pushing manifests") - _, err = w.Commit("Add bootstrap manifests", &git.CommitOptions{ - Author: &object.Signature{ - Name: "tk", - Email: "tk@@users.noreply.github.com", - When: time.Now(), - }, - }) - if err != nil { - return err - } - + if changed { // push install manifests if err := pushGitHubRepository(ctx, repo, ghToken); err != nil { return err } - logSuccess("manifests pushed") + logSuccess("components manifests pushed") + } else { + logSuccess("components are up to date") + } + isInstall := shouldInstallGitHub(ctx, kubeClient, namespace) + + if isInstall { // apply install manifests logAction("installing components in %s namespace", namespace) - command := fmt.Sprintf("cat %s | kubectl apply -f-", manifest) + command := fmt.Sprintf("kubectl apply -f %s", manifest) if _, err := utils.execCommand(ctx, ModeOS, command); err != nil { return fmt.Errorf("install failed") } @@ -166,37 +154,26 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { // generate and push source kustomization if isInstall { logAction("generating kustomization manifests") - if err := generateGitHubKustomization(ghURL, namespace, namespace, tmpDir); err != nil { + if err := generateGitHubKustomization(ghURL, namespace, namespace, tmpDir, ghInterval); err != nil { return err } // stage manifests - _, err = w.Add(fmt.Sprintf("%s", namespace)) - if err != nil { - return err - } - - // commit manifests - logAction("pushing kustomization manifests") - _, err = w.Commit("Add kustomization manifests", &git.CommitOptions{ - Author: &object.Signature{ - Name: "tk", - Email: "tk@@users.noreply.github.com", - When: time.Now(), - }, - }) + changed, err = commitGitHubManifests(repo, namespace) if err != nil { return err } // push install manifests - if err := pushGitHubRepository(ctx, repo, ghToken); err != nil { - return err + if changed { + if err := pushGitHubRepository(ctx, repo, ghToken); err != nil { + return err + } } logSuccess("kustomization manifests pushed") logAction("applying kustomization manifests") - if err := applyGitHubKustomization(ctx, namespace, namespace, tmpDir); err != nil { + if err := applyGitHubKustomization(ctx, kubeClient, namespace, namespace, tmpDir); err != nil { return err } } @@ -257,6 +234,63 @@ func checkoutGitHubRepository(ctx context.Context, path, url, token string) (*gi return repo, nil } +func generateGitHubInstall(namespace, tmpDir string) (string, error) { + kDir := path.Join(tmpDir, ".kustomization") + defer os.RemoveAll(kDir) + + if err := os.MkdirAll(kDir, os.ModePerm); err != nil { + return "", fmt.Errorf("generating manifests failed: %w", err) + } + + if err := genInstallManifests(bootstrapVersion, namespace, components, kDir); err != nil { + return "", fmt.Errorf("generating manifests failed: %w", err) + } + + manifestsDir := path.Join(tmpDir, namespace) + if err := os.MkdirAll(manifestsDir, os.ModePerm); err != nil { + return "", fmt.Errorf("generating manifests failed: %w", err) + } + + manifest := path.Join(manifestsDir, "toolkit.yaml") + if err := buildKustomization(kDir, manifest); err != nil { + return "", fmt.Errorf("build kustomization failed: %w", err) + } + + return manifest, nil +} + +func commitGitHubManifests(repo *git.Repository, namespace string) (bool, error) { + w, err := repo.Worktree() + if err != nil { + return false, err + } + + _, err = w.Add(namespace) + if err != nil { + return false, err + } + + status, err := w.Status() + if err != nil { + return false, err + } + + if !status.IsClean() { + if _, err := w.Commit("Add manifests", &git.CommitOptions{ + Author: &object.Signature{ + Name: "tk", + Email: "tk@@users.noreply.github.com", + When: time.Now(), + }, + }); err != nil { + return false, err + } + return true, nil + } + + return false, nil +} + func pushGitHubRepository(ctx context.Context, repo *git.Repository, token string) error { auth := &http.BasicAuth{ Username: "git", @@ -272,7 +306,7 @@ func pushGitHubRepository(ctx context.Context, repo *git.Repository, token strin return nil } -func generateGitHubKustomization(url, name, namespace, tmpDir string) error { +func generateGitHubKustomization(url, name, namespace, tmpDir string, interval time.Duration) error { gvk := sourcev1.GroupVersion.WithKind("GitRepository") gitRepository := sourcev1.GitRepository{ TypeMeta: metav1.TypeMeta{ @@ -286,7 +320,7 @@ func generateGitHubKustomization(url, name, namespace, tmpDir string) error { Spec: sourcev1.GitRepositorySpec{ URL: url, Interval: metav1.Duration{ - Duration: 1 * time.Minute, + Duration: interval, }, Reference: &sourcev1.GitRepositoryRef{ Branch: "master", @@ -343,12 +377,7 @@ func generateGitHubKustomization(url, name, namespace, tmpDir string) error { return nil } -func applyGitHubKustomization(ctx context.Context, name, namespace, tmpDir string) error { - kubeClient, err := utils.kubeClient(kubeconfig) - if err != nil { - return err - } - +func applyGitHubKustomization(ctx context.Context, kubeClient client.Client, name, namespace, tmpDir string) error { command := fmt.Sprintf("kubectl apply -f %s", filepath.Join(tmpDir, namespace)) if _, err := utils.execCommand(ctx, ModeStderrOS, command); err != nil { return err @@ -362,3 +391,16 @@ func applyGitHubKustomization(ctx context.Context, name, namespace, tmpDir strin return nil } + +func shouldInstallGitHub(ctx context.Context, kubeClient client.Client, namespace string) bool { + namespacedName := types.NamespacedName{ + Namespace: namespace, + Name: namespace, + } + var kustomization kustomizev1.Kustomization + if err := kubeClient.Get(ctx, namespacedName, &kustomization); err != nil { + return true + } + + return kustomization.Status.LastAppliedRevision == "" +} From d4cbc45e1604b16088a3847e67d2fab7eb169b24 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sat, 6 Jun 2020 15:23:57 +0300 Subject: [PATCH 03/11] Add personal and public options to bootstrap --- cmd/tk/bootstrap_github.go | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/cmd/tk/bootstrap_github.go b/cmd/tk/bootstrap_github.go index 1231de44..6fd12dab 100644 --- a/cmd/tk/bootstrap_github.go +++ b/cmd/tk/bootstrap_github.go @@ -39,8 +39,11 @@ the bootstrap command will perform an upgrade if needed.`, Example: ` # Create a GitHub personal access token and export it as an env var export GITHUB_TOKEN= - # Run bootstrap - tk bootstrap github --owner= --repository= + # Run bootstrap for a private repo owned by a GitHub organization + tk bootstrap github --owner= --repository= + + # Run bootstrap for a public repository on a personal account + tk bootstrap github --owner= --repository= --private=false --personal=true `, RunE: bootstrapGitHubCmdRun, } @@ -49,6 +52,8 @@ var ( ghOwner string ghRepository string ghInterval time.Duration + ghPersonal bool + ghPrivate bool ) const ghTokenName = "GITHUB_TOKEN" @@ -56,17 +61,19 @@ const ghTokenName = "GITHUB_TOKEN" func init() { bootstrapGitHubCmd.Flags().StringVar(&ghOwner, "owner", "", "GitHub user or organization name") bootstrapGitHubCmd.Flags().StringVar(&ghRepository, "repository", "", "GitHub repository name") + bootstrapGitHubCmd.Flags().BoolVar(&ghPersonal, "personal", false, "personal repository") + bootstrapGitHubCmd.Flags().BoolVar(&ghPrivate, "private", true, "private repository") bootstrapGitHubCmd.Flags().DurationVar(&ghInterval, "interval", time.Minute, "sync interval") bootstrapCmd.AddCommand(bootstrapGitHubCmd) } func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { ghToken := os.Getenv(ghTokenName) - ghURL := fmt.Sprintf("https://github.com/%s/%s", ghOwner, ghRepository) if ghToken == "" { return fmt.Errorf("%s environment variable not found", ghTokenName) } + ghURL := fmt.Sprintf("https://github.com/%s/%s", ghOwner, ghRepository) if ghOwner == "" || ghRepository == "" { return fmt.Errorf("owner and repository are required") } @@ -87,7 +94,7 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { // create GitHub repository if doesn't exists logAction("connecting to GitHub") - if err := initGitHubRepository(ctx, ghOwner, ghRepository, ghToken); err != nil { + if err := initGitHubRepository(ctx, ghOwner, ghRepository, ghToken, ghPrivate, ghPersonal); err != nil { return err } @@ -182,17 +189,21 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { return nil } -func initGitHubRepository(ctx context.Context, owner, name, token string) error { - isPrivate := true - isAutoInit := true +func initGitHubRepository(ctx context.Context, owner, name, token string, isPrivate, isPersonal bool) error { + autoInit := true auth := github.BasicAuthTransport{ Username: "git", Password: token, } + org := "" + if !isPersonal { + org = owner + } + client := github.NewClient(auth.Client()) - _, _, err := client.Repositories.Create(ctx, owner, &github.Repository{ - AutoInit: &isAutoInit, + _, _, err := client.Repositories.Create(ctx, org, &github.Repository{ + AutoInit: &autoInit, Name: &name, Private: &isPrivate, }) From 34ea89c3be5fdc931061c302c415885d81951191 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 7 Jun 2020 11:59:47 +0300 Subject: [PATCH 04/11] Create GitHub repo if it doesn't exists --- cmd/tk/bootstrap_github.go | 67 +++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/cmd/tk/bootstrap_github.go b/cmd/tk/bootstrap_github.go index 6fd12dab..706135d4 100644 --- a/cmd/tk/bootstrap_github.go +++ b/cmd/tk/bootstrap_github.go @@ -56,13 +56,19 @@ var ( ghPrivate bool ) -const ghTokenName = "GITHUB_TOKEN" +const ( + ghTokenName = "GITHUB_TOKEN" + ghBranch = "master" + ghInstallManifest = "toolkit.yaml" + ghSourceManifest = "toolkit-source.yaml" + ghKustomizationManifest = "toolkit-kustomization.yaml" +) func init() { bootstrapGitHubCmd.Flags().StringVar(&ghOwner, "owner", "", "GitHub user or organization name") bootstrapGitHubCmd.Flags().StringVar(&ghRepository, "repository", "", "GitHub repository name") - bootstrapGitHubCmd.Flags().BoolVar(&ghPersonal, "personal", false, "personal repository") - bootstrapGitHubCmd.Flags().BoolVar(&ghPrivate, "private", true, "private repository") + bootstrapGitHubCmd.Flags().BoolVar(&ghPersonal, "personal", false, "is personal repository") + bootstrapGitHubCmd.Flags().BoolVar(&ghPrivate, "private", true, "is private repository") bootstrapGitHubCmd.Flags().DurationVar(&ghInterval, "interval", time.Minute, "sync interval") bootstrapCmd.AddCommand(bootstrapGitHubCmd) } @@ -94,12 +100,12 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { // create GitHub repository if doesn't exists logAction("connecting to GitHub") - if err := initGitHubRepository(ctx, ghOwner, ghRepository, ghToken, ghPrivate, ghPersonal); err != nil { + if err := createGitHubRepository(ctx, ghOwner, ghRepository, ghToken, ghPrivate, ghPersonal); err != nil { return err } // clone repository and checkout the master branch - repo, err := checkoutGitHubRepository(ctx, tmpDir, ghURL, ghToken) + repo, err := checkoutGitHubRepository(ctx, ghURL, ghBranch, ghToken, tmpDir) if err != nil { return err } @@ -118,8 +124,8 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { return err } + // push install manifests if changed { - // push install manifests if err := pushGitHubRepository(ctx, repo, ghToken); err != nil { return err } @@ -128,6 +134,7 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { logSuccess("components are up to date") } + // determine if repo synchronization is working isInstall := shouldInstallGitHub(ctx, kubeClient, namespace) if isInstall { @@ -139,7 +146,7 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { } logSuccess("install completed") - // check rollout status + // check installation logWaiting("verifying installation") for _, deployment := range components { command = fmt.Sprintf("kubectl -n %s rollout status deployment %s --timeout=%s", @@ -153,14 +160,16 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { } // create or update auth secret + // TODO: replace this with SSH deploy key if err := generateBasicAuth(ctx, namespace, namespace, "git", ghToken); err != nil { return err } logSuccess("authentication configured") - // generate and push source kustomization + // configure repo synchronization if isInstall { - logAction("generating kustomization manifests") + // generate source and kustomization manifests + logAction("generating sync manifests") if err := generateGitHubKustomization(ghURL, namespace, namespace, tmpDir, ghInterval); err != nil { return err } @@ -171,15 +180,16 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { return err } - // push install manifests + // push manifests if changed { if err := pushGitHubRepository(ctx, repo, ghToken); err != nil { return err } } - logSuccess("kustomization manifests pushed") + logSuccess("sync manifests pushed") - logAction("applying kustomization manifests") + // apply manifests and waiting for sync + logAction("applying sync manifests") if err := applyGitHubKustomization(ctx, kubeClient, namespace, namespace, tmpDir); err != nil { return err } @@ -189,20 +199,24 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { return nil } -func initGitHubRepository(ctx context.Context, owner, name, token string, isPrivate, isPersonal bool) error { +func createGitHubRepository(ctx context.Context, owner, name, token string, isPrivate, isPersonal bool) error { autoInit := true auth := github.BasicAuthTransport{ Username: "git", Password: token, } + gh := github.NewClient(auth.Client()) org := "" if !isPersonal { org = owner } - client := github.NewClient(auth.Client()) - _, _, err := client.Repositories.Create(ctx, org, &github.Repository{ + if _, _, err := gh.Repositories.Get(ctx, org, name); err == nil { + return nil + } + + _, _, err := gh.Repositories.Create(ctx, org, &github.Repository{ AutoInit: &autoInit, Name: &name, Private: &isPrivate, @@ -217,8 +231,7 @@ func initGitHubRepository(ctx context.Context, owner, name, token string, isPriv return nil } -func checkoutGitHubRepository(ctx context.Context, path, url, token string) (*git.Repository, error) { - branch := "master" +func checkoutGitHubRepository(ctx context.Context, url, branch, token, path string) (*git.Repository, error) { auth := &http.BasicAuth{ Username: "git", Password: token, @@ -246,14 +259,14 @@ func checkoutGitHubRepository(ctx context.Context, path, url, token string) (*gi } func generateGitHubInstall(namespace, tmpDir string) (string, error) { - kDir := path.Join(tmpDir, ".kustomization") - defer os.RemoveAll(kDir) + tkDir := path.Join(tmpDir, ".tk") + defer os.RemoveAll(tkDir) - if err := os.MkdirAll(kDir, os.ModePerm); err != nil { + if err := os.MkdirAll(tkDir, os.ModePerm); err != nil { return "", fmt.Errorf("generating manifests failed: %w", err) } - if err := genInstallManifests(bootstrapVersion, namespace, components, kDir); err != nil { + if err := genInstallManifests(bootstrapVersion, namespace, components, tkDir); err != nil { return "", fmt.Errorf("generating manifests failed: %w", err) } @@ -262,8 +275,8 @@ func generateGitHubInstall(namespace, tmpDir string) (string, error) { return "", fmt.Errorf("generating manifests failed: %w", err) } - manifest := path.Join(manifestsDir, "toolkit.yaml") - if err := buildKustomization(kDir, manifest); err != nil { + manifest := path.Join(manifestsDir, ghInstallManifest) + if err := buildKustomization(tkDir, manifest); err != nil { return "", fmt.Errorf("build kustomization failed: %w", err) } @@ -290,7 +303,7 @@ func commitGitHubManifests(repo *git.Repository, namespace string) (bool, error) if _, err := w.Commit("Add manifests", &git.CommitOptions{ Author: &object.Signature{ Name: "tk", - Email: "tk@@users.noreply.github.com", + Email: "tk@users.noreply.github.com", When: time.Now(), }, }); err != nil { @@ -347,7 +360,7 @@ func generateGitHubKustomization(url, name, namespace, tmpDir string, interval t return err } - if err := utils.writeFile(string(gitData), filepath.Join(tmpDir, namespace, "toolkit-source.yaml")); err != nil { + if err := utils.writeFile(string(gitData), filepath.Join(tmpDir, namespace, ghSourceManifest)); err != nil { return err } @@ -381,7 +394,7 @@ func generateGitHubKustomization(url, name, namespace, tmpDir string, interval t return err } - if err := utils.writeFile(string(ksData), filepath.Join(tmpDir, namespace, "toolkit-kustomization.yaml")); err != nil { + if err := utils.writeFile(string(ksData), filepath.Join(tmpDir, namespace, ghKustomizationManifest)); err != nil { return err } @@ -394,7 +407,7 @@ func applyGitHubKustomization(ctx context.Context, kubeClient client.Client, nam return err } - logWaiting("waiting for kustomization sync") + logWaiting("waiting for cluster sync") if err := wait.PollImmediate(pollInterval, timeout, isKustomizationReady(ctx, kubeClient, name, namespace)); err != nil { return err From f5b738198f54c41673ba727c450c80e77e1ff366 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 8 Jun 2020 13:28:20 +0300 Subject: [PATCH 05/11] Add support for GitHub Enterprise --- cmd/tk/bootstrap_github.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/cmd/tk/bootstrap_github.go b/cmd/tk/bootstrap_github.go index 706135d4..35fb159c 100644 --- a/cmd/tk/bootstrap_github.go +++ b/cmd/tk/bootstrap_github.go @@ -44,6 +44,9 @@ the bootstrap command will perform an upgrade if needed.`, # Run bootstrap for a public repository on a personal account tk bootstrap github --owner= --repository= --private=false --personal=true + + # Run bootstrap for a private repo hosted on GitHub Enterprise + tk bootstrap github --owner= --repository= --hostname= `, RunE: bootstrapGitHubCmdRun, } @@ -54,6 +57,7 @@ var ( ghInterval time.Duration ghPersonal bool ghPrivate bool + ghHostname string ) const ( @@ -62,6 +66,7 @@ const ( ghInstallManifest = "toolkit.yaml" ghSourceManifest = "toolkit-source.yaml" ghKustomizationManifest = "toolkit-kustomization.yaml" + ghDefaultHostname = "github.com" ) func init() { @@ -70,6 +75,7 @@ func init() { bootstrapGitHubCmd.Flags().BoolVar(&ghPersonal, "personal", false, "is personal repository") bootstrapGitHubCmd.Flags().BoolVar(&ghPrivate, "private", true, "is private repository") bootstrapGitHubCmd.Flags().DurationVar(&ghInterval, "interval", time.Minute, "sync interval") + bootstrapGitHubCmd.Flags().StringVar(&ghHostname, "hostname", ghDefaultHostname, "GitHub hostname") bootstrapCmd.AddCommand(bootstrapGitHubCmd) } @@ -79,7 +85,7 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { return fmt.Errorf("%s environment variable not found", ghTokenName) } - ghURL := fmt.Sprintf("https://github.com/%s/%s", ghOwner, ghRepository) + ghURL := fmt.Sprintf("https://%s/%s/%s", ghHostname, ghOwner, ghRepository) if ghOwner == "" || ghRepository == "" { return fmt.Errorf("owner and repository are required") } @@ -99,8 +105,8 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { defer cancel() // create GitHub repository if doesn't exists - logAction("connecting to GitHub") - if err := createGitHubRepository(ctx, ghOwner, ghRepository, ghToken, ghPrivate, ghPersonal); err != nil { + logAction("connecting to %s", ghHostname) + if err := createGitHubRepository(ctx, ghHostname, ghOwner, ghRepository, ghToken, ghPrivate, ghPersonal); err != nil { return err } @@ -199,14 +205,22 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { return nil } -func createGitHubRepository(ctx context.Context, owner, name, token string, isPrivate, isPersonal bool) error { +func createGitHubRepository(ctx context.Context, hostname, owner, name, token string, isPrivate, isPersonal bool) error { autoInit := true auth := github.BasicAuthTransport{ Username: "git", Password: token, } gh := github.NewClient(auth.Client()) - + if hostname != ghDefaultHostname { + baseURL := fmt.Sprintf("https://%s/api/v3/", hostname) + uploadURL := fmt.Sprintf("https://%s/api/uploads/", hostname) + if g, err := github.NewEnterpriseClient(baseURL, uploadURL, auth.Client()); err == nil { + gh = g + } else { + return fmt.Errorf("github client error: %w", err) + } + } org := "" if !isPersonal { org = owner From 1bc637a43fe07aa98b4ea147a39dde83890d98bf Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Tue, 9 Jun 2020 12:15:31 +0300 Subject: [PATCH 06/11] Add export flag to create commands --- cmd/tk/create.go | 2 ++ cmd/tk/create_kustomization.go | 13 ++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/cmd/tk/create.go b/cmd/tk/create.go index a227e327..d8f1aa0e 100644 --- a/cmd/tk/create.go +++ b/cmd/tk/create.go @@ -13,9 +13,11 @@ var createCmd = &cobra.Command{ var ( interval time.Duration + export bool ) func init() { createCmd.PersistentFlags().DurationVarP(&interval, "interval", "", time.Minute, "source sync interval") + createCmd.PersistentFlags().BoolVar(&export, "export", false, "export in yaml format to stdout") rootCmd.AddCommand(createCmd) } diff --git a/cmd/tk/create_kustomization.go b/cmd/tk/create_kustomization.go index b8981483..24bc652a 100644 --- a/cmd/tk/create_kustomization.go +++ b/cmd/tk/create_kustomization.go @@ -3,11 +3,9 @@ package main import ( "context" "fmt" - kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1alpha1" "strings" "time" - sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -15,6 +13,9 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" + + kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1alpha1" + sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1" ) var createKsCmd = &cobra.Command{ @@ -106,7 +107,9 @@ func createKsCmdRun(cmd *cobra.Command, args []string) error { return err } - logGenerate("generating kustomization") + if !export { + logGenerate("generating kustomization") + } emptyAPIGroup := "" kustomization := kustomizev1.Kustomization{ @@ -171,6 +174,10 @@ func createKsCmdRun(cmd *cobra.Command, args []string) error { } } + if export { + return exportKs(kustomization) + } + logAction("applying kustomization") if err := upsertKustomization(ctx, kubeClient, kustomization); err != nil { return err From e70d86e843c8edd22016f91594a1af0d7eb9fa69 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Tue, 9 Jun 2020 15:23:19 +0300 Subject: [PATCH 07/11] Fix SSH secret upsert --- cmd/tk/create_source_git.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/tk/create_source_git.go b/cmd/tk/create_source_git.go index 97f4934f..bbad2099 100644 --- a/cmd/tk/create_source_git.go +++ b/cmd/tk/create_source_git.go @@ -287,7 +287,7 @@ func upsertSecret(ctx context.Context, kubeClient client.Client, secret corev1.S err := kubeClient.Get(ctx, namespacedName, &existing) if err != nil { if errors.IsNotFound(err) { - if err := kubeClient.Create(ctx, &existing); err != nil { + if err := kubeClient.Create(ctx, &secret); err != nil { return err } else { return nil From e612a8a49605ba391d77718f8fccc75235a15f76 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Tue, 9 Jun 2020 15:24:36 +0300 Subject: [PATCH 08/11] Implement SSH deploy key bootstrap --- cmd/tk/bootstrap_github.go | 114 ++++++++++++++++++++++++++++++++----- go.mod | 2 + go.sum | 8 ++- 3 files changed, 109 insertions(+), 15 deletions(-) diff --git a/cmd/tk/bootstrap_github.go b/cmd/tk/bootstrap_github.go index 35fb159c..64d8d5d8 100644 --- a/cmd/tk/bootstrap_github.go +++ b/cmd/tk/bootstrap_github.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io/ioutil" + "net/url" "os" "path" "path/filepath" @@ -40,13 +41,13 @@ the bootstrap command will perform an upgrade if needed.`, export GITHUB_TOKEN= # Run bootstrap for a private repo owned by a GitHub organization - tk bootstrap github --owner= --repository= + bootstrap github --owner= --repository= # Run bootstrap for a public repository on a personal account - tk bootstrap github --owner= --repository= --private=false --personal=true + bootstrap github --owner= --repository= --private=false --personal=true # Run bootstrap for a private repo hosted on GitHub Enterprise - tk bootstrap github --owner= --repository= --hostname= + bootstrap github --owner= --repository= --hostname= `, RunE: bootstrapGitHubCmdRun, } @@ -86,6 +87,7 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { } ghURL := fmt.Sprintf("https://%s/%s/%s", ghHostname, ghOwner, ghRepository) + sshURL := fmt.Sprintf("ssh://git@%s/%s/%s", ghHostname, ghOwner, ghRepository) if ghOwner == "" || ghRepository == "" { return fmt.Errorf("owner and repository are required") } @@ -165,18 +167,30 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { } } - // create or update auth secret - // TODO: replace this with SSH deploy key - if err := generateBasicAuth(ctx, namespace, namespace, "git", ghToken); err != nil { - return err + // setup SSH deploy key + if shouldCreateGitHubDeployKey(ctx, kubeClient, namespace) { + logAction("configuring deploy key") + u, err := url.Parse(sshURL) + if err != nil { + return fmt.Errorf("git URL parse failed: %w", err) + } + + key, err := generateGitHubDeployKey(ctx, kubeClient, u, namespace) + if err != nil { + return fmt.Errorf("generating deploy key failed: %w", err) + } + + if err := createGitHubDeployKey(ctx, key, ghHostname, ghOwner, ghRepository, ghToken, ghPersonal); err != nil { + return nil + } + logSuccess("deploy key configured") } - logSuccess("authentication configured") // configure repo synchronization if isInstall { // generate source and kustomization manifests logAction("generating sync manifests") - if err := generateGitHubKustomization(ghURL, namespace, namespace, tmpDir, ghInterval); err != nil { + if err := generateGitHubKustomization(sshURL, namespace, namespace, tmpDir, ghInterval); err != nil { return err } @@ -205,12 +219,12 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { return nil } -func createGitHubRepository(ctx context.Context, hostname, owner, name, token string, isPrivate, isPersonal bool) error { - autoInit := true +func makeGitHubClient(hostname, token string) (*github.Client, error) { auth := github.BasicAuthTransport{ Username: "git", Password: token, } + gh := github.NewClient(auth.Client()) if hostname != ghDefaultHostname { baseURL := fmt.Sprintf("https://%s/api/v3/", hostname) @@ -218,9 +232,18 @@ func createGitHubRepository(ctx context.Context, hostname, owner, name, token st if g, err := github.NewEnterpriseClient(baseURL, uploadURL, auth.Client()); err == nil { gh = g } else { - return fmt.Errorf("github client error: %w", err) + return nil, fmt.Errorf("github client error: %w", err) } } + + return gh, nil +} + +func createGitHubRepository(ctx context.Context, hostname, owner, name, token string, isPrivate, isPersonal bool) error { + gh, err := makeGitHubClient(hostname, token) + if err != nil { + return err + } org := "" if !isPersonal { org = owner @@ -230,7 +253,8 @@ func createGitHubRepository(ctx context.Context, hostname, owner, name, token st return nil } - _, _, err := gh.Repositories.Create(ctx, org, &github.Repository{ + autoInit := true + _, _, err = gh.Repositories.Create(ctx, org, &github.Repository{ AutoInit: &autoInit, Name: &name, Private: &isPrivate, @@ -442,3 +466,67 @@ func shouldInstallGitHub(ctx context.Context, kubeClient client.Client, namespac return kustomization.Status.LastAppliedRevision == "" } + +func shouldCreateGitHubDeployKey(ctx context.Context, kubeClient client.Client, namespace string) bool { + namespacedName := types.NamespacedName{ + Namespace: namespace, + Name: namespace, + } + + var existing corev1.Secret + if err := kubeClient.Get(ctx, namespacedName, &existing); err != nil { + return true + } + return false +} + +func generateGitHubDeployKey(ctx context.Context, kubeClient client.Client, url *url.URL, namespace string) (string, error) { + pair, err := generateKeyPair(ctx) + if err != nil { + return "", err + } + + hostKey, err := scanHostKey(ctx, url) + if err != nil { + return "", err + } + + secret := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + Namespace: namespace, + }, + StringData: map[string]string{ + "identity": string(pair.PrivateKey), + "identity.pub": string(pair.PublicKey), + "known_hosts": string(hostKey), + }, + } + if err := upsertSecret(ctx, kubeClient, secret); err != nil { + return "", err + } + + return string(pair.PublicKey), nil +} + +func createGitHubDeployKey(ctx context.Context, key, hostname, owner, name, token string, isPersonal bool) error { + gh, err := makeGitHubClient(hostname, token) + if err != nil { + return err + } + keyName := fmt.Sprintf("tk-%s", namespace) + org := "" + if !isPersonal { + org = owner + } + isReadOnly := true + _, _, err = gh.Repositories.CreateKey(ctx, org, name, &github.Key{ + Title: &keyName, + Key: &key, + ReadOnly: &isReadOnly, + }) + if err != nil { + return fmt.Errorf("github create deploy key error: %w", err) + } + return nil +} diff --git a/go.mod b/go.mod index 1d0ca0cc..e0d48ecb 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,8 @@ require ( github.com/blang/semver v3.5.1+incompatible github.com/fluxcd/kustomize-controller v0.0.1-beta.1 github.com/fluxcd/source-controller v0.0.1-beta.1 + github.com/go-git/go-git/v5 v5.0.0 + github.com/google/go-github/v32 v32.0.0 github.com/manifoldco/promptui v0.7.0 github.com/spf13/cobra v1.0.0 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 diff --git a/go.sum b/go.sum index 7d9cdf05..29a312b9 100644 --- a/go.sum +++ b/go.sum @@ -95,6 +95,7 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.2 h1:ForxmXkA6tPIvffbrDAcPUIB32QgXkt2XFj+F0UxetA= github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20200107194136-26c1120b8d41/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY= @@ -125,6 +126,7 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= +github.com/deislabs/oras v0.8.1 h1:If674KraJVpujYR00rzdi0QAmW4BxzMJPVAZJKuhQ0c= github.com/deislabs/oras v0.8.1/go.mod h1:Mx0rMSbBNaNfY9hjpccEnxkOqJL6KGjtxNHPLC4G4As= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -491,8 +493,10 @@ github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34= github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= @@ -667,8 +671,6 @@ golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM= -golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -717,6 +719,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -814,6 +817,7 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= From 51ca2ee9d89eaf06637603f81f3e64ee6cc5390e Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Tue, 9 Jun 2020 16:12:40 +0300 Subject: [PATCH 09/11] Improve cluster sync wait --- cmd/tk/bootstrap_github.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cmd/tk/bootstrap_github.go b/cmd/tk/bootstrap_github.go index 64d8d5d8..05361ca8 100644 --- a/cmd/tk/bootstrap_github.go +++ b/cmd/tk/bootstrap_github.go @@ -181,7 +181,7 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { } if err := createGitHubDeployKey(ctx, key, ghHostname, ghOwner, ghRepository, ghToken, ghPersonal); err != nil { - return nil + return err } logSuccess("deploy key configured") } @@ -446,6 +446,12 @@ func applyGitHubKustomization(ctx context.Context, kubeClient client.Client, nam } logWaiting("waiting for cluster sync") + + if err := wait.PollImmediate(pollInterval, timeout, + isGitRepositoryReady(ctx, kubeClient, name, namespace)); err != nil { + return err + } + if err := wait.PollImmediate(pollInterval, timeout, isKustomizationReady(ctx, kubeClient, name, namespace)); err != nil { return err @@ -515,12 +521,8 @@ func createGitHubDeployKey(ctx context.Context, key, hostname, owner, name, toke return err } keyName := fmt.Sprintf("tk-%s", namespace) - org := "" - if !isPersonal { - org = owner - } isReadOnly := true - _, _, err = gh.Repositories.CreateKey(ctx, org, name, &github.Key{ + _, _, err = gh.Repositories.CreateKey(ctx, owner, name, &github.Key{ Title: &keyName, Key: &key, ReadOnly: &isReadOnly, From c047a4476e5eba86ed872b92939aa03662a3241d Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Tue, 9 Jun 2020 16:14:04 +0300 Subject: [PATCH 10/11] Add bootstrap docs --- docs/cmd/tk.md | 3 +- docs/cmd/tk_bootstrap.md | 31 ++++++++++++++ docs/cmd/tk_bootstrap_github.md | 62 ++++++++++++++++++++++++++++ docs/cmd/tk_check.md | 2 +- docs/cmd/tk_completion.md | 2 +- docs/cmd/tk_create.md | 3 +- docs/cmd/tk_create_kustomization.md | 3 +- docs/cmd/tk_create_source.md | 3 +- docs/cmd/tk_create_source_git.md | 30 ++++++++++---- docs/cmd/tk_delete.md | 2 +- docs/cmd/tk_delete_kustomization.md | 2 +- docs/cmd/tk_delete_source.md | 2 +- docs/cmd/tk_delete_source_git.md | 2 +- docs/cmd/tk_export.md | 2 +- docs/cmd/tk_export_kustomization.md | 2 +- docs/cmd/tk_export_source.md | 2 +- docs/cmd/tk_export_source_git.md | 2 +- docs/cmd/tk_get.md | 2 +- docs/cmd/tk_get_kustomizations.md | 2 +- docs/cmd/tk_get_sources.md | 2 +- docs/cmd/tk_get_sources_git.md | 2 +- docs/cmd/tk_install.md | 2 +- docs/cmd/tk_resume.md | 2 +- docs/cmd/tk_resume_kustomization.md | 2 +- docs/cmd/tk_suspend.md | 2 +- docs/cmd/tk_suspend_kustomization.md | 2 +- docs/cmd/tk_sync.md | 2 +- docs/cmd/tk_sync_kustomization.md | 2 +- docs/cmd/tk_sync_source.md | 2 +- docs/cmd/tk_sync_source_git.md | 2 +- docs/cmd/tk_uninstall.md | 2 +- 31 files changed, 146 insertions(+), 37 deletions(-) create mode 100644 docs/cmd/tk_bootstrap.md create mode 100644 docs/cmd/tk_bootstrap_github.md diff --git a/docs/cmd/tk.md b/docs/cmd/tk.md index a45028bf..a957dea0 100644 --- a/docs/cmd/tk.md +++ b/docs/cmd/tk.md @@ -77,6 +77,7 @@ Command line utility for assembling Kubernetes CD pipelines the GitOps way. ### SEE ALSO +* [tk bootstrap](tk_bootstrap.md) - Bootstrap commands * [tk check](tk_check.md) - Check requirements and installation * [tk completion](tk_completion.md) - Generates bash completion scripts * [tk create](tk_create.md) - Create commands @@ -89,4 +90,4 @@ Command line utility for assembling Kubernetes CD pipelines the GitOps way. * [tk sync](tk_sync.md) - Synchronize commands * [tk uninstall](tk_uninstall.md) - Uninstall the toolkit components -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_bootstrap.md b/docs/cmd/tk_bootstrap.md new file mode 100644 index 00000000..7d0990d7 --- /dev/null +++ b/docs/cmd/tk_bootstrap.md @@ -0,0 +1,31 @@ +## tk bootstrap + +Bootstrap commands + +### Synopsis + +Bootstrap commands + +### Options + +``` + -h, --help help for bootstrap + --version string toolkit tag or branch (default "master") +``` + +### Options inherited from parent commands + +``` + --components strings list of components, accepts comma-separated values (default [source-controller,kustomize-controller]) + --kubeconfig string path to the kubeconfig file (default "~/.kube/config") + --namespace string the namespace scope for this operation (default "gitops-system") + --timeout duration timeout for this operation (default 5m0s) + --verbose print generated objects +``` + +### SEE ALSO + +* [tk](tk.md) - Command line utility for assembling Kubernetes CD pipelines +* [tk bootstrap github](tk_bootstrap_github.md) - Bootstrap GitHub repository + +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_bootstrap_github.md b/docs/cmd/tk_bootstrap_github.md new file mode 100644 index 00000000..2210f60c --- /dev/null +++ b/docs/cmd/tk_bootstrap_github.md @@ -0,0 +1,62 @@ +## tk bootstrap github + +Bootstrap GitHub repository + +### Synopsis + + +The bootstrap command creates the GitHub repository if it doesn't exists and +commits the toolkit components manifests to the master branch. +Then it configure the target cluster to synchronize with the repository. +If the toolkit components are present on the cluster, +the bootstrap command will perform an upgrade if needed. + +``` +tk bootstrap github [flags] +``` + +### Examples + +``` + # Create a GitHub personal access token and export it as an env var + export GITHUB_TOKEN= + + # Run bootstrap for a private repo owned by a GitHub organization + bootstrap github --owner= --repository= + + # Run bootstrap for a public repository on a personal account + bootstrap github --owner= --repository= --private=false --personal=true + + # Run bootstrap for a private repo hosted on GitHub Enterprise + bootstrap github --owner= --repository= --hostname= + +``` + +### Options + +``` + -h, --help help for github + --hostname string GitHub hostname (default "github.com") + --interval duration sync interval (default 1m0s) + --owner string GitHub user or organization name + --personal is personal repository + --private is private repository (default true) + --repository string GitHub repository name +``` + +### Options inherited from parent commands + +``` + --components strings list of components, accepts comma-separated values (default [source-controller,kustomize-controller]) + --kubeconfig string path to the kubeconfig file (default "~/.kube/config") + --namespace string the namespace scope for this operation (default "gitops-system") + --timeout duration timeout for this operation (default 5m0s) + --verbose print generated objects + --version string toolkit tag or branch (default "master") +``` + +### SEE ALSO + +* [tk bootstrap](tk_bootstrap.md) - Bootstrap commands + +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_check.md b/docs/cmd/tk_check.md index 778fe8e2..3f065592 100644 --- a/docs/cmd/tk_check.md +++ b/docs/cmd/tk_check.md @@ -44,4 +44,4 @@ tk check [flags] * [tk](tk.md) - Command line utility for assembling Kubernetes CD pipelines -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_completion.md b/docs/cmd/tk_completion.md index 5b75d31a..14f7c146 100644 --- a/docs/cmd/tk_completion.md +++ b/docs/cmd/tk_completion.md @@ -44,4 +44,4 @@ To configure your bash shell to load completions for each session add to your ba * [tk](tk.md) - Command line utility for assembling Kubernetes CD pipelines -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_create.md b/docs/cmd/tk_create.md index 1c1a88b3..286a204c 100644 --- a/docs/cmd/tk_create.md +++ b/docs/cmd/tk_create.md @@ -9,6 +9,7 @@ Create commands ### Options ``` + --export export in yaml format to stdout -h, --help help for create --interval duration source sync interval (default 1m0s) ``` @@ -29,4 +30,4 @@ Create commands * [tk create kustomization](tk_create_kustomization.md) - Create or update a kustomization resource * [tk create source](tk_create_source.md) - Create source commands -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_create_kustomization.md b/docs/cmd/tk_create_kustomization.md index d174648a..6cfb8f84 100644 --- a/docs/cmd/tk_create_kustomization.md +++ b/docs/cmd/tk_create_kustomization.md @@ -66,6 +66,7 @@ tk create kustomization [name] [flags] ``` --components strings list of components, accepts comma-separated values (default [source-controller,kustomize-controller]) + --export export in yaml format to stdout --interval duration source sync interval (default 1m0s) --kubeconfig string path to the kubeconfig file (default "~/.kube/config") --namespace string the namespace scope for this operation (default "gitops-system") @@ -77,4 +78,4 @@ tk create kustomization [name] [flags] * [tk create](tk_create.md) - Create commands -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_create_source.md b/docs/cmd/tk_create_source.md index 4b235005..4d2559a6 100644 --- a/docs/cmd/tk_create_source.md +++ b/docs/cmd/tk_create_source.md @@ -16,6 +16,7 @@ Create source commands ``` --components strings list of components, accepts comma-separated values (default [source-controller,kustomize-controller]) + --export export in yaml format to stdout --interval duration source sync interval (default 1m0s) --kubeconfig string path to the kubeconfig file (default "~/.kube/config") --namespace string the namespace scope for this operation (default "gitops-system") @@ -28,4 +29,4 @@ Create source commands * [tk create](tk_create.md) - Create commands * [tk create source git](tk_create_source_git.md) - Create or update a git source -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_create_source_git.md b/docs/cmd/tk_create_source_git.md index 2239fe80..b6d2687b 100644 --- a/docs/cmd/tk_create_source_git.md +++ b/docs/cmd/tk_create_source_git.md @@ -31,11 +31,19 @@ tk create source git [name] [flags] --url=https://github.com/stefanprodan/podinfo \ --tag-semver=">=3.2.0 <3.3.0" - # Create a source from a Git repository using SSH authentication + # Create a source from a Git repository using SSH authentication create source git podinfo \ --url=ssh://git@github.com/stefanprodan/podinfo \ --branch=master + # Create a source from a Git repository using SSH authentication and an + # ECDSA P-521 curve public key + create source git podinfo \ + --url=ssh://git@github.com/stefanprodan/podinfo \ + --branch=master \ + --ssh-key-algorithm=ecdsa \ + --ssh-ecdsa-curve=p521 + # Create a source from a Git repository using basic authentication create source git podinfo \ --url=https://github.com/stefanprodan/podinfo \ @@ -47,19 +55,23 @@ tk create source git [name] [flags] ### Options ``` - --branch string git branch (default "master") - -h, --help help for git - -p, --password string basic authentication password - --tag string git tag - --tag-semver string git tag semver range - --url string git address, e.g. ssh://git@host/org/repository - -u, --username string basic authentication username + --branch string git branch (default "master") + -h, --help help for git + -p, --password string basic authentication password + --ssh-ecdsa-curve ecdsaCurve SSH ECDSA public key curve (p521, p256, p384) (default p384) + --ssh-key-algorithm publicKeyAlgorithm SSH public key algorithm (rsa, ecdsa, ed25519) (default rsa) + --ssh-rsa-bits rsaKeyBits SSH RSA public key bit size (multiplies of 8) (default 2048) + --tag string git tag + --tag-semver string git tag semver range + --url string git address, e.g. ssh://git@host/org/repository + -u, --username string basic authentication username ``` ### Options inherited from parent commands ``` --components strings list of components, accepts comma-separated values (default [source-controller,kustomize-controller]) + --export export in yaml format to stdout --interval duration source sync interval (default 1m0s) --kubeconfig string path to the kubeconfig file (default "~/.kube/config") --namespace string the namespace scope for this operation (default "gitops-system") @@ -71,4 +83,4 @@ tk create source git [name] [flags] * [tk create source](tk_create_source.md) - Create source commands -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_delete.md b/docs/cmd/tk_delete.md index 15d4f0fc..09d149fc 100644 --- a/docs/cmd/tk_delete.md +++ b/docs/cmd/tk_delete.md @@ -29,4 +29,4 @@ Delete commands * [tk delete kustomization](tk_delete_kustomization.md) - Delete kustomization * [tk delete source](tk_delete_source.md) - Delete sources commands -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_delete_kustomization.md b/docs/cmd/tk_delete_kustomization.md index fae3cb97..be27ba4a 100644 --- a/docs/cmd/tk_delete_kustomization.md +++ b/docs/cmd/tk_delete_kustomization.md @@ -31,4 +31,4 @@ tk delete kustomization [name] [flags] * [tk delete](tk_delete.md) - Delete commands -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_delete_source.md b/docs/cmd/tk_delete_source.md index 3f48d586..0aaac494 100644 --- a/docs/cmd/tk_delete_source.md +++ b/docs/cmd/tk_delete_source.md @@ -28,4 +28,4 @@ Delete sources commands * [tk delete](tk_delete.md) - Delete commands * [tk delete source git](tk_delete_source_git.md) - Delete git source -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_delete_source_git.md b/docs/cmd/tk_delete_source_git.md index 07e00df0..45b5c40e 100644 --- a/docs/cmd/tk_delete_source_git.md +++ b/docs/cmd/tk_delete_source_git.md @@ -31,4 +31,4 @@ tk delete source git [name] [flags] * [tk delete source](tk_delete_source.md) - Delete sources commands -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_export.md b/docs/cmd/tk_export.md index 3cb7d970..b9675062 100644 --- a/docs/cmd/tk_export.md +++ b/docs/cmd/tk_export.md @@ -29,4 +29,4 @@ Export commands * [tk export kustomization](tk_export_kustomization.md) - Export kustomization in YAML format * [tk export source](tk_export_source.md) - Export source commands -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_export_kustomization.md b/docs/cmd/tk_export_kustomization.md index f04f4bdb..b1383578 100644 --- a/docs/cmd/tk_export_kustomization.md +++ b/docs/cmd/tk_export_kustomization.md @@ -42,4 +42,4 @@ tk export kustomization [name] [flags] * [tk export](tk_export.md) - Export commands -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_export_source.md b/docs/cmd/tk_export_source.md index 126a546c..c9fed0ab 100644 --- a/docs/cmd/tk_export_source.md +++ b/docs/cmd/tk_export_source.md @@ -29,4 +29,4 @@ Export source commands * [tk export](tk_export.md) - Export commands * [tk export source git](tk_export_source_git.md) - Export git sources in YAML format -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_export_source_git.md b/docs/cmd/tk_export_source_git.md index c38d86e6..222aca9e 100644 --- a/docs/cmd/tk_export_source_git.md +++ b/docs/cmd/tk_export_source_git.md @@ -43,4 +43,4 @@ tk export source git [name] [flags] * [tk export source](tk_export_source.md) - Export source commands -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_get.md b/docs/cmd/tk_get.md index 5209fa64..4ed527cf 100644 --- a/docs/cmd/tk_get.md +++ b/docs/cmd/tk_get.md @@ -28,4 +28,4 @@ Get commands * [tk get kustomizations](tk_get_kustomizations.md) - Get kustomizations status * [tk get sources](tk_get_sources.md) - Get sources commands -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_get_kustomizations.md b/docs/cmd/tk_get_kustomizations.md index 8394c27c..e41c66e5 100644 --- a/docs/cmd/tk_get_kustomizations.md +++ b/docs/cmd/tk_get_kustomizations.md @@ -31,4 +31,4 @@ tk get kustomizations [flags] * [tk get](tk_get.md) - Get commands -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_get_sources.md b/docs/cmd/tk_get_sources.md index c29b0930..c5a31c8c 100644 --- a/docs/cmd/tk_get_sources.md +++ b/docs/cmd/tk_get_sources.md @@ -27,4 +27,4 @@ Get sources commands * [tk get](tk_get.md) - Get commands * [tk get sources git](tk_get_sources_git.md) - Get git sources status -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_get_sources_git.md b/docs/cmd/tk_get_sources_git.md index be514577..630ab8c6 100644 --- a/docs/cmd/tk_get_sources_git.md +++ b/docs/cmd/tk_get_sources_git.md @@ -31,4 +31,4 @@ tk get sources git [flags] * [tk get sources](tk_get_sources.md) - Get sources commands -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_install.md b/docs/cmd/tk_install.md index 76318c76..0da21720 100644 --- a/docs/cmd/tk_install.md +++ b/docs/cmd/tk_install.md @@ -49,4 +49,4 @@ tk install [flags] * [tk](tk.md) - Command line utility for assembling Kubernetes CD pipelines -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_resume.md b/docs/cmd/tk_resume.md index 5662caa2..5127bb75 100644 --- a/docs/cmd/tk_resume.md +++ b/docs/cmd/tk_resume.md @@ -27,4 +27,4 @@ Resume commands * [tk](tk.md) - Command line utility for assembling Kubernetes CD pipelines * [tk resume kustomization](tk_resume_kustomization.md) - Resume kustomization -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_resume_kustomization.md b/docs/cmd/tk_resume_kustomization.md index c081538f..a0de59ba 100644 --- a/docs/cmd/tk_resume_kustomization.md +++ b/docs/cmd/tk_resume_kustomization.md @@ -30,4 +30,4 @@ tk resume kustomization [name] [flags] * [tk resume](tk_resume.md) - Resume commands -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_suspend.md b/docs/cmd/tk_suspend.md index 5079105a..b15f0e89 100644 --- a/docs/cmd/tk_suspend.md +++ b/docs/cmd/tk_suspend.md @@ -27,4 +27,4 @@ Suspend commands * [tk](tk.md) - Command line utility for assembling Kubernetes CD pipelines * [tk suspend kustomization](tk_suspend_kustomization.md) - Suspend kustomization -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_suspend_kustomization.md b/docs/cmd/tk_suspend_kustomization.md index 72d905cb..50844a5f 100644 --- a/docs/cmd/tk_suspend_kustomization.md +++ b/docs/cmd/tk_suspend_kustomization.md @@ -30,4 +30,4 @@ tk suspend kustomization [name] [flags] * [tk suspend](tk_suspend.md) - Suspend commands -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_sync.md b/docs/cmd/tk_sync.md index 724e631e..25351c40 100644 --- a/docs/cmd/tk_sync.md +++ b/docs/cmd/tk_sync.md @@ -28,4 +28,4 @@ Synchronize commands * [tk sync kustomization](tk_sync_kustomization.md) - Synchronize kustomization * [tk sync source](tk_sync_source.md) - Synchronize source commands -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_sync_kustomization.md b/docs/cmd/tk_sync_kustomization.md index a3ac33c4..24b06fc8 100644 --- a/docs/cmd/tk_sync_kustomization.md +++ b/docs/cmd/tk_sync_kustomization.md @@ -43,4 +43,4 @@ tk sync kustomization [name] [flags] * [tk sync](tk_sync.md) - Synchronize commands -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_sync_source.md b/docs/cmd/tk_sync_source.md index 0b511215..35ba621f 100644 --- a/docs/cmd/tk_sync_source.md +++ b/docs/cmd/tk_sync_source.md @@ -27,4 +27,4 @@ Synchronize source commands * [tk sync](tk_sync.md) - Synchronize commands * [tk sync source git](tk_sync_source_git.md) - Synchronize git source -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_sync_source_git.md b/docs/cmd/tk_sync_source_git.md index 8a4952b0..a08420c8 100644 --- a/docs/cmd/tk_sync_source_git.md +++ b/docs/cmd/tk_sync_source_git.md @@ -39,4 +39,4 @@ tk sync source git [name] [flags] * [tk sync source](tk_sync_source.md) - Synchronize source commands -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 diff --git a/docs/cmd/tk_uninstall.md b/docs/cmd/tk_uninstall.md index aace41c2..467ddeb3 100644 --- a/docs/cmd/tk_uninstall.md +++ b/docs/cmd/tk_uninstall.md @@ -46,4 +46,4 @@ tk uninstall [flags] * [tk](tk.md) - Command line utility for assembling Kubernetes CD pipelines -###### Auto generated by spf13/cobra on 5-May-2020 +###### Auto generated by spf13/cobra on 9-Jun-2020 From 0cd820e923fdc47f325800b02e10ccc7794afda9 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Tue, 9 Jun 2020 16:22:35 +0300 Subject: [PATCH 11/11] Add export for public git repos --- cmd/tk/create_source_git.go | 48 ++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/cmd/tk/create_source_git.go b/cmd/tk/create_source_git.go index bbad2099..43cdf3c9 100644 --- a/cmd/tk/create_source_git.go +++ b/cmd/tk/create_source_git.go @@ -121,6 +121,32 @@ func createSourceGitCmdRun(cmd *cobra.Command, args []string) error { return err } + gitRepository := sourcev1.GitRepository{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: sourcev1.GitRepositorySpec{ + URL: sourceGitURL, + Interval: metav1.Duration{ + Duration: interval, + }, + Reference: &sourcev1.GitRepositoryRef{}, + }, + } + + if sourceGitSemver != "" { + gitRepository.Spec.Reference.SemVer = sourceGitSemver + } else if sourceGitTag != "" { + gitRepository.Spec.Reference.Tag = sourceGitTag + } else { + gitRepository.Spec.Reference.Branch = sourceGitBranch + } + + if export { + return exportGit(gitRepository) + } + withAuth := false // TODO(hidde): move all auth prep to separate func? if u.Scheme == "ssh" { @@ -187,34 +213,12 @@ func createSourceGitCmdRun(cmd *cobra.Command, args []string) error { logGenerate("generating source") - gitRepository := sourcev1.GitRepository{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: sourcev1.GitRepositorySpec{ - URL: sourceGitURL, - Interval: metav1.Duration{ - Duration: interval, - }, - Reference: &sourcev1.GitRepositoryRef{}, - }, - } - if withAuth { gitRepository.Spec.SecretRef = &corev1.LocalObjectReference{ Name: name, } } - if sourceGitSemver != "" { - gitRepository.Spec.Reference.SemVer = sourceGitSemver - } else if sourceGitTag != "" { - gitRepository.Spec.Reference.Tag = sourceGitTag - } else { - gitRepository.Spec.Reference.Branch = sourceGitBranch - } - logAction("applying source") if err := upsertGitRepository(ctx, kubeClient, gitRepository); err != nil { return err