mirror of
https://github.com/fluxcd/flux2.git
synced 2026-05-23 01:45:53 +00:00
Make SOPS metadata stripping opt-in
Add --strip-sops-metadata flags to build and diff kustomization\ncommands and plumb the option into the build engine.\n\nKeep top-level .sops removal for non-Secret resources disabled by\ndefault, while preserving secret masking behavior.\n\nUpdate unit tests to cover both opt-in strip and default behavior, and\nupdate command tests that assert stripped output. Signed-off-by: Sebastien Tardif <SebTardif@ncf.ca> Assisted-by: GitHub Copilot/GPT-5.3-Codex
This commit is contained in:
Sebastien Tardif
parent
0610c4745c
commit
ce8d638489
5 changed files with 185 additions and 87 deletions
|
|
@ -68,6 +68,7 @@ type buildKsFlags struct {
|
|||
kustomizationFile string
|
||||
path string
|
||||
ignorePaths []string
|
||||
stripSopsMetadata bool
|
||||
dryRun bool
|
||||
strictSubst bool
|
||||
recursive bool
|
||||
|
|
@ -81,6 +82,8 @@ func init() {
|
|||
buildKsCmd.Flags().StringVar(&buildKsArgs.path, "path", "", "Path to the manifests location.")
|
||||
buildKsCmd.Flags().StringVar(&buildKsArgs.kustomizationFile, "kustomization-file", "", "Path to the Flux Kustomization YAML file.")
|
||||
buildKsCmd.Flags().StringSliceVar(&buildKsArgs.ignorePaths, "ignore-paths", nil, "set paths to ignore in .gitignore format")
|
||||
buildKsCmd.Flags().BoolVar(&buildKsArgs.stripSopsMetadata, "strip-sops-metadata", false,
|
||||
"Strip top-level .sops metadata from non-Secret resources in build output.")
|
||||
buildKsCmd.Flags().BoolVar(&buildKsArgs.dryRun, "dry-run", false, "Dry run mode.")
|
||||
buildKsCmd.Flags().BoolVar(&buildKsArgs.strictSubst, "strict-substitute", false,
|
||||
"When enabled, the post build substitutions will fail if a var without a default value is declared in files but is missing from the input vars.")
|
||||
|
|
@ -128,6 +131,7 @@ func buildKsCmdRun(cmd *cobra.Command, args []string) (err error) {
|
|||
build.WithTimeout(rootArgs.timeout),
|
||||
build.WithKustomizationFile(buildKsArgs.kustomizationFile),
|
||||
build.WithDryRun(buildKsArgs.dryRun),
|
||||
build.WithStripSopsMetadata(buildKsArgs.stripSopsMetadata),
|
||||
build.WithNamespace(*kubeconfigArgs.Namespace),
|
||||
build.WithIgnore(buildKsArgs.ignorePaths),
|
||||
build.WithStrictSubstitute(buildKsArgs.strictSubst),
|
||||
|
|
@ -140,6 +144,7 @@ func buildKsCmdRun(cmd *cobra.Command, args []string) (err error) {
|
|||
build.WithClientConfig(kubeconfigArgs, kubeclientOptions),
|
||||
build.WithTimeout(rootArgs.timeout),
|
||||
build.WithKustomizationFile(buildKsArgs.kustomizationFile),
|
||||
build.WithStripSopsMetadata(buildKsArgs.stripSopsMetadata),
|
||||
build.WithIgnore(buildKsArgs.ignorePaths),
|
||||
build.WithStrictSubstitute(buildKsArgs.strictSubst),
|
||||
build.WithRecursive(buildKsArgs.recursive),
|
||||
|
|
|
|||
|
|
@ -213,13 +213,13 @@ spec:
|
|||
},
|
||||
{
|
||||
name: "build helmrelease with sops metadata",
|
||||
args: "build kustomization podinfo --kustomization-file " + tmpFile + " --path ./testdata/build-kustomization/sops-helmrelease",
|
||||
args: "build kustomization podinfo --kustomization-file " + tmpFile + " --path ./testdata/build-kustomization/sops-helmrelease --strip-sops-metadata",
|
||||
resultFile: "./testdata/build-kustomization/sops-helmrelease-result.yaml",
|
||||
assertFunc: "assertGoldenTemplateFile",
|
||||
},
|
||||
{
|
||||
name: "build configmap with sops metadata",
|
||||
args: "build kustomization podinfo --kustomization-file " + tmpFile + " --path ./testdata/build-kustomization/sops-configmap",
|
||||
args: "build kustomization podinfo --kustomization-file " + tmpFile + " --path ./testdata/build-kustomization/sops-configmap --strip-sops-metadata",
|
||||
resultFile: "./testdata/build-kustomization/sops-configmap-result.yaml",
|
||||
assertFunc: "assertGoldenTemplateFile",
|
||||
},
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ type diffKsFlags struct {
|
|||
kustomizationFile string
|
||||
path string
|
||||
ignorePaths []string
|
||||
stripSopsMetadata bool
|
||||
progressBar bool
|
||||
strictSubst bool
|
||||
recursive bool
|
||||
|
|
@ -73,6 +74,8 @@ func init() {
|
|||
diffKsCmd.Flags().BoolVar(&diffKsArgs.progressBar, "progress-bar", true, "Boolean to set the progress bar. The default value is true.")
|
||||
diffKsCmd.Flags().StringSliceVar(&diffKsArgs.ignorePaths, "ignore-paths", nil, "set paths to ignore in .gitignore format")
|
||||
diffKsCmd.Flags().StringVar(&diffKsArgs.kustomizationFile, "kustomization-file", "", "Path to the Flux Kustomization YAML file.")
|
||||
diffKsCmd.Flags().BoolVar(&diffKsArgs.stripSopsMetadata, "strip-sops-metadata", false,
|
||||
"Strip top-level .sops metadata from non-Secret resources in diff output.")
|
||||
diffKsCmd.Flags().BoolVar(&diffKsArgs.strictSubst, "strict-substitute", false,
|
||||
"When enabled, the post build substitutions will fail if a var without a default value is declared in files but is missing from the input vars.")
|
||||
diffKsCmd.Flags().BoolVarP(&diffKsArgs.recursive, "recursive", "r", false, "Recursively diff Kustomizations")
|
||||
|
|
@ -115,6 +118,7 @@ func diffKsCmdRun(cmd *cobra.Command, args []string) error {
|
|||
build.WithKustomizationFile(diffKsArgs.kustomizationFile),
|
||||
build.WithProgressBar(),
|
||||
build.WithIgnore(diffKsArgs.ignorePaths),
|
||||
build.WithStripSopsMetadata(diffKsArgs.stripSopsMetadata),
|
||||
build.WithStrictSubstitute(diffKsArgs.strictSubst),
|
||||
build.WithRecursive(diffKsArgs.recursive),
|
||||
build.WithLocalSources(diffKsArgs.localSources),
|
||||
|
|
@ -128,6 +132,7 @@ func diffKsCmdRun(cmd *cobra.Command, args []string) error {
|
|||
build.WithTimeout(rootArgs.timeout),
|
||||
build.WithKustomizationFile(diffKsArgs.kustomizationFile),
|
||||
build.WithIgnore(diffKsArgs.ignorePaths),
|
||||
build.WithStripSopsMetadata(diffKsArgs.stripSopsMetadata),
|
||||
build.WithStrictSubstitute(diffKsArgs.strictSubst),
|
||||
build.WithRecursive(diffKsArgs.recursive),
|
||||
build.WithLocalSources(diffKsArgs.localSources),
|
||||
|
|
|
|||
|
|
@ -146,6 +146,9 @@ type Builder struct {
|
|||
strictSubst bool
|
||||
recursive bool
|
||||
localSources map[string]string
|
||||
// stripSopsMetadata controls whether top-level .sops metadata is stripped
|
||||
// from non-Secret resources in build/diff output.
|
||||
stripSopsMetadata bool
|
||||
// diff needs to handle kustomizations one by one, and opt-in to ignore kustomizations missing on cluster
|
||||
singleKustomization bool
|
||||
ignoreNotFound bool
|
||||
|
|
@ -256,6 +259,15 @@ func WithLocalSources(localSources map[string]string) BuilderOptionFunc {
|
|||
}
|
||||
}
|
||||
|
||||
// WithStripSopsMetadata enables stripping top-level .sops metadata from
|
||||
// non-Secret resources in build/diff output.
|
||||
func WithStripSopsMetadata(strip bool) BuilderOptionFunc {
|
||||
return func(b *Builder) error {
|
||||
b.stripSopsMetadata = strip
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithInMemoryBuild sets the in-memory build backend
|
||||
func WithInMemoryBuild(inMemoryBuild bool) BuilderOptionFunc {
|
||||
return func(b *Builder) error {
|
||||
|
|
@ -493,7 +505,7 @@ func (b *Builder) build() (m resmap.ResMap, err error) {
|
|||
}
|
||||
|
||||
// make sure secrets are masked
|
||||
err = maskSopsData(res)
|
||||
err = maskSopsData(res, b.stripSopsMetadata)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
|
@ -520,6 +532,7 @@ func (b *Builder) kustomizationBuild(k *kustomizev1.Kustomization) ([]*unstructu
|
|||
WithStrictSubstitute(b.strictSubst),
|
||||
WithRecursive(b.recursive),
|
||||
WithLocalSources(b.localSources),
|
||||
WithStripSopsMetadata(b.stripSopsMetadata),
|
||||
WithDryRun(b.dryRun),
|
||||
withFsBackend(b.fsBackend),
|
||||
)
|
||||
|
|
@ -672,7 +685,7 @@ func (b *Builder) setOwnerLabels(res *resource.Resource) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func maskSopsData(res *resource.Resource) error {
|
||||
func maskSopsData(res *resource.Resource, stripNonSecretSopsMetadata bool) error {
|
||||
// sopsMess is the base64 encoded mask
|
||||
sopsMess := base64.StdEncoding.EncodeToString([]byte(mask))
|
||||
|
||||
|
|
@ -744,7 +757,7 @@ func maskSopsData(res *resource.Resource) error {
|
|||
return fmt.Errorf("failed to mask secret %s sops data: %w", res.GetName(), err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
} else if stripNonSecretSopsMetadata {
|
||||
// For non-Secret resources (e.g. HelmRelease), strip top-level .sops metadata
|
||||
// so it is not persisted in the cluster or exposed in build/diff output.
|
||||
sopsField, err := res.Pipe(yaml.Lookup("sops"))
|
||||
|
|
|
|||
|
|
@ -146,13 +146,16 @@ type: kubernetes.io/dockerconfigjson
|
|||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r, err := yaml.Parse(tc.yamlStr)
|
||||
input := strings.ReplaceAll(tc.yamlStr, "\t", " ")
|
||||
expected := strings.ReplaceAll(tc.expected, "\t", " ")
|
||||
|
||||
r, err := yaml.Parse(input)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to parse yaml: %v", err)
|
||||
}
|
||||
|
||||
resource := &resource.Resource{RNode: *r}
|
||||
err = maskSopsData(resource)
|
||||
err = maskSopsData(resource, true)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to trim sops data: %v", err)
|
||||
}
|
||||
|
|
@ -161,7 +164,7 @@ type: kubernetes.io/dockerconfigjson
|
|||
if err != nil {
|
||||
t.Fatalf("unable to convert sanitized resources to yaml: %v", err)
|
||||
}
|
||||
if diff := cmp.Diff(string(sYaml), tc.expected); diff != "" {
|
||||
if diff := cmp.Diff(string(sYaml), expected); diff != "" {
|
||||
t.Errorf("unexpected sanitized resources: (-got +want)%v", diff)
|
||||
}
|
||||
})
|
||||
|
|
@ -172,133 +175,205 @@ func TestMaskSopsDataNonSecret(t *testing.T) {
|
|||
testCases := []struct {
|
||||
name string
|
||||
yamlStr string
|
||||
strip bool
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
// A SOPS-encrypted HelmRelease (values block encrypted) must have its
|
||||
// .sops metadata stripped so it is safe for build/diff output and does
|
||||
// not cause a server-side apply schema error.
|
||||
name: "HelmRelease with sops metadata",
|
||||
name: "HelmRelease with sops metadata and opt-in strip",
|
||||
strip: true,
|
||||
yamlStr: `apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: mysql
|
||||
namespace: default
|
||||
name: mysql
|
||||
namespace: default
|
||||
spec:
|
||||
chart:
|
||||
spec:
|
||||
chart: mysql
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: bitnami
|
||||
values:
|
||||
mysql:
|
||||
rootPassword: ENC[AES256_GCM,data:abc123,iv:xyz,tag:tag,type:str]
|
||||
replicationPassword: ENC[AES256_GCM,data:def456,iv:xyz,tag:tag,type:str]
|
||||
chart:
|
||||
spec:
|
||||
chart: mysql
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: bitnami
|
||||
values:
|
||||
mysql:
|
||||
rootPassword: ENC[AES256_GCM,data:abc123,iv:xyz,tag:tag,type:str]
|
||||
replicationPassword: ENC[AES256_GCM,data:def456,iv:xyz,tag:tag,type:str]
|
||||
sops:
|
||||
kms: []
|
||||
gcp_kms: []
|
||||
azure_kv: []
|
||||
hc_vault: []
|
||||
age:
|
||||
- recipient: age10la2ge0wtvx3qr7datqf7rs4yngxszdal927fs9rukamr8u2pshsvtz7ce
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
abc
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2023-07-15T00:00:00Z"
|
||||
mac: ENC[AES256_GCM,data:mac,iv:iv,tag:tag,type:str]
|
||||
encrypted_regex: ^(values)$
|
||||
version: 3.7.3
|
||||
kms: []
|
||||
gcp_kms: []
|
||||
azure_kv: []
|
||||
hc_vault: []
|
||||
age:
|
||||
- recipient: age10la2ge0wtvx3qr7datqf7rs4yngxszdal927fs9rukamr8u2pshsvtz7ce
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
abc
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2023-07-15T00:00:00Z"
|
||||
mac: ENC[AES256_GCM,data:mac,iv:iv,tag:tag,type:str]
|
||||
encrypted_regex: ^(values)$
|
||||
version: 3.7.3
|
||||
`,
|
||||
expected: `apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: mysql
|
||||
namespace: default
|
||||
name: mysql
|
||||
namespace: default
|
||||
spec:
|
||||
chart:
|
||||
spec:
|
||||
chart: mysql
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: bitnami
|
||||
values:
|
||||
mysql:
|
||||
replicationPassword: ENC[AES256_GCM,data:def456,iv:xyz,tag:tag,type:str]
|
||||
rootPassword: ENC[AES256_GCM,data:abc123,iv:xyz,tag:tag,type:str]
|
||||
chart:
|
||||
spec:
|
||||
chart: mysql
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: bitnami
|
||||
values:
|
||||
mysql:
|
||||
replicationPassword: ENC[AES256_GCM,data:def456,iv:xyz,tag:tag,type:str]
|
||||
rootPassword: ENC[AES256_GCM,data:abc123,iv:xyz,tag:tag,type:str]
|
||||
`,
|
||||
},
|
||||
{
|
||||
// A HelmRelease without any SOPS metadata must pass through unchanged.
|
||||
name: "HelmRelease without sops metadata",
|
||||
name: "HelmRelease without sops metadata",
|
||||
strip: true,
|
||||
yamlStr: `apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: podinfo
|
||||
namespace: default
|
||||
name: podinfo
|
||||
namespace: default
|
||||
spec:
|
||||
chart:
|
||||
spec:
|
||||
chart: podinfo
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: podinfo
|
||||
values:
|
||||
replicaCount: 2
|
||||
chart:
|
||||
spec:
|
||||
chart: podinfo
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: podinfo
|
||||
values:
|
||||
replicaCount: 2
|
||||
`,
|
||||
expected: `apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: podinfo
|
||||
namespace: default
|
||||
name: podinfo
|
||||
namespace: default
|
||||
spec:
|
||||
chart:
|
||||
spec:
|
||||
chart: podinfo
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: podinfo
|
||||
values:
|
||||
replicaCount: 2
|
||||
chart:
|
||||
spec:
|
||||
chart: podinfo
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: podinfo
|
||||
values:
|
||||
replicaCount: 2
|
||||
`,
|
||||
},
|
||||
{
|
||||
// Strip top-level sops metadata whenever present, even if mac is absent.
|
||||
name: "ConfigMap with top-level sops but no mac",
|
||||
name: "ConfigMap with top-level sops and opt-in strip",
|
||||
strip: true,
|
||||
yamlStr: `apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: app-config
|
||||
namespace: default
|
||||
name: app-config
|
||||
namespace: default
|
||||
data:
|
||||
values.yaml: |
|
||||
hello: world
|
||||
values.yaml: |
|
||||
hello: world
|
||||
sops:
|
||||
version: 3.7.0
|
||||
encrypted_regex: ^(data)$
|
||||
version: 3.7.0
|
||||
encrypted_regex: ^(data)$
|
||||
`,
|
||||
expected: `apiVersion: v1
|
||||
data:
|
||||
values.yaml: |
|
||||
hello: world
|
||||
values.yaml: |
|
||||
hello: world
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: app-config
|
||||
namespace: default
|
||||
name: app-config
|
||||
namespace: default
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: "HelmRelease with sops metadata without opt-in strip",
|
||||
strip: false,
|
||||
yamlStr: `apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: mysql
|
||||
namespace: default
|
||||
spec:
|
||||
chart:
|
||||
spec:
|
||||
chart: mysql
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: bitnami
|
||||
values:
|
||||
mysql:
|
||||
rootPassword: ENC[AES256_GCM,data:abc123,iv:xyz,tag:tag,type:str]
|
||||
replicationPassword: ENC[AES256_GCM,data:def456,iv:xyz,tag:tag,type:str]
|
||||
sops:
|
||||
version: 3.7.3
|
||||
`,
|
||||
expected: `apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: mysql
|
||||
namespace: default
|
||||
sops:
|
||||
version: 3.7.3
|
||||
spec:
|
||||
chart:
|
||||
spec:
|
||||
chart: mysql
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: bitnami
|
||||
values:
|
||||
mysql:
|
||||
replicationPassword: ENC[AES256_GCM,data:def456,iv:xyz,tag:tag,type:str]
|
||||
rootPassword: ENC[AES256_GCM,data:abc123,iv:xyz,tag:tag,type:str]
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: "ConfigMap with top-level sops without opt-in strip",
|
||||
strip: false,
|
||||
yamlStr: `apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: app-config
|
||||
namespace: default
|
||||
data:
|
||||
values.yaml: |
|
||||
hello: world
|
||||
sops:
|
||||
version: 3.7.0
|
||||
encrypted_regex: ^(data)$
|
||||
`,
|
||||
expected: `apiVersion: v1
|
||||
data:
|
||||
values.yaml: |
|
||||
hello: world
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: app-config
|
||||
namespace: default
|
||||
sops:
|
||||
encrypted_regex: ^(data)$
|
||||
version: 3.7.0
|
||||
`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r, err := yaml.Parse(tc.yamlStr)
|
||||
input := strings.ReplaceAll(tc.yamlStr, "\t", " ")
|
||||
expected := strings.ReplaceAll(tc.expected, "\t", " ")
|
||||
|
||||
r, err := yaml.Parse(input)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to parse yaml: %v", err)
|
||||
}
|
||||
|
||||
res := &resource.Resource{RNode: *r}
|
||||
if err := maskSopsData(res); err != nil {
|
||||
if err := maskSopsData(res, tc.strip); err != nil {
|
||||
t.Fatalf("maskSopsData returned unexpected error: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -306,7 +381,7 @@ metadata:
|
|||
if err != nil {
|
||||
t.Fatalf("unable to convert resource to yaml: %v", err)
|
||||
}
|
||||
if diff := cmp.Diff(string(got), tc.expected); diff != "" {
|
||||
if diff := cmp.Diff(string(got), expected); diff != "" {
|
||||
t.Errorf("unexpected output (-got +want):\n%v", diff)
|
||||
}
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue