From 709ec5a52ceae189db5d37e2d6b6473192c29314 Mon Sep 17 00:00:00 2001 From: Cyril Jouve Date: Tue, 29 Aug 2023 23:31:16 +0200 Subject: [PATCH 1/2] use .helmignore when identifying changed charts Signed-off-by: Cyril Jouve --- ct/cmd/root.go | 1 + doc/ct_install.md | 1 + doc/ct_lint-and-install.md | 1 + doc/ct_lint.md | 1 + doc/ct_list-changed.md | 1 + go.mod | 2 + go.sum | 8 +++- pkg/chart/chart.go | 32 +++++++++++--- pkg/chart/chart_test.go | 47 +++++++++++++++++++++ pkg/config/config.go | 1 + pkg/config/config_test.go | 1 + pkg/config/test_config.json | 3 +- pkg/config/test_config.yaml | 1 + pkg/ignore/ignore.go | 83 +++++++++++++++++++++++++++++++++++++ pkg/ignore/ignore_test.go | 19 +++++++++ 15 files changed, 194 insertions(+), 8 deletions(-) create mode 100644 pkg/ignore/ignore.go create mode 100644 pkg/ignore/ignore_test.go diff --git a/ct/cmd/root.go b/ct/cmd/root.go index 16b40485..3afb2621 100644 --- a/ct/cmd/root.go +++ b/ct/cmd/root.go @@ -80,6 +80,7 @@ func addCommonFlags(flags *pflag.FlagSet) { flags.Bool("github-groups", false, heredoc.Doc(` Change the delimiters for github to create collapsible groups for command output`)) + flags.Bool("use-helmignore", false, "Use .helmignore when identifying changed charts") } func addCommonLintAndInstallFlags(flags *pflag.FlagSet) { diff --git a/doc/ct_install.md b/doc/ct_install.md index 355a14c3..fe1e72d3 100644 --- a/doc/ct_install.md +++ b/doc/ct_install.md @@ -77,6 +77,7 @@ ct install [flags] --target-branch string The name of the target branch used to identify changed charts (default "main") --upgrade Whether to test an in-place upgrade of each chart from its previous revision if the current version should not introduce a breaking change according to the SemVer spec + --use-helmignore Use .helmignore when identifying changed charts ``` ### SEE ALSO diff --git a/doc/ct_lint-and-install.md b/doc/ct_lint-and-install.md index 765448fc..215a70fe 100644 --- a/doc/ct_lint-and-install.md +++ b/doc/ct_lint-and-install.md @@ -72,6 +72,7 @@ ct lint-and-install [flags] --target-branch string The name of the target branch used to identify changed charts (default "main") --upgrade Whether to test an in-place upgrade of each chart from its previous revision if the current version should not introduce a breaking change according to the SemVer spec + --use-helmignore Use .helmignore when identifying changed charts --validate-chart-schema Enable schema validation of 'Chart.yaml' using Yamale (default true) --validate-maintainers Enable validation of maintainer account names in chart.yml. Works for GitHub, GitLab, and Bitbucket (default true) diff --git a/doc/ct_lint.md b/doc/ct_lint.md index 5fb4ef62..a08644aa 100644 --- a/doc/ct_lint.md +++ b/doc/ct_lint.md @@ -70,6 +70,7 @@ ct lint [flags] --remote string The name of the Git remote used to identify changed charts (default "origin") --since string The Git reference used to identify changed charts (default "HEAD") --target-branch string The name of the target branch used to identify changed charts (default "main") + --use-helmignore Use .helmignore when identifying changed charts --validate-chart-schema Enable schema validation of 'Chart.yaml' using Yamale (default true) --validate-maintainers Enable validation of maintainer account names in chart.yml. Works for GitHub, GitLab, and Bitbucket (default true) diff --git a/doc/ct_list-changed.md b/doc/ct_list-changed.md index 5080a62f..109f6c49 100644 --- a/doc/ct_list-changed.md +++ b/doc/ct_list-changed.md @@ -28,6 +28,7 @@ ct list-changed [flags] --remote string The name of the Git remote used to identify changed charts (default "origin") --since string The Git reference used to identify changed charts (default "HEAD") --target-branch string The name of the target branch used to identify changed charts (default "main") + --use-helmignore Use .helmignore when identifying changed charts ``` ### SEE ALSO diff --git a/go.mod b/go.mod index 24542c46..4e12bc7b 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/spf13/viper v1.18.2 github.com/stretchr/testify v1.9.0 gopkg.in/yaml.v2 v2.4.0 + helm.sh/helm/v3 v3.14.4 ) require ( @@ -27,6 +28,7 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect diff --git a/go.sum b/go.sum index 6992ee6c..18a1a42e 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -53,6 +53,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -110,3 +112,5 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +helm.sh/helm/v3 v3.14.4 h1:6FSpEfqyDalHq3kUr4gOMThhgY55kXUEjdQoyODYnrM= +helm.sh/helm/v3 v3.14.4/go.mod h1:Tje7LL4gprZpuBNTbG34d1Xn5NmRT3OWfBRwpOSer9I= diff --git a/pkg/chart/chart.go b/pkg/chart/chart.go index 4e2f3e10..4790b825 100644 --- a/pkg/chart/chart.go +++ b/pkg/chart/chart.go @@ -22,9 +22,11 @@ import ( "strings" "github.com/Masterminds/semver" + helmignore "helm.sh/helm/v3/pkg/ignore" "github.com/helm/chart-testing/v3/pkg/config" "github.com/helm/chart-testing/v3/pkg/exec" + "github.com/helm/chart-testing/v3/pkg/ignore" "github.com/helm/chart-testing/v3/pkg/tool" "github.com/helm/chart-testing/v3/pkg/util" ) @@ -242,6 +244,7 @@ type Testing struct { directoryLister DirectoryLister utils Utils previousRevisionWorktree string + loadRules func(string) (*helmignore.Rules, error) } // TestResults holds results and overall status @@ -272,6 +275,7 @@ func NewTesting(config config.Configuration, extraSetArgs string) (Testing, erro accountValidator: tool.AccountValidator{}, directoryLister: util.DirectoryLister{}, utils: util.Utils{}, + loadRules: ignore.LoadRules, } versionString, err := testing.helm.Version() @@ -746,7 +750,7 @@ func (t *Testing) ComputeChangedChartDirectories() ([]string, error) { return nil, fmt.Errorf("failed creating diff: %w", err) } - var changedChartDirs []string + changedChartFiles := map[string][]string{} for _, file := range allChangedChartFiles { pathElements := strings.SplitN(filepath.ToSlash(file), "/", 3) if len(pathElements) < 2 || util.StringSliceContains(cfg.ExcludedCharts, pathElements[1]) { @@ -763,15 +767,33 @@ func (t *Testing) ComputeChangedChartDirectories() ([]string, error) { continue } } - // Only add it if not already in the list - if !util.StringSliceContains(changedChartDirs, chartDir) { - changedChartDirs = append(changedChartDirs, chartDir) - } + changedChartFiles[chartDir] = append(changedChartFiles[chartDir], strings.TrimPrefix(file, chartDir+"/")) } else { fmt.Fprintf(os.Stderr, "Directory %q is not a valid chart directory. Skipping...\n", dir) } } + changedChartDirs := []string{} + if t.config.UseHelmignore { + for chartDir, changedChartFiles := range changedChartFiles { + rules, err := t.loadRules(chartDir) + if err != nil { + return nil, err + } + filteredChartFiles, err := ignore.FilterFiles(changedChartFiles, rules) + if err != nil { + return nil, err + } + if len(filteredChartFiles) > 0 { + changedChartDirs = append(changedChartDirs, chartDir) + } + } + } else { + for chartDir := range changedChartFiles { + changedChartDirs = append(changedChartDirs, chartDir) + } + } + return changedChartDirs, nil } diff --git a/pkg/chart/chart_test.go b/pkg/chart/chart_test.go index a45ff829..678bb801 100644 --- a/pkg/chart/chart_test.go +++ b/pkg/chart/chart_test.go @@ -23,6 +23,7 @@ import ( "github.com/helm/chart-testing/v3/pkg/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + helmignore "helm.sh/helm/v3/pkg/ignore" ) type fakeGit struct{} @@ -152,6 +153,26 @@ func newTestingMock(cfg config.Configuration) Testing { accountValidator: fakeAccountValidator{}, linter: fakeMockLinter, helm: new(fakeHelm), + loadRules: func(dir string) (*helmignore.Rules, error) { + rules := helmignore.Empty() + if dir == "test_charts/foo" { + var err error + rules, err = helmignore.Parse(strings.NewReader("Chart.yaml\n")) + if err != nil { + return nil, err + } + rules.AddDefaults() + } + if dir == "test_chart_at_multi_level/foo/baz" { + var err error + rules, err = helmignore.Parse(strings.NewReader("Chart.yaml\n")) + if err != nil { + return nil, err + } + rules.AddDefaults() + } + return rules, nil + }, } } @@ -165,6 +186,19 @@ func TestComputeChangedChartDirectories(t *testing.T) { assert.Nil(t, err) } +func TestComputeChangedChartDirectoriesWithHelmignore(t *testing.T) { + cfg := config.Configuration{ + ExcludedCharts: []string{"excluded"}, + ChartDirs: []string{"test_charts", "."}, + UseHelmignore: true, + } + ct := newTestingMock(cfg) + actual, err := ct.ComputeChangedChartDirectories() + expected := []string{"test_charts/bar", "test_chart_at_root"} + assert.Nil(t, err) + assert.ElementsMatch(t, expected, actual) +} + func TestComputeChangedChartDirectoriesWithMultiLevelChart(t *testing.T) { cfg := config.Configuration{ ExcludedCharts: []string{"excluded"}, @@ -180,6 +214,19 @@ func TestComputeChangedChartDirectoriesWithMultiLevelChart(t *testing.T) { assert.Nil(t, err) } +func TestComputeChangedChartDirectoriesWithMultiLevelChartWithHelmIgnore(t *testing.T) { + cfg := config.Configuration{ + ExcludedCharts: []string{"excluded"}, + ChartDirs: []string{"test_chart_at_multi_level/foo"}, + UseHelmignore: true, + } + ct := newTestingMock(cfg) + actual, err := ct.ComputeChangedChartDirectories() + expected := []string{"test_chart_at_multi_level/foo/bar"} + assert.Nil(t, err) + assert.ElementsMatch(t, expected, actual) +} + func TestReadAllChartDirectories(t *testing.T) { actual, err := ct.ReadAllChartDirectories() expected := []string{ diff --git a/pkg/config/config.go b/pkg/config/config.go index c6b82874..989c467b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -73,6 +73,7 @@ type Configuration struct { KubectlTimeout time.Duration `mapstructure:"kubectl-timeout"` PrintLogs bool `mapstructure:"print-logs"` GithubGroups bool `mapstructure:"github-groups"` + UseHelmignore bool `mapstructure:"use-helmignore"` } func LoadConfiguration(cfgFile string, cmd *cobra.Command, printConfig bool) (*Configuration, error) { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 80b33f0e..f07d912e 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -61,6 +61,7 @@ func loadAndAssertConfigFromFile(t *testing.T, configFile string) { require.Equal(t, true, cfg.ExcludeDeprecated) require.Equal(t, 120*time.Second, cfg.KubectlTimeout) require.Equal(t, true, cfg.SkipCleanUp) + require.Equal(t, true, cfg.UseHelmignore) } func Test_findConfigFile(t *testing.T) { diff --git a/pkg/config/test_config.json b/pkg/config/test_config.json index 4dc445a8..73cd4574 100644 --- a/pkg/config/test_config.json +++ b/pkg/config/test_config.json @@ -32,5 +32,6 @@ "release-label": "release", "exclude-deprecated": true, "kubectl-timeout": "120s", - "skip-clean-up": true + "skip-clean-up": true, + "use-helmignore": true } diff --git a/pkg/config/test_config.yaml b/pkg/config/test_config.yaml index 6fe391de..0d1c7c02 100644 --- a/pkg/config/test_config.yaml +++ b/pkg/config/test_config.yaml @@ -28,3 +28,4 @@ release-label: release exclude-deprecated: true kubectl-timeout: 120s skip-clean-up: true +use-helmignore: true diff --git a/pkg/ignore/ignore.go b/pkg/ignore/ignore.go new file mode 100644 index 00000000..99d19c2c --- /dev/null +++ b/pkg/ignore/ignore.go @@ -0,0 +1,83 @@ +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ignore + +import ( + "io/fs" + "os" + "path/filepath" + "testing/fstest" + + helmignore "helm.sh/helm/v3/pkg/ignore" +) + +func LoadRules(dir string) (*helmignore.Rules, error) { + rules, err := helmignore.ParseFile(filepath.Join(dir, helmignore.HelmIgnore)) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + if rules == nil { + rules = helmignore.Empty() + } + rules.AddDefaults() + return rules, nil +} + +func FilterFiles(files []string, rules *helmignore.Rules) ([]string, error) { + fsys := fstest.MapFS{} + for _, file := range files { + fsys[file] = &fstest.MapFile{} + } + + filteredFiles := []string{} + + err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + fi, err := d.Info() + if err != nil { + return err + } + + // Normalize to / since it will also work on Windows + path = filepath.ToSlash(path) + + if fi.IsDir() { + // Directory-based ignore rules should involve skipping the entire + // contents of that directory. + if rules.Ignore(path, fi) { + return filepath.SkipDir + } + return nil + } + + // If a .helmignore file matches, skip this file. + if rules.Ignore(path, fi) { + return nil + } + + filteredFiles = append(filteredFiles, path) + return nil + }) + if err != nil { + return nil, err + } + + return filteredFiles, nil +} diff --git a/pkg/ignore/ignore_test.go b/pkg/ignore/ignore_test.go new file mode 100644 index 00000000..b1e94349 --- /dev/null +++ b/pkg/ignore/ignore_test.go @@ -0,0 +1,19 @@ +package ignore + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + helmignore "helm.sh/helm/v3/pkg/ignore" +) + +func TestFilter(t *testing.T) { + rules, err := helmignore.Parse(strings.NewReader("/bar/\nREADME.md\n")) + assert.Nil(t, err) + files := []string{"Chart.yaml", "bar/xxx", "template/svc.yaml", "baz/bar/biz.txt", "README.md"} + actual, err := FilterFiles(files, rules) + assert.Nil(t, err) + expected := []string{"Chart.yaml", "baz/bar/biz.txt", "template/svc.yaml"} + assert.ElementsMatch(t, expected, actual) +} From be92a0d156504aa091f486fbb227fcc38eed658a Mon Sep 17 00:00:00 2001 From: Cyril Jouve Date: Tue, 29 Aug 2023 23:31:16 +0200 Subject: [PATCH 2/2] use .helmignore when identifying changed charts Signed-off-by: Cyril Jouve --- pkg/ignore/ignore_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/ignore/ignore_test.go b/pkg/ignore/ignore_test.go index b1e94349..4976b1fa 100644 --- a/pkg/ignore/ignore_test.go +++ b/pkg/ignore/ignore_test.go @@ -1,3 +1,19 @@ +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package ignore import (