feat: add skip_path option in watch

As part of #20, we want to be able to negate specific paths in a given
`watch` path by using `skip_path`.

This PR adds support for specifying a `skip_path` option that support
either a path or a list of paths that shouldn't result in adding the
`trigger` step as part of the dynamically generated pipeline.

Note even in cases where a line in the `diff` is skipped due to
`skip_path` and there are other paths in the `diff` that match `path`
but doesn't match `skip_path`, the step will still be added to the
dynamically generated pipeline. i.e. for the step to be effectively
ignored, the paths in `skip_path` should match all of those in the given
`diff`.

This is required due to the lack of support for negation cases in
`doublestar`, which we use for glob pattern matching[0].

Closes #20.

[0]: bmatcuk/doublestar#49

README.md

@@ -27,6 +27,12 @@ It defines a list of paths or path to monitor for changes in the monorepo. It ch
 
 A path or a list of paths to be watched, This part specifies which directory should be monitored. It can also be a glob pattern. For example specify `path: "**/*.md"` to match all markdown files. A list of paths can be provided to trigger the desired pipeline or run command or even do a pipeline upload.
 
+#### `skip_path`
+
+A path or a list of paths to be ignored, which can be an exact path, or a glob.
+
+This is intended to be used in conjunction with `path`, and allows omitting specific paths from being matched.
+
 #### `config`
 
 This is a sub-section that provides configuration for running commands or triggering another pipeline when changes occur in the specified path

pipeline.go

@@ -112,10 +112,25 @@ func stepsToTrigger(files []string, watch []WatchConfig) ([]Step, error) {
 		for _, p := range w.Paths {
 			for _, f := range files {
 				match, err := matchPath(p, f)
+
+				skip := false
+				for _, sp := range w.SkipPaths {
+					skipMatch, errSkip := matchPath(sp, f)
+
+					if errSkip != nil {
+						return nil, errSkip
+					}
+
+					if skipMatch {
+						skip = true
+					}
+				}
+
 				if err != nil {
 					return nil, err
 				}
-				if match {
+
+				if match && !skip {
 					steps = append(steps, w.Step)
 					break
 				}

pipeline_test.go

@@ -251,7 +251,93 @@ func TestPipelinesStepsToTrigger(t *testing.T) {
 				{Command: "buildkite-agent pipeline upload other_tests.yml"},
 			},
 		},
+		"skips service-2": {
+			ChangedFiles: []string{
+				"watch-path/text.txt",
+			},
+			WatchConfigs: []WatchConfig{
+				{
+					Paths: []string{"watch-path"},
+					Step:  Step{Trigger: "service-1"},
+				},
+				{
+					Paths:     []string{"watch-path"},
+					SkipPaths: []string{"watch-path/text.txt"},
+					Step:      Step{Trigger: "service-2"},
+				}},
+			Expected: []Step{
+				{Trigger: "service-1"},
+			},
+		},
+		"skips extension wildcard": {
+			ChangedFiles: []string{
+				"text.secret.txt",
+			},
+			WatchConfigs: []WatchConfig{
+				{
+					Paths: []string{"*.txt"},
+					Step:  Step{Trigger: "service-1"},
+				},
+				{
+					Paths:     []string{"*.txt"},
+					SkipPaths: []string{"*.secret.txt"},
+					Step:      Step{Trigger: "service-2"},
+				}},
+			Expected: []Step{
+				{Trigger: "service-1"},
+			},
+		},
+		"skips extension wildcard in subdir": {
+			ChangedFiles: []string{
+				"docs/text.secret.txt",
+			},
+			WatchConfigs: []WatchConfig{
+				{
+					Paths: []string{"**/*.txt"},
+					Step:  Step{Trigger: "service-1"},
+				},
+				{
+					Paths:     []string{"**/*.txt"},
+					SkipPaths: []string{"docs/*.txt"},
+					Step:      Step{Trigger: "service-2"},
+				}},
+			Expected: []Step{
+				{Trigger: "service-1"},
+			},
+		},
+		"step is included even when one of the files is skipped": {
+			ChangedFiles: []string{
+				"docs/text.secret.txt",
+				"docs/text.txt",
+			},
+			WatchConfigs: []WatchConfig{
+				{
+					Paths: []string{"**/*.txt"},
+					Step:  Step{Trigger: "service-1"},
+				},
+				{
+					Paths:     []string{"**/*.txt"},
+					SkipPaths: []string{"docs/*.secret.txt"},
+					Step:      Step{Trigger: "service-2"},
+				}},
+			Expected: []Step{
+				{Trigger: "service-1"},
+				{Trigger: "service-2"},
+			},
+		},
+		"fails if not path is included": {
+			ChangedFiles: []string{
+				"docs/text.txt",
+			},
+			WatchConfigs: []WatchConfig{
+				{
+					SkipPaths: []string{"docs/*.secret.txt"},
+					Step:      Step{Trigger: "service-1"},
+				}},
+			Expected: []Step{},
+		},
 	}
+
 	for name, tc := range testCases {
 		t.Run(name, func(t *testing.T) {
 			steps, err := stepsToTrigger(tc.ChangedFiles, tc.WatchConfigs)

plugin.go

@@ -31,10 +31,12 @@ type HookConfig struct {
 
 // WatchConfig Plugin watch configuration
 type WatchConfig struct {
-	RawPath interface{} `json:"path"`
-	Paths   []string
-	Step    Step        `json:"config"`
-	Default interface{} `json:"default"`
+	RawPath     interface{} `json:"path"`
+	Paths       []string
+	Step        Step        `json:"config"`
+	Default     interface{} `json:"default"`
+	RawSkipPath interface{} `json:"skip_path"`
+	SkipPaths   []string
 }
 
 type Group struct {
@@ -122,8 +124,8 @@ func (plugin *Plugin) UnmarshalJSON(data []byte) error {
 
 	setPluginNotify(&plugin.Notify, &plugin.RawNotify)
 
-	// Path can be string or an array of strings,
-	// handle both cases and create an array of paths.
+	// Path and SkipPath can be string or an array of strings,
+	// handle both cases and create an array of paths on both.
 	for i, p := range plugin.Watch {
 		if p.Default != nil {
 			plugin.Watch[i].Paths = []string{}
@@ -146,6 +148,15 @@ func (plugin *Plugin) UnmarshalJSON(data []byte) error {
 			}
 		}
 
+		switch p.RawSkipPath.(type) {
+		case string:
+			plugin.Watch[i].SkipPaths = []string{plugin.Watch[i].RawSkipPath.(string)}
+		case []interface{}:
+			for _, v := range plugin.Watch[i].RawSkipPath.([]interface{}) {
+				plugin.Watch[i].SkipPaths = append(plugin.Watch[i].SkipPaths, v.(string))
+			}
+		}
+
 		if plugin.Watch[i].Step.Trigger != "" {
 			setBuild(&plugin.Watch[i].Step.Build)
 		}
@@ -157,6 +168,7 @@ func (plugin *Plugin) UnmarshalJSON(data []byte) error {
 		appendEnv(&plugin.Watch[i], plugin.Env)
 
 		p.RawPath = nil
+		p.RawSkipPath = nil
 	}
 
 	return nil
@@ -313,6 +325,7 @@ func appendEnv(watch *WatchConfig, env map[string]string) {
 	watch.Step.RawEnv = nil
 	watch.Step.Build.RawEnv = nil
 	watch.RawPath = nil
+	watch.RawSkipPath = nil
 }
 
 // parse env in format from env=env-value to map[env] = env-value

tests/command.bats

@@ -242,6 +242,23 @@ EOM
           "config": {
             "command": ["echo one", "echo two"]
           }
+        },
+        {
+          "path": "**/*.md",
+          "skip_path": "**/*.secret.md",
+          "config": {
+            "trigger": "markdown-pipeline"
+          }
+        },
+        {
+          "path": "**/*.md",
+          "skip_path": [
+            "**/*.secret.md",
+            "CONTRIBUTING.md",
+          ],
+          "config": {
+            "trigger": "markdown-pipeline"
+          }
         }
       ]
     }