Refactor cli command layout

Having each subcommand defined similarly will hopefully make the
common root level flag overrides easier to implement.

.gitignore

@@ -8,4 +8,5 @@ TODO.md
 release_notes.md
 .idea
 .vscode
-.op
+.op
+benthos

Makefile

@@ -2,6 +2,12 @@
 
 GOMAXPROCS ?= 1
 
+build:
+	@go build ./cmd/benthos
+
+install:
+	@go install ./cmd/benthos
+
 deps:
 	@go mod tidy
 

internal/cli/app.go

@@ -0,0 +1,172 @@
+package cli
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"runtime/debug"
+	"strings"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/redpanda-data/benthos/v4/internal/cli/blobl"
+	"github.com/redpanda-data/benthos/v4/internal/cli/common"
+	"github.com/redpanda-data/benthos/v4/internal/cli/studio"
+	clitemplate "github.com/redpanda-data/benthos/v4/internal/cli/template"
+	"github.com/redpanda-data/benthos/v4/internal/cli/test"
+)
+
+// Build stamps.
+var (
+	Version   = "unknown"
+	DateBuilt = "unknown"
+)
+
+func init() {
+	if Version != "unknown" {
+		return
+	}
+	if info, ok := debug.ReadBuildInfo(); ok {
+		for _, mod := range info.Deps {
+			if mod.Path == "github.com/redpanda-data/benthos/v4" {
+				if mod.Version != "(devel)" {
+					Version = mod.Version
+				}
+				if mod.Replace != nil {
+					v := mod.Replace.Version
+					if v != "" && v != "(devel)" {
+						Version = v
+					}
+				}
+			}
+		}
+		for _, s := range info.Settings {
+			if s.Key == "vcs.revision" && Version == "unknown" {
+				Version = s.Value
+			}
+			if s.Key == "vcs.time" && DateBuilt == "unknown" {
+				DateBuilt = s.Value
+			}
+		}
+	}
+}
+
+//------------------------------------------------------------------------------
+
+type pluginHelp struct {
+	Path  string   `json:"path,omitempty"`
+	Short string   `json:"short,omitempty"`
+	Long  string   `json:"long,omitempty"`
+	Args  []string `json:"args,omitempty"`
+}
+
+// In support of --help-autocomplete.
+func traverseHelp(cmd *cli.Command, pieces []string) []pluginHelp {
+	pieces = append(pieces, cmd.Name)
+	var args []string
+	for _, a := range cmd.Flags {
+		for _, n := range a.Names() {
+			if len(n) > 1 {
+				args = append(args, "--"+n)
+			} else {
+				args = append(args, "-"+n)
+			}
+		}
+	}
+	help := []pluginHelp{{
+		Path:  strings.Join(pieces, "_"),
+		Short: cmd.Usage,
+		Long:  cmd.Description,
+		Args:  args,
+	}}
+	for _, cmd := range cmd.Subcommands {
+		if cmd.Hidden {
+			continue
+		}
+		help = append(help, traverseHelp(cmd, pieces)...)
+	}
+	return help
+}
+
+// App returns the full CLI app definition, this is useful for writing unit
+// tests around the CLI.
+func App(opts *common.CLIOpts) *cli.App {
+	flags := []cli.Flag{
+		&cli.BoolFlag{
+			Name:    "version",
+			Aliases: []string{"v"},
+			Value:   false,
+			Usage:   "display version info, then exit",
+		},
+		&cli.BoolFlag{
+			Name:   "help-autocomplete",
+			Value:  false,
+			Usage:  "print json serialised cli argument definitions to assist with autocomplete",
+			Hidden: true,
+		},
+		&cli.StringFlag{
+			Name:    "config",
+			Aliases: []string{"c"},
+			Hidden:  true,
+			Value:   "",
+			Usage:   "a path to a configuration file",
+		},
+	}
+	flags = append(flags, common.RunFlags(opts, true)...)
+	flags = append(flags, common.EnvFileAndTemplateFlags(opts, true)...)
+
+	app := &cli.App{
+		Name:  opts.BinaryName,
+		Usage: opts.ExecTemplate("A stream processor for mundane tasks - {{.DocumentationURL}}"),
+		Description: opts.ExecTemplate(`
+Either run {{.ProductName}} as a stream processor or choose a command:
+
+  {{.BinaryName}} list inputs
+  {{.BinaryName}} create kafka//file > ./config.yaml
+  {{.BinaryName}} -c ./config.yaml
+  {{.BinaryName}} -r "./production/*.yaml" -c ./config.yaml`)[1:],
+		Flags: flags,
+		Before: func(c *cli.Context) error {
+			return common.PreApplyEnvFilesAndTemplates(c, opts)
+		},
+		Action: func(c *cli.Context) error {
+			if c.Bool("version") {
+				fmt.Printf("Version: %v\nDate: %v\n", opts.Version, opts.DateBuilt)
+				os.Exit(0)
+			}
+			if c.Bool("help-autocomplete") {
+				_ = json.NewEncoder(os.Stdout).Encode(traverseHelp(c.Command, nil))
+				os.Exit(0)
+			}
+			if c.Args().Len() > 0 {
+				fmt.Fprintf(os.Stderr, "Unrecognised command: %v\n", c.Args().First())
+				_ = cli.ShowAppHelp(c)
+				os.Exit(1)
+			}
+
+			if code := common.RunService(c, opts, false); code != 0 {
+				os.Exit(code)
+			}
+			return nil
+		},
+		Commands: []*cli.Command{
+			echoCliCommand(opts),
+			lintCliCommand(opts),
+			runCliCommand(opts),
+			streamsCliCommand(opts),
+			listCliCommand(opts),
+			createCliCommand(opts),
+			test.CliCommand(opts),
+			clitemplate.CliCommand(opts),
+			blobl.CliCommand(opts),
+			studio.CliCommand(opts),
+		},
+	}
+
+	app.OnUsageError = func(context *cli.Context, err error, isSubcommand bool) error {
+		fmt.Printf("Usage error: %v\n", err)
+		_ = cli.ShowAppHelp(context)
+		return err
+	}
+	return app
+}

internal/cli/common/opts.go

@@ -23,7 +23,6 @@ type CLIOpts struct {
 	ProductName      string
 	DocumentationURL string
 
-	ShowRunCommand    bool
 	ConfigSearchPaths []string
 
 	Environment      *bundle.Environment
@@ -47,7 +46,6 @@ func NewCLIOpts(version, dateBuilt string) *CLIOpts {
 		BinaryName:       binaryName,
 		ProductName:      "Benthos",
 		DocumentationURL: "https://benthos.dev/docs",
-		ShowRunCommand:   false,
 		ConfigSearchPaths: []string{
 			"/benthos.yaml",
 			"/etc/benthos/config.yaml",

internal/cli/common/run_flags.go

@@ -0,0 +1,114 @@
+package common
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/redpanda-data/benthos/v4/internal/bloblang/parser"
+	"github.com/redpanda-data/benthos/v4/internal/filepath"
+	"github.com/redpanda-data/benthos/v4/internal/filepath/ifs"
+	"github.com/redpanda-data/benthos/v4/internal/template"
+	"github.com/urfave/cli/v2"
+)
+
+func RunFlags(opts *CLIOpts, hidden bool) []cli.Flag {
+	return []cli.Flag{
+		&cli.StringFlag{
+			Name:   "log.level",
+			Hidden: hidden,
+			Value:  "",
+			Usage:  "override the configured log level, options are: off, error, warn, info, debug, trace",
+		},
+		&cli.StringSliceFlag{
+			Name:    "set",
+			Hidden:  hidden,
+			Aliases: []string{"s"},
+			Usage:   "set a field (identified by a dot path) in the main configuration file, e.g. `\"metrics.type=prometheus\"`",
+		},
+		&cli.StringSliceFlag{
+			Name:    "resources",
+			Hidden:  hidden,
+			Aliases: []string{"r"},
+			Usage:   "pull in extra resources from a file, which can be referenced the same as resources defined in the main config, supports glob patterns (requires quotes)",
+		},
+		&cli.BoolFlag{
+			Name:   "chilled",
+			Hidden: hidden,
+			Value:  false,
+			Usage:  "continue to execute a config containing linter errors",
+		},
+		&cli.BoolFlag{
+			Name:    "watcher",
+			Hidden:  hidden,
+			Aliases: []string{"w"},
+			Value:   false,
+			Usage:   "EXPERIMENTAL: watch config files for changes and automatically apply them",
+		},
+	}
+}
+
+func EnvFileAndTemplateFlags(opts *CLIOpts, hidden bool) []cli.Flag {
+	return []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:    "env-file",
+			Hidden:  hidden,
+			Aliases: []string{"e"},
+			Value:   cli.NewStringSlice(),
+			Usage:   "import environment variables from a dotenv file",
+		},
+		&cli.StringSliceFlag{
+			Name:    "templates",
+			Hidden:  hidden,
+			Aliases: []string{"t"},
+			Usage:   opts.ExecTemplate("EXPERIMENTAL: import {{.ProductName}} templates, supports glob patterns (requires quotes)"),
+		},
+	}
+}
+
+// PreApplyEnvFilesAndTemplates takes a cli context and checks for flags
+// `env-file` and `templates` in order to parse and execute them before the CLI
+// proceeds onto the next behaviour.
+func PreApplyEnvFilesAndTemplates(c *cli.Context, opts *CLIOpts) error {
+	dotEnvPaths, err := filepath.Globs(ifs.OS(), c.StringSlice("env-file"))
+	if err != nil {
+		fmt.Printf("Failed to resolve env file glob pattern: %v\n", err)
+		os.Exit(1)
+	}
+	for _, dotEnvFile := range dotEnvPaths {
+		dotEnvBytes, err := ifs.ReadFile(ifs.OS(), dotEnvFile)
+		if err != nil {
+			fmt.Printf("Failed to read dotenv file: %v\n", err)
+			os.Exit(1)
+		}
+		vars, err := parser.ParseDotEnvFile(dotEnvBytes)
+		if err != nil {
+			fmt.Printf("Failed to parse dotenv file: %v\n", err)
+			os.Exit(1)
+		}
+		for k, v := range vars {
+			if err = os.Setenv(k, v); err != nil {
+				fmt.Printf("Failed to set env var '%v': %v\n", k, err)
+				os.Exit(1)
+			}
+		}
+	}
+
+	templatesPaths, err := filepath.Globs(ifs.OS(), c.StringSlice("templates"))
+	if err != nil {
+		fmt.Printf("Failed to resolve template glob pattern: %v\n", err)
+		os.Exit(1)
+	}
+	lints, err := template.InitTemplates(templatesPaths...)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Template file read error: %v\n", err)
+		os.Exit(1)
+	}
+	if !c.Bool("chilled") && len(lints) > 0 {
+		for _, lint := range lints {
+			fmt.Fprintln(os.Stderr, lint)
+		}
+		fmt.Println(opts.ExecTemplate("Shutting down due to linter errors, to prevent shutdown run {{.ProductName}} with --chilled"))
+		os.Exit(1)
+	}
+	return nil
+}

internal/cli/echo.go

@@ -0,0 +1,50 @@
+package cli
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/redpanda-data/benthos/v4/internal/cli/common"
+	"github.com/redpanda-data/benthos/v4/internal/docs"
+	"github.com/urfave/cli/v2"
+	"gopkg.in/yaml.v3"
+)
+
+func echoCliCommand(opts *common.CLIOpts) *cli.Command {
+	return &cli.Command{
+		Name:  "echo",
+		Usage: "Parse a config file and echo back a normalised version",
+		Description: opts.ExecTemplate(`
+This simple command is useful for sanity checking a config if it isn't
+behaving as expected, as it shows you a normalised version after environment
+variables have been resolved:
+
+  {{.BinaryName}} -c ./config.yaml echo | less`)[1:],
+		Action: func(c *cli.Context) error {
+			_, _, confReader := common.ReadConfig(c, opts, false)
+			_, pConf, _, err := confReader.Read()
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Configuration file read error: %v\n", err)
+				os.Exit(1)
+			}
+			var node yaml.Node
+			if err = node.Encode(pConf.Raw()); err == nil {
+				sanitConf := docs.NewSanitiseConfig(opts.Environment)
+				sanitConf.RemoveTypeField = true
+				sanitConf.ScrubSecrets = true
+				err = opts.MainConfigSpecCtor().SanitiseYAML(&node, sanitConf)
+			}
+			if err == nil {
+				var configYAML []byte
+				if configYAML, err = docs.MarshalYAML(node); err == nil {
+					fmt.Println(string(configYAML))
+				}
+			}
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Echo error: %v\n", err)
+				os.Exit(1)
+			}
+			return nil
+		},
+	}
+}