changes for v0.2.2 release

CHANGES.md

@@ -1,5 +1,15 @@
 # goatCLI changelog
 
+## v0.2.2
+
+* NEW: output formatters can now accept optins (`--opt X`)
+* NEW: output annotator: output formatters can get probe ASN, CC and prefix
+  information. Probe metadata is cached accross runs, for a week.
+* CHANGED: output formatters now have a `start()` method to signal processing
+  of (a batch of) results, this maybe even multiple times - just like with
+  `finish()` from now on. `setup()` is only called once, before any results are
+  processed.
+
 ## v0.2.1
 
 * CHANGED: internal changes on how output formatters work

README.md

@@ -130,12 +130,13 @@ Or perhaps:
 $ cat some-results.jsonl | ./goatcli result
 ```
 
-Loading results from a file expects one result per line ("format=txt" a.k.a. JSONL)
+Loading results from a file expects one result per line ("format=txt" a.k.a. JSONL a.k.a. NDJSON)
 
 The output formatters are extensible, feel free to write your own -- and conntribute that back to this repo! You only need to make a new package under `output` that implements three functions:
-* `setup()` to prepare for processing results
+* `setup()` to initialise the output processor
+* `start()` to prepare for processing results; may be called once per batch of results
 * `process()` to deal with one incoming result
-* `finish()` to finish processing, make a summary, etc.
+* `finish()` to finish processing, make a summary, etc.; may be called once per batch of results
 
 New output processors need to be registered in `output.go`. See `some.go` or `native.go` for examples.
 

config.go

@@ -9,6 +9,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"goatcli/output/annotate"
 	"os"
 
 	"github.com/google/uuid"
@@ -35,11 +36,12 @@ var (
 	apiKeys map[string]uuid.UUID // collected from config file
 )
 
-const version = "v0.2.1"
+const version = "v0.2.2"
 const CLIName = "goatCLI " + version
 
 var defaultConfigDir = os.Getenv("HOME") + "/.config"
 var defaultConfigFile = defaultConfigDir + "/goat.ini"
+var CacheDir string = os.Getenv("HOME") + "/.cache/goat/"
 
 var Subcommands map[string]*flag.FlagSet
 
@@ -126,6 +128,7 @@ func configure() {
 	}
 
 	goatapi.ModifyUserAgent(CLIName)
+	annotate.SetCacheDir(CacheDir, flagVerbose)
 }
 
 // readConfig deals with configuration file loading
@@ -144,23 +147,31 @@ func readConfig(confFile string) bool {
 
 	// record stuff that was in the config file
 	loadApiKey(cfg, "list_measurements")
-	// TODO: add more API key variaations here
+	// TODO: add more API key variations here
 
 	// allow config to override where the API is
 	apibase := cfg.Section("").Key("apibase").MustString("")
 	if apibase != "" {
 		goatapi.SetAPIBase(apibase)
 	}
 
+	// allow config to override the cache directory
+	cachedir := cfg.Section("").Key("cachedir").MustString("")
+	if cachedir != "" {
+		CacheDir = cachedir
+	}
+	// we deliberately ignore errors on creating this dir as it may exist
+	_ = os.MkdirAll(CacheDir, os.FileMode(0755))
+
 	return true
 }
 
 // createConfig tries to create a (default) config file
 func createConfig(confFile string) {
 	// TODO: perhaps try to make the config directory first?
 
-	// we deliberately ignre errors on creating this dir as it may exist
-	_ = os.MkdirAll(defaultConfigDir, os.FileMode(0755))
+	// we deliberately ignore errors on creating this dir as it may exist
+	_ = os.MkdirAll(defaultConfigDir, os.FileMode(0700))
 
 	f, err := os.Create(confFile)
 	if err != nil && flagVerbose {
@@ -179,6 +190,9 @@ func createConfig(confFile string) {
 # useful only for compatible APIs, i.e. proxies, API development, ...
 apibase = ""
 
+# cachedir defines where to put cache files (ie. probe data)
+cachedir = "~/.cache/goat/"
+
 # apikeys is where the various (private) API keys are defined
 [apikeys]
 

findanchor.go

@@ -23,9 +23,10 @@ type findAnchorFlags struct {
 	filterCC     string
 	filterSearch string
 
-	output string
-	limit  uint
-	count  bool
+	output  string
+	outopts multioption
+	limit   uint
+	count   bool
 }
 
 // Implementation of the "find anchor" subcommand. Parses command line flags
@@ -56,7 +57,8 @@ func commandFindAnchor(args []string) {
 	go filter.GetAnchors(flagVerbose, anchors)
 
 	// produce output; exact format depends on the "format" option
-	output.Setup(formatter, flagVerbose)
+	output.Setup(formatter, flagVerbose, flags.outopts)
+	output.Start(formatter)
 	for anchor := range anchors {
 		if anchor.Error != nil {
 			fmt.Fprintf(os.Stderr, "ERROR: %v\n", anchor.Error)
@@ -136,6 +138,7 @@ func parseFindAnchorArgs(args []string) *findAnchorFlags {
 	// options
 	flagsFindAnchor.BoolVar(&flags.count, "count", false, "Count only, don't show the actual results")
 	flagsFindAnchor.StringVar(&flags.output, "output", "some", "Output format: 'id', 'idcsv', 'some' or 'most'")
+	flagsFindAnchor.Var(&flags.outopts, "opt", "Options to pass to the output formatter")
 
 	// limit
 	flagsFindAnchor.UintVar(&flags.limit, "limit", 100, "Maximum amount of anchors to retrieve")

findmsm.go

@@ -58,10 +58,11 @@ type findMsmFlags struct {
 	filterProtocol      string
 	filterMy            bool
 
-	output string
-	sort   string
-	limit  uint
-	count  bool
+	output  string
+	outopts multioption
+	sort    string
+	limit   uint
+	count   bool
 }
 
 // Implementation of the "find measurement" subcommand. Parses command line flags
@@ -92,7 +93,8 @@ func commandFindMsm(args []string) {
 	go filter.GetMeasurements(flagVerbose, measurements)
 
 	// produce output; exact format depends on the "format" option
-	output.Setup(formatter, flagVerbose)
+	output.Setup(formatter, flagVerbose, []string(flags.outopts))
+	output.Start(formatter)
 	for measurement := range measurements {
 		if measurement.Error != nil {
 			fmt.Fprintf(os.Stderr, "ERROR: %v\n", measurement.Error)
@@ -421,6 +423,7 @@ func parseFindMsmArgs(args []string) *findMsmFlags {
 	flagsFindMsm.BoolVar(&flags.count, "count", false, "Count only, don't show the actual results")
 	flagsFindMsm.StringVar(&flags.sort, "sort", "-id", "Result ordering: "+strings.Join(goatapi.MeasurementListSortOrders, ","))
 	flagsFindMsm.StringVar(&flags.output, "output", "some", "Output format: 'id', 'idcsv', 'some' or 'most'")
+	flagsFindMsm.Var(&flags.outopts, "opt", "Options to pass to the output formatter")
 
 	// limit
 	flagsFindMsm.UintVar(&flags.limit, "limit", 100, "Maximum amount of measurements to retrieve")

findprobe.go

@@ -52,10 +52,11 @@ type findProbeFlags struct {
 	filterNotPublic bool
 	filterTags      string
 
-	output string
-	sort   string
-	limit  uint
-	count  bool
+	output  string
+	outopts multioption
+	sort    string
+	limit   uint
+	count   bool
 }
 
 // Implementation of the "find probe" subcommand. Parses command line flags
@@ -86,7 +87,8 @@ func commandFindProbe(args []string) {
 	go filter.GetProbes(flagVerbose, probes)
 
 	// produce output; exact format depends on the "format" option
-	output.Setup(formatter, flagVerbose)
+	output.Setup(formatter, flagVerbose, flags.outopts)
+	output.Start(formatter)
 	for probe := range probes {
 		if probe.Error != nil {
 			fmt.Fprintf(os.Stderr, "ERROR: %v\n", probe.Error)
@@ -332,6 +334,7 @@ func parseFindProbeArgs(args []string) *findProbeFlags {
 	flagsFindProbe.BoolVar(&flags.count, "count", false, "Count only, don't show the actual results")
 	flagsFindProbe.StringVar(&flags.sort, "sort", "-id", "Result ordering: "+strings.Join(goatapi.ProbeListSortOrders, ","))
 	flagsFindProbe.StringVar(&flags.output, "output", "some", "Output format: 'id', 'idcsv', 'some' or 'most'")
+	flagsFindProbe.Var(&flags.outopts, "opt", "Options to pass to the output formatter")
 
 	// limit
 	flagsFindProbe.UintVar(&flags.limit, "limit", 100, "Maximum amount of probes to retrieve")