Add json parser, and ignore lines that match with an empty key

docs/expressions.md

@@ -107,3 +107,11 @@ hf: Float
 hi: Int
 
 Formats a number based with appropriate placement of commas and decimals
+
+## Json
+
+Syntax: `{json field expression}`
+
+Extract a JSON value based on the expression statement from [gjson](https://github.com/tidwall/gjson)
+
+See: [json](json.md) for more information.

docs/json.md

@@ -0,0 +1,45 @@
+# Json
+
+Syntax: `{json field expression}`
+
+Extract a JSON value based on the expression statement
+from [gjson](https://github.com/tidwall/gjson)
+
+When using, you likely often want to extract a specific bit of
+json from the line.  If you want to match the entire line you
+will likely want to leave the match at the default `.*`, and
+provide `{0}` as the field.
+
+For example, this command would extract lastname from below:
+`rare filter -e '{json {0} "name.last"}`
+
+Example expressions from their documentation:
+
+```GJson Expressions
+"name.last"          >> "Anderson"
+"age"                >> 37
+"children"           >> ["Sara","Alex","Jack"]
+"children.#"         >> 3
+"children.1"         >> "Alex"
+"child*.2"           >> "Jack"
+"c?ildren.0"         >> "Sara"
+"fav\.movie"         >> "Deer Hunter"
+"friends.#.first"    >> ["Dale","Roger","Jane"]
+"friends.1.last"     >> "Craig"
+```
+
+With the given example:
+
+```json
+{
+  "name": {"first": "Tom", "last": "Anderson"},
+  "age":37,
+  "children": ["Sara","Alex","Jack"],
+  "fav.movie": "Deer Hunter",
+  "friends": [
+    {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]},
+    {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]},
+    {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]}
+  ]
+}
+```

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/mdempsky/gocode v0.0.0-20190203001940-7fb65232883f // indirect
 	github.com/rogpeppe/go-internal v1.5.0 // indirect
 	github.com/stretchr/testify v1.4.0
+	github.com/tidwall/gjson v1.3.2
 	github.com/urfave/cli v1.22.1
 	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect
 	golang.org/x/sys v0.0.0-20191010194322-b09406accb47 // indirect

go.sum

@@ -71,6 +71,12 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/tidwall/gjson v1.3.2 h1:+7p3qQFaH3fOMXAJSrdZwGKcOO/lYdGS0HqGhPqDdTI=
+github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
+github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
+github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=

pkg/expressions/jsonFunctions.go

@@ -0,0 +1,14 @@
+package expressions
+
+import "github.com/tidwall/gjson"
+
+func kfJson(args []KeyBuilderStage) KeyBuilderStage {
+	if len(args) != 2 {
+		return stageError(ErrorArgCount)
+	}
+	return KeyBuilderStage(func(context KeyBuilderContext) string {
+		json := args[0](context)
+		expression := args[1](context)
+		return gjson.Get(json, expression).Str
+	})
+}

pkg/expressions/keyFunctions.go

@@ -314,4 +314,5 @@ var defaultFunctions = map[string]KeyBuilderFunction{
 	"format": KeyBuilderFunction(kfFormat),
 	"hi":     KeyBuilderFunction(kfHumanizeInt),
 	"hf":     KeyBuilderFunction(kfHumanizeFloat),
+	"json":   KeyBuilderFunction(kfJson),
 }

pkg/extractor/extractor.go

@@ -76,18 +76,23 @@ func (s *Extractor) processLineSync(line string) {
 	if len(matches) > 0 {
 		slices := indexToSlices(line, matches)
 		if s.ignore == nil || !s.ignore.IgnoreMatch(slices...) {
-			matchNum := atomic.AddUint64(&s.matchedLines, 1)
-
 			context := expressions.KeyBuilderContextArray{
 				Elements: slices,
 			}
-			s.readChan <- &Match{
-				Line:        line,
-				Groups:      slices,
-				Indices:     matches,
-				Extracted:   s.keyBuilder.BuildKey(&context),
-				LineNumber:  lineNum,
-				MatchNumber: matchNum,
+			extractedKey := s.keyBuilder.BuildKey(&context)
+
+			if len(extractedKey) > 0 {
+				matchNum := atomic.AddUint64(&s.matchedLines, 1)
+				s.readChan <- &Match{
+					Line:        line,
+					Groups:      slices,
+					Indices:     matches,
+					Extracted:   extractedKey,
+					LineNumber:  lineNum,
+					MatchNumber: matchNum,
+				}
+			} else {
+				atomic.AddUint64(&s.ignoredLines, 1)
 			}
 		} else {
 			atomic.AddUint64(&s.ignoredLines, 1)