Add -skip option

#1

.gitignore

@@ -19,3 +19,6 @@ frangipanni_mac
 frangipanni.tgz
 frangipanni.zip
 sandbox
+
+# IDEs
+.idea

README.md

@@ -1,17 +1,17 @@
 # frangipanni
-Program to convert lines of text into a beautiful tree structure.
+Program to convert lines of text into beautiful tree structures.
 
 <img src="frangipanni.jpg" alt="A Tree" width="200" align="right">
 
-The program reads each line on the standard input in turn. It breaks the line into tokens, then adds the sequence of tokens into a tree structure which is printed as indented lines or JSON formats. Alternatively the tree can be passed to a gopher-Lua script which you can write for any format.
+The program reads each line on the standard input in turn. It breaks each line into tokens, then adds the sequence of tokens into a tree structure. Lines with the same leading tokens are placed in the same branch of the tree. The tree is printed as indented lines or JSON format. Alternatively the tree can be passed to a user-provided Lua script which can produce any output format.
 
-Options control where the line is broken into tokens, and output considerations.
+Options control where the line is broken into tokens, and how it is analysed and output.
 
 ## Basic Operation
 
-To explain the action of the program here is a simple example. Given this  command `sudo find /etc -maxdepth 3 | tail -9 `, 
+Here is a simple example. Given this  command `sudo find /etc -maxdepth 3 | tail -9 `, 
 
-We get this input data:
+We get this data:
 
 ```
 /etc/bluetooth/rfcomm.conf.dpkg-remove
@@ -25,11 +25,11 @@ We get this input data:
 /etc/fish/completions/task.fish
 ```
 
-The program, when invoked with :
+When we pipe this into the `frangipanni` program :
 ```
 sudo find /etc -maxdepth 3 | tail -9 | frangipanni
 ```
-produces this output:
+we see this output:
 
 ```
 etc
@@ -42,9 +42,9 @@ etc
         main.conf
     fish/completions/task.fish
 ```
-The program reads each line and splits them into tokens on any non-alphanumeric character. 
+By default, it reads each line and splits them into tokens when it finds a non-alphanumeric character. 
 
-In this example we're process a list of files produced by `find` so we only want to break on directories. So we can specify `-breaks /`. 
+In this next example we're processing a list of files produced by `find` so we only want to break on directories. So we can specify `-breaks /`. 
 
 The default behaviour is to _fold_ tree branches with no sub-branches into a single line of output. e.g. `fish/completions/task.fish` We turn off folding by specifying the `-no-fold` option. With the refined command
 ```
@@ -114,6 +114,8 @@ cat <input> | frangipanni [options]
         Sort order input|alpha. Sort the childs either in input order or via character ordering (default "input")
   -separators
         Print leading separators.
+  -skip int
+        Number of leading fields to skip.
   -spacer string
         Characters to indent lines with. (default " ")
 ```
@@ -158,6 +160,25 @@ May 10
     : Starting Docker Cleanup
 ```
 
+with the `-skip 5` option we can ignore the data and time at the beginning of each line. The output is
+
+```
+localhost
+    systemd
+        Removed slice User Slice of root
+        Stopping User Slice of root
+        Starting Docker Cleanup
+        Started Docker Cleanup
+    dockerd-current: time="2020-05-10T04:00:00
+        629849861+10:00" level=debug msg="Calling GET /_ping
+        629948000+10:00" level=debug msg="Unable to determine container for
+        630103455+10:00" level=debug msg="{Action=_ping, LoginUID=12345678, PID=21075
+        630684502+10:00" level=debug msg="Calling GET /v1.26/containers/json?all=1&filters=%7B%22status%22%3A%7B%22dead%22%3Atrue%7D%7D
+        630704513+10:00" level=debug msg="Unable to determine container for containers
+        630735545+10:00" level=debug msg="{Action=json, LoginUID=12345678, PID=21075
+```
+
+
 ## Data from environment variables
 
 Give this input, from `env | egrep '^XDG'`:

frangipanni.go

@@ -5,15 +5,15 @@ package main
 //
 import (
 	"bufio"
+	"encoding/json"
 	"flag"
 	"fmt"
-	"io"
-	"log"
-	"os"
-	"encoding/json"
 	luajson "github.com/layeh/gopher-json"
 	"github.com/yuin/gopher-lua"
+	"io"
+	"log"
 	"math"
+	"os"
 	"sort"
 	"strconv"
 	"strings"
@@ -310,7 +310,7 @@ func fakeCounts(n *node) {
 }
 
 func makeLuaTableFromNode(L *lua.LState, n *node) *lua.LTable {
-	tb := L.CreateTable(6,6)
+	tb := L.CreateTable(6, 6)
 
 	tb.RawSet(lua.LString("lineNumber"), lua.LNumber(n.lineNumber))
 	tb.RawSet(lua.LString("text"), lua.LString(n.text))
@@ -335,7 +335,6 @@ func luaRun(out io.Writer, root *node) {
 	}
 }
 
-
 // Nasty Globals for options ;-)
 var printSeparators bool
 var noFold bool
@@ -349,6 +348,7 @@ var printDepth int
 var indentWidth int
 var indentString string
 var luaFile string
+var skipLevel int
 
 func main() {
 
@@ -368,6 +368,7 @@ func main() {
 	flag.IntVar(&indentWidth, "indent", 4, "Number of spaces to indent per level.")
 	flag.StringVar(&indentString, "spacer", " ", "Characters to indent lines with.")
 	flag.StringVar(&luaFile, "lua", "", "Lua Script to run")
+	flag.IntVar(&skipLevel, "skip", 0, "Number of leading fields to skip.")
 
 	flag.Parse()
 	if maxLevel < 0 {
@@ -416,12 +417,19 @@ func main() {
 			t = strings.FieldsFunc(line, isSep)
 			seps = strings.FieldsFunc(line, isNotSep)
 			if isNotSep(rune(line[0])) {
-				// line didn't start with a seperator, so insert a fake one
+				// line didn't start with a separator, so insert a fake one
 				seps = append(seps, "") // add space at the end
 				copy(seps[1:], seps)    // shift right
 				seps[0] = ""            // inject fake at the front
 			}
 		}
+		//
+		// Skip leading fields if required
+		for s := skipLevel; s > 0 && len(t) > 1 && len(seps) > 1; s-- {
+			t = t[1:]
+			seps = seps[1:]
+		}
+
 		if len(t) <= maxLevel {
 			add(nr, &root, t, seps)
 		} else {

go.mod

@@ -0,0 +1,8 @@
+module github.com/birchb1024/frangipanni
+
+go 1.15
+
+require (
+	github.com/layeh/gopher-json v0.0.0-20201124131017-552bb3c4c3bf
+	github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da
+)

go.sum

@@ -0,0 +1,8 @@
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/layeh/gopher-json v0.0.0-20201124131017-552bb3c4c3bf h1:bg6J/5S/AeTz7K9i/luJRj31BJ8f+LgYwKQBSOZxSEM=
+github.com/layeh/gopher-json v0.0.0-20201124131017-552bb3c4c3bf/go.mod h1:E/q28EyUVBgBQnONAVPIdwvEsv4Ve0vaCA9JWim4+3I=
+github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da h1:NimzV1aGyq29m5ukMK0AMWEhFaL/lrEOaephfuoiARg=
+github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da/go.mod h1:E1AXubJBdNmFERAOucpDIxNzeGfLzg0mYh+UfMWdChA=
+golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

test/confidence.sh

@@ -24,6 +24,9 @@ done
 
     echo "fixtures/simplechars.txt--- -lua xml.lua ----------------------------------------------------------------------------------------------------" >> "$tempfile"
     <fixtures/simplechars.txt ../frangipanni -lua ../xml.lua >> "$tempfile"
+# -skip
+    echo "fixtures/log-file.txt--- -skip 3 ----------------------------------------------------------------------------------------------------" >> "$tempfile"
+    <fixtures/log-file.txt ../frangipanni -skip 5 >> "$tempfile"
 
 set -x
-diff "$tempfile" $scriptdir/fixtures.log || kdiff3 "$tempfile" $scriptdir/fixtures.log
+diff "$tempfile" $scriptdir/fixtures.log || meld "$tempfile" $scriptdir/fixtures.log

test/fixtures.log

@@ -6612,3 +6612,17 @@ fixtures/simplechars.txt--- -lua xml.lua ---------------------------------------
       <2 count="1" sep="."/>
    </x.a>
 </root>
+fixtures/log-file.txt--- -skip 3 ----------------------------------------------------------------------------------------------------
+localhost
+    systemd
+        Removed slice User Slice of root
+        Stopping User Slice of root
+        Starting Docker Cleanup
+        Started Docker Cleanup
+    dockerd-current: time="2020-05-10T04:00:00
+        629849861+10:00" level=debug msg="Calling GET /_ping
+        629948000+10:00" level=debug msg="Unable to determine container for
+        630103455+10:00" level=debug msg="{Action=_ping, LoginUID=12345678, PID=21075
+        630684502+10:00" level=debug msg="Calling GET /v1.26/containers/json?all=1&filters=%7B%22status%22%3A%7B%22dead%22%3Atrue%7D%7D
+        630704513+10:00" level=debug msg="Unable to determine container for containers
+        630735545+10:00" level=debug msg="{Action=json, LoginUID=12345678, PID=21075