Separate RestartDictionaries and RestartCodecs frame flags (#48)

Fixes #47

Previously the flags were joint and incorrectly implemented.
The writers previously always reset dictionaries and encoders
at the same time, while readers only reset the dictionaries.

Now the flags are separate and each indicates the resetting
of either dictionaries or codecs. The writers and readers
correctly implement each flag.

Enabled previously disabled randomized write/read tests, which
were failing because of incorrect implementation. Also added a
few more randomized tests for other combinations of write options.

go/pkg/frameflags.go

@@ -7,6 +7,8 @@ const (
 	RestartDictionaries FrameFlags = 1 << iota
 	// RestartCompression resets and restarts the compression stream at frame beginning.
 	RestartCompression
+	// RestartCodecs resets the state of all encoders/decoders at frame beginning.
+	RestartCodecs
 
-	FrameFlagsMask = RestartDictionaries | RestartCompression
+	FrameFlagsMask = RestartDictionaries | RestartCompression | RestartCodecs
 )

go/pkg/writeropts.go

@@ -42,6 +42,13 @@ type WriterOptions struct {
 	//   carries over through the frames, which makes impossible to skip
 	//   frames and start decompressing from the next frame.
 	//   This flag has effect only if Compression!=CompressionNone.
+	//
+	// RestartEncoders - the encoder's state will be cleared. All new frames will
+	//   start with initial state of encoders.
+	//
+	// A combination of RestartDictionaries|RestartCompression|RestartEncoders flags
+	// ensures that a frame is readable and decodable on its own, without the need
+	// to read any preceding frames.
 	FrameRestartFlags FrameFlags
 
 	// MaxTotalDictSize is the maximum total byte size of all dictionaries.

stef-spec/specification.md

@@ -432,7 +432,8 @@ Frame structure is used to represent either a VarHeader or a Data Frame.
 Frame {
     RestartDictionaries: 1
     RestartCompression: 1
-    Random: 6
+    RestartCodecs: 1
+    Random: 5
     UncompressedSize: U64
     /CompressedSize: U64/
     Content: ..
@@ -450,6 +451,9 @@ reset when this bit is set. If this bit is unset the state of the compression en
 carries over through the Content field of frames. This bit has effect only if Flags
 field in the Header specifies a compression.
 
+RestartCodecs indicates that the state of encoders/decoders is cleared and started
+anew from this frame's Content.
+
 The UncompressedSize size field specifies the total size in bytes of the frame Content
 in uncompressed form. If no compression is used the UncompressedSize field is
 also the same as the size of the Content field in bytes.

stefgen/templates/array.go.tmpl

@@ -357,7 +357,10 @@ func (d *{{ .ArrayName }}Decoder) Continue() {
 }
 
 func (d *{{ .ArrayName }}Decoder) Reset() {
+	d.prevLen = 0
+	{{- if not .Recursive}}
 	d.decoder.Reset()
+	{{- end}}
 }
 
 func (d *{{ .ArrayName }}Decoder) Decode(dst *{{.ArrayName}}) error {

stefgen/templates/dicts.go.tmpl

@@ -34,7 +34,7 @@ func (d *WriterState) Init(opts *pkg.WriterOptions) {
     {{end}}
 }
 
-func (d *WriterState) Reset() {
+func (d *WriterState) ResetDicts() {
     d.limiter.ResetDict()
     {{range .Dicts -}}
     d.{{.DictName}}.Reset()

stefgen/templates/oneof.go.tmpl

@@ -312,9 +312,7 @@ func (e *{{ .StructName }}Encoder) Init(state* WriterState, columns *pkg.WriteCo
 }
 
 func (e *{{ .StructName }}Encoder) Reset() {
-	// Since we are resetting the state of encoder make sure the next Encode()
-	// call forcedly writes all fields and does not attempt to skip.
-	e.prevType = 0
+    e.prevType = 0
     {{- range .Fields}}
     e.{{.name}}Encoder.Reset()
     {{- end}}

stefgen/templates/reader.go.tmpl

@@ -89,6 +89,12 @@ func (r *{{.StructName}}Reader) nextFrame() error {
 		r.state.ResetDicts()
 	}
 
+	if frameFlags&pkg.RestartCodecs != 0 {
+		// The frame that has just started indicates that the decoders
+		// must be restarted.
+		r.decoder.Reset()
+	}
+
 	r.decoder.Continue()
 	return nil
 }