diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 2f463835..157d0190 100755 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM adrienaury/go-devcontainer:v2.0 +FROM adrienaury/go-devcontainer:v4.1 USER root diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dcc128e..0cdafd2c 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ Types of changes - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. +## [1.26.1] + +- `Fixed` performance issues on JSON serialization + ## [1.26.0] - `Added` property `preserve-list` to masking definition to be able to ignore masking specific values diff --git a/cmd/pimo/default.pgo b/cmd/pimo/default.pgo new file mode 100644 index 00000000..5e2f2e1b Binary files /dev/null and b/cmd/pimo/default.pgo differ diff --git a/go.mod b/go.mod index 4772b0d5..c1e6ed66 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,17 @@ module github.com/cgi-fr/pimo -go 1.20 +go 1.21 + +toolchain go1.22.2 require ( github.com/CGI-FR/xixo v0.1.8 github.com/Masterminds/sprig/v3 v3.3.0 github.com/adrienaury/zeromdc v0.1.1 github.com/capitalone/fpe v1.2.1 + github.com/goccy/go-json v0.10.3 github.com/goccy/go-yaml v1.12.0 + github.com/iancoleman/orderedmap v0.3.0 github.com/invopop/jsonschema v0.12.0 github.com/kenshaw/baseconv v0.1.1 github.com/labstack/echo/v4 v4.12.0 @@ -19,7 +23,6 @@ require ( github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea - gitlab.com/c0b/go-ordered-json v0.0.0-20201030195603-febf46534d5a golang.org/x/crypto v0.27.0 golang.org/x/exp v0.0.0-20240707233637-46b078467d37 golang.org/x/text v0.18.0 diff --git a/go.sum b/go.sum index 208af9f8..f4638093 100644 --- a/go.sum +++ b/go.sum @@ -29,15 +29,22 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.12.0 h1:/1WHjnMsI1dlIBQutrvSMGZRQufVO3asrHfTwfACoPM= github.com/goccy/go-yaml v1.12.0/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gxui v0.0.0-20151028112939-f85e0a97b3a4 h1:OL2d27ueTKnlQJoqLW2fc9pWYulFnJYLWzomGV7HqZo= github.com/google/gxui v0.0.0-20151028112939-f85e0a97b3a4/go.mod h1:Pw1H1OjSNHiqeuxAduB1BKYXIwFtsyrY47nEqSgEiCM= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y= @@ -48,6 +55,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/iancoleman/orderedmap v0.3.0 h1:5cbR2grmZR/DiVt+VJopEhtVs9YGInGIxAoMJn+Ichc= +github.com/iancoleman/orderedmap v0.3.0/go.mod h1:XuLcCUkdL5owUCQeF2Ue9uuw1EptkJDkXXS7VoV7XGE= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -59,12 +68,15 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/kenshaw/baseconv v0.1.1 h1:oAu/C7ipUT2PqT9DT0mZDGDg4URIglizZMjPv9oCu0E= github.com/kenshaw/baseconv v0.1.1/go.mod h1:yy9zGmnnR6vgOxOQb702nVdAG30JhyYZpj/5/m0siRI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0= github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM= github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/anko v0.1.9 h1:OpdRDvjOY/FQXoGXkN0n7b0br4a4fHB5hhsaq0Cd3ds= @@ -88,6 +100,7 @@ github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDj github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= @@ -118,8 +131,6 @@ github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/ github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea h1:CyhwejzVGvZ3Q2PSbQ4NRRYn+ZWv5eS1vlaEusT+bAI= github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea/go.mod h1:eNr558nEUjP8acGw8FFjTeWvSgU1stO7FAO6eknhHe4= -gitlab.com/c0b/go-ordered-json v0.0.0-20201030195603-febf46534d5a h1:DxppxFKRqJ8WD6oJ3+ZXKDY0iMONQDl5UTg2aTyHh8k= -gitlab.com/c0b/go-ordered-json v0.0.0-20201030195603-febf46534d5a/go.mod h1:NREvu3a57BaK0R1+ztrEzHWiZAihohNLQ6trPxlIqZI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= diff --git a/pkg/model/ordered_dict.go b/pkg/model/ordered_dict.go index d19eef79..e9412566 100644 --- a/pkg/model/ordered_dict.go +++ b/pkg/model/ordered_dict.go @@ -1,11 +1,14 @@ package model import ( + "bytes" "encoding/json" "fmt" "github.com/rs/zerolog/log" - "gitlab.com/c0b/go-ordered-json" + + goccy "github.com/goccy/go-json" + ordered "github.com/iancoleman/orderedmap" ) type Dictionary struct { @@ -17,7 +20,7 @@ func NewPackedDictionary() Dictionary { } func NewDictionary() Dictionary { - return Dictionary{ordered.NewOrderedMap()} + return Dictionary{ordered.New()} } func Copy(other Entry) Entry { @@ -37,19 +40,15 @@ func CopyDictionary(other Dictionary) Dictionary { return NewDictionary() } - om := ordered.NewOrderedMap() - iter := other.EntriesIter() + om := ordered.New() - for { - pair, ok := iter() - if !ok { - break - } - switch typedVal := pair.Value.(type) { + for _, key := range other.Keys() { + value := other.Get(key) + switch typedVal := value.(type) { case Dictionary: - om.Set(pair.Key, CopyDictionary(typedVal)) + om.Set(key, CopyDictionary(typedVal)) default: - om.Set(pair.Key, pair.Value) + om.Set(key, value) } } return Dictionary{om} @@ -72,31 +71,29 @@ func CleanTypes(inter interface{}) interface{} { } return dict case *Dictionary: - iter := typedInter.EntriesIter() dict := NewDictionary() - for pair, ok := iter(); ok; pair, ok = iter() { - dict.Set(pair.Key, CleanTypes(pair.Value)) + for _, key := range typedInter.Keys() { + dict.Set(key, CleanTypes(typedInter.Get(key))) } return dict case Dictionary: - iter := typedInter.EntriesIter() dict := NewDictionary() - for pair, ok := iter(); ok; pair, ok = iter() { - dict.Set(pair.Key, CleanTypes(pair.Value)) + for _, key := range typedInter.Keys() { + dict.Set(key, CleanTypes(typedInter.Get(key))) } return dict case *ordered.OrderedMap: - iter := typedInter.EntriesIter() dict := NewDictionary() - for pair, ok := iter(); ok; pair, ok = iter() { - dict.Set(pair.Key, CleanTypes(pair.Value)) + for _, key := range typedInter.Keys() { + value, _ := typedInter.Get(key) + dict.Set(key, CleanTypes(value)) } return dict case ordered.OrderedMap: - iter := typedInter.EntriesIter() dict := NewDictionary() - for pair, ok := iter(); ok; pair, ok = iter() { - dict.Set(pair.Key, CleanTypes(pair.Value)) + for _, key := range typedInter.Keys() { + value, _ := typedInter.Get(key) + dict.Set(key, CleanTypes(value)) } return dict case []interface{}: @@ -165,31 +162,29 @@ func Untyped(inter interface{}) interface{} { } return cleanmap case *Dictionary: - iter := typedInter.EntriesIter() cleanmap := map[string]interface{}{} - for pair, ok := iter(); ok; pair, ok = iter() { - cleanmap[pair.Key] = Untyped(pair.Value) + for _, key := range typedInter.Keys() { + cleanmap[key] = Untyped(typedInter.Get(key)) } return cleanmap case Dictionary: - iter := typedInter.EntriesIter() cleanmap := map[string]interface{}{} - for pair, ok := iter(); ok; pair, ok = iter() { - cleanmap[pair.Key] = Untyped(pair.Value) + for _, key := range typedInter.Keys() { + cleanmap[key] = Untyped(typedInter.Get(key)) } return cleanmap case *ordered.OrderedMap: - iter := typedInter.EntriesIter() cleanmap := map[string]interface{}{} - for pair, ok := iter(); ok; pair, ok = iter() { - cleanmap[pair.Key] = Untyped(pair.Value) + for _, key := range typedInter.Keys() { + value, _ := typedInter.Get(key) + cleanmap[key] = Untyped(value) } return cleanmap case ordered.OrderedMap: - iter := typedInter.EntriesIter() cleanmap := map[string]interface{}{} - for pair, ok := iter(); ok; pair, ok = iter() { - cleanmap[pair.Key] = Untyped(pair.Value) + for _, key := range typedInter.Keys() { + value, _ := typedInter.Get(key) + cleanmap[key] = Untyped(value) } return cleanmap case []interface{}: @@ -234,31 +229,29 @@ func UnorderedTypes(inter interface{}) interface{} { } return cleanmap case *Dictionary: - iter := typedInter.EntriesIter() cleanmap := map[string]Entry{} - for pair, ok := iter(); ok; pair, ok = iter() { - cleanmap[pair.Key] = UnorderedTypes(pair.Value) + for _, key := range typedInter.Keys() { + cleanmap[key] = UnorderedTypes(typedInter.Get(key)) } return cleanmap case Dictionary: - iter := typedInter.EntriesIter() cleanmap := map[string]Entry{} - for pair, ok := iter(); ok; pair, ok = iter() { - cleanmap[pair.Key] = UnorderedTypes(pair.Value) + for _, key := range typedInter.Keys() { + cleanmap[key] = UnorderedTypes(typedInter.Get(key)) } return cleanmap case *ordered.OrderedMap: - iter := typedInter.EntriesIter() cleanmap := map[string]Entry{} - for pair, ok := iter(); ok; pair, ok = iter() { - cleanmap[pair.Key] = UnorderedTypes(pair.Value) + for _, key := range typedInter.Keys() { + value, _ := typedInter.Get(key) + cleanmap[key] = UnorderedTypes(value) } return cleanmap case ordered.OrderedMap: - iter := typedInter.EntriesIter() cleanmap := map[string]Entry{} - for pair, ok := iter(); ok; pair, ok = iter() { - cleanmap[pair.Key] = UnorderedTypes(pair.Value) + for _, key := range typedInter.Keys() { + value, _ := typedInter.Get(key) + cleanmap[key] = UnorderedTypes(value) } return cleanmap case []interface{}: @@ -294,7 +287,7 @@ func UnorderedTypes(inter interface{}) interface{} { } func (d Dictionary) IsPacked() bool { - _, packed := d.GetValue(".") + _, packed := d.OrderedMap.Get(".") return packed } @@ -353,3 +346,36 @@ func (d Dictionary) With(key string, value interface{}) Dictionary { result.Set(key, CleanTypes(value)) return result } + +func (d Dictionary) GetValue(key string) (Entry, bool) { + return d.OrderedMap.Get(key) +} + +func (d Dictionary) Get(key string) Entry { + entry, _ := d.OrderedMap.Get(key) + return entry +} + +func (d Dictionary) MarshalJSON() ([]byte, error) { + var buf bytes.Buffer + buf.WriteByte('{') + encoder := goccy.NewEncoder(&buf) + for i, k := range d.Keys() { + if i > 0 { + buf.WriteByte(',') + } + // add key + if err := encoder.Encode(k); err != nil { + return nil, err + } + buf.Truncate(buf.Len() - 1) // remove last new line + buf.WriteByte(':') + // add value + if err := encoder.Encode(d.Get(k)); err != nil { + return nil, err + } + buf.Truncate(buf.Len() - 1) // remove last new line + } + buf.WriteByte('}') + return buf.Bytes(), nil +} diff --git a/pkg/model/process_mask.go b/pkg/model/process_mask.go index fd073293..723759bf 100644 --- a/pkg/model/process_mask.go +++ b/pkg/model/process_mask.go @@ -77,7 +77,7 @@ func (mep *MaskEngineProcess) ProcessDictionary(dictionary Dictionary, out Colle if ret == nil { out.Collect(result) - return + return nil } if ret != nil && skipLineOnError { diff --git a/pkg/timeline/timeline.go b/pkg/timeline/timeline.go index 2b34fe92..5d261dda 100644 --- a/pkg/timeline/timeline.go +++ b/pkg/timeline/timeline.go @@ -131,14 +131,8 @@ func (me MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.Ent initialState := map[string]*int64{} if dict, ok := e.(model.Dictionary); ok { - iter := dict.EntriesIter() - for { - pair, ok := iter() - if !ok { - break - } - - initialState[pair.Key] = me.formatTimestamp(pair.Value) + for _, key := range dict.Keys() { + initialState[key] = me.formatTimestamp(dict.Get(key)) } }