From 8eebb08d6cfca609944b4166a8c5d2fc4f2697d6 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Thu, 12 Oct 2023 10:42:57 +0400 Subject: [PATCH 01/23] Better LS connection pool balancer --- adnl/dht/client_test.go | 2 +- liteclient/connection.go | 4 ++ liteclient/pool.go | 91 ++++++++++++++++++++-------------------- ton/transactions.go | 4 ++ 4 files changed, 54 insertions(+), 47 deletions(-) diff --git a/adnl/dht/client_test.go b/adnl/dht/client_test.go index 99b3e57c..1276ac12 100644 --- a/adnl/dht/client_test.go +++ b/adnl/dht/client_test.go @@ -802,7 +802,7 @@ func TestClient_StoreAddressIntegration(t *testing.T) { ExpireAt: 0, } - _, _, err = dhtClient.StoreAddress(ctx, addrList, 12*time.Minute, key, 2) + _, _, err = dhtClient.StoreAddress(ctx, addrList, 12*time.Minute, key, 3) if err != nil { t.Fatal(err) } diff --git a/liteclient/connection.go b/liteclient/connection.go index 9c52a577..2efb196b 100644 --- a/liteclient/connection.go +++ b/liteclient/connection.go @@ -43,6 +43,9 @@ type connection struct { authed bool authEvt chan bool + weight int64 + lastRespTime int64 + pool *ConnectionPool } @@ -138,6 +141,7 @@ func (c *ConnectionPool) AddConnection(ctx context.Context, addr, serverKey stri reqs: make(chan *ADNLRequest), pool: c, id: crc32.ChecksumIEEE([]byte(serverKey)), + weight: 1000, } // get timeout if exists diff --git a/liteclient/pool.go b/liteclient/pool.go index 5d4a0f7f..9312bb6a 100644 --- a/liteclient/pool.go +++ b/liteclient/pool.go @@ -22,7 +22,6 @@ type OnDisconnectCallback func(addr, key string) type ADNLResponse struct { Data tl.Serializable - Err error } type ADNLRequest struct { @@ -151,6 +150,14 @@ func (c *ConnectionPool) QueryADNL(ctx context.Context, request tl.Serializable, return err } + _, hasDeadline := ctx.Deadline() + if !hasDeadline { + // fallback timeout to not stuck forever with background context + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, 15*time.Second) + defer cancel() + } + // buffered channel to not block listener ch := make(chan *ADNLResponse, 1) req := &ADNLRequest{ @@ -171,52 +178,51 @@ func (c *ConnectionPool) QueryADNL(ctx context.Context, request tl.Serializable, c.reqMx.Unlock() }() - var host string + tm := time.Now() + + var node *connection if nodeID, ok := ctx.Value(_StickyCtxKey).(uint32); ok && nodeID > 0 { - host, err = c.querySticky(nodeID, req) + node, err = c.querySticky(nodeID, req) if err != nil { return err } } else { - host, err = c.queryWithBalancer(req) + node, err = c.queryWithSmartBalancer(req) if err != nil { return err } } - _, hasDeadline := ctx.Deadline() - if !hasDeadline { - // fallback timeout to not stuck forever with background context - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, 30*time.Second) - defer cancel() - } - // wait for response select { case resp := <-ch: - if resp.Err != nil { - return resp.Err - } + atomic.AddInt64(&node.weight, 1) + atomic.StoreInt64(&node.lastRespTime, int64(time.Since(tm))) reflect.ValueOf(result).Elem().Set(reflect.ValueOf(resp.Data)) return nil case <-ctx.Done(): + if time.Since(tm) < 200*time.Millisecond { + // consider it as too short timeout to punish node + atomic.AddInt64(&node.weight, 1) + } + if !hasDeadline { - return fmt.Errorf("adnl request timeout, node %s", host) + return fmt.Errorf("adnl request timeout, node %s", node.addr) } - return fmt.Errorf("deadline exceeded, node %s, err: %w", host, ctx.Err()) + return fmt.Errorf("deadline exceeded, node %s, err: %w", node.addr, ctx.Err()) } } -func (c *ConnectionPool) querySticky(id uint32, req *ADNLRequest) (string, error) { +func (c *ConnectionPool) querySticky(id uint32, req *ADNLRequest) (*connection, error) { c.nodesMx.RLock() for _, node := range c.activeNodes { if node.id == id { - host, err := node.queryAdnl(req.QueryID, req.Data) + atomic.AddInt64(&node.weight, -1) + _, err := node.queryAdnl(req.QueryID, req.Data) if err == nil { c.nodesMx.RUnlock() - return host, nil + return node, nil } break } @@ -224,40 +230,33 @@ func (c *ConnectionPool) querySticky(id uint32, req *ADNLRequest) (string, error c.nodesMx.RUnlock() // fallback if bounded node is not available - return c.queryWithBalancer(req) + return c.queryWithSmartBalancer(req) } -func (c *ConnectionPool) queryWithBalancer(req *ADNLRequest) (string, error) { - nodeOffset := atomic.AddUint64(&c.roundRobinOffset, 1) +func (c *ConnectionPool) queryWithSmartBalancer(req *ADNLRequest) (*connection, error) { + var reqNode *connection - var firstNode *connection - for { - c.nodesMx.RLock() - if len(c.activeNodes) == 0 { - c.nodesMx.RUnlock() - return "", ErrNoActiveConnections - } - reqNode := c.activeNodes[nodeOffset%uint64(len(c.activeNodes))] - c.nodesMx.RUnlock() - - if firstNode == nil { - firstNode = reqNode - } else if reqNode == firstNode { - // all nodes were tried, nothing works - return "", ErrNoActiveConnections + c.nodesMx.RLock() + for _, node := range c.activeNodes { + if reqNode == nil { + reqNode = node + continue } - host, err := reqNode.queryAdnl(req.QueryID, req.Data) - if err != nil { - if !errors.Is(err, NetworkErr{}) { - return "", err - } - nodeOffset++ - continue + nw, old := atomic.LoadInt64(&node.weight), atomic.LoadInt64(&reqNode.weight) + if nw > old || (nw == old && atomic.LoadInt64(&node.lastRespTime) < atomic.LoadInt64(&reqNode.lastRespTime)) { + reqNode = node } + } + c.nodesMx.RUnlock() + + atomic.AddInt64(&reqNode.weight, -1) - return host, nil + _, err := reqNode.queryAdnl(req.QueryID, req.Data) + if err != nil { + return nil, err } + return reqNode, nil } func (c *ConnectionPool) SetOnDisconnect(cb OnDisconnectCallback) { diff --git a/ton/transactions.go b/ton/transactions.go index f6aaabc0..a87145c8 100644 --- a/ton/transactions.go +++ b/ton/transactions.go @@ -116,6 +116,10 @@ func (c *APIClient) GetTransaction(ctx context.Context, block *BlockIDExt, addr switch t := resp.(type) { case TransactionInfo: + if len(t.Transaction) == 0 { + return nil, ErrNoTransactionsWereFound + } + if !t.ID.Equals(block) { return nil, fmt.Errorf("incorrect block in response") } From ec14b0dbdeaa463e443979e24460b36bef5d6254 Mon Sep 17 00:00:00 2001 From: iam047801 Date: Fri, 13 Oct 2023 13:59:37 +0530 Subject: [PATCH 02/23] tlb: register type with custom name --- tlb/register.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tlb/register.go b/tlb/register.go index 3c35ced3..1bbcf582 100644 --- a/tlb/register.go +++ b/tlb/register.go @@ -9,8 +9,7 @@ var registered = map[string]reflect.Type{} var magicType = reflect.TypeOf(Magic{}) -func Register(typ any) { - t := reflect.TypeOf(typ) +func register(name string, t reflect.Type) { magic := t.Field(0) if magic.Type != magicType { panic("first field is not magic") @@ -21,5 +20,15 @@ func Register(typ any) { panic("invalid magic tag") } - registered[t.Name()] = t + registered[name] = t +} + +func RegisterWithName(name string, typ any) { + t := reflect.TypeOf(typ) + register(name, t) +} + +func Register(typ any) { + t := reflect.TypeOf(typ) + register(t.Name(), t) } From 5d668686a931769987099ae87507f14d6fdc40fb Mon Sep 17 00:00:00 2001 From: iam047801 Date: Sun, 15 Oct 2023 17:48:38 +0530 Subject: [PATCH 03/23] tlb: load dictionary into the map --- tlb/loader.go | 36 +++++++++++++++++++++++++++++++++++- tlb/loader_test.go | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/tlb/loader.go b/tlb/loader.go index 65577904..adda5034 100644 --- a/tlb/loader.go +++ b/tlb/loader.go @@ -329,7 +329,41 @@ func loadFromCell(v any, slice *cell.Slice, skipProofBranches, skipMagic bool) e } } - setVal(reflect.ValueOf(dict)) + if len(settings) < 4 || settings[2] != "->" { + setVal(reflect.ValueOf(dict)) + continue + } + if structField.Type.Kind() != reflect.Map { + return fmt.Errorf("can map dictionary only into the map") + } + if structField.Type.Key() != reflect.TypeOf("") { + return fmt.Errorf("can map dictionary only into the map with string key") + } + + mappedDict := reflect.MakeMapWithSize(reflect.MapOf(structField.Type.Key(), structField.Type.Elem()), 0) + for _, kv := range dict.All() { + dictK, err := kv.Key.BeginParse().LoadBigUInt(uint(sz)) + if err != nil { + return fmt.Errorf("failed to load dict key for %s: %w", structField.Name, err) + } + + dictVT := reflect.StructOf([]reflect.StructField{{ + Name: "Value", + Type: structField.Type.Elem(), + Tag: reflect.StructTag(fmt.Sprintf("tlb:%q", strings.Join(settings[3:], " "))), + }}) + + dictV := reflect.New(dictVT).Interface() + + err = loadFromCell(dictV, kv.Value.BeginParse(), skipProofBranches, false) + if err != nil { + return fmt.Errorf("failed to load dict value for %v: %w", structField.Name, err) + } + + mappedDict.SetMapIndex(reflect.ValueOf(dictK.String()), reflect.ValueOf(dictV).Elem().Field(0)) + } + + setVal(mappedDict) continue } else if settings[0] == "var" { if settings[1] == "uint" { diff --git a/tlb/loader_test.go b/tlb/loader_test.go index 212fa331..c9eb1723 100644 --- a/tlb/loader_test.go +++ b/tlb/loader_test.go @@ -207,3 +207,42 @@ func TestLoadFromCell(t *testing.T) { t.Fatal("cell hashes not same after From to") } } + +func TestLoadFromCell_MappedDict(t *testing.T) { + dict := cell.NewDict(3) + + b := cell.BeginCell() + + err := b.StoreBoolBit(true) + if err != nil { + t.Fatal(err) + } + + err = dict.SetIntKey(big.NewInt(1), b.EndCell()) + if err != nil { + t.Fatal(err) + } + + b = cell.BeginCell() + if err := b.StoreDict(dict); err != nil { + t.Fatal(err) + } + + var ret struct { + Value map[string]bool `tlb:"dict 3 -> bool"` + } + + err = LoadFromCell(&ret, b.EndCell().BeginParse()) + if err != nil { + t.Fatal(err) + } + + j, err := json.Marshal(ret) + if err != nil { + t.Fatal(err) + } + + if string(j) != "{\"Value\":{\"1\":true}}" { + t.Fatal("wrong map json") + } +} From db292e6fe3333a266388564ebbdc50654348eb96 Mon Sep 17 00:00:00 2001 From: iam047801 Date: Sun, 15 Oct 2023 20:40:06 +0530 Subject: [PATCH 04/23] tlb: dictionary to cell from the map --- tlb/loader.go | 53 +++++++++++++++++++++++++++++++++++++++++- tlb/loader_test.go | 58 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 107 insertions(+), 4 deletions(-) diff --git a/tlb/loader.go b/tlb/loader.go index adda5034..8312ab6d 100644 --- a/tlb/loader.go +++ b/tlb/loader.go @@ -627,7 +627,58 @@ func ToCell(v any) (*cell.Cell, error) { return nil, fmt.Errorf("failed to store magic: %w", err) } } else if settings[0] == "dict" { - err := builder.StoreDict(fieldVal.Interface().(*cell.Dictionary)) + var dict *cell.Dictionary + + if len(settings) < 4 || settings[2] != "->" { + dict = fieldVal.Interface().(*cell.Dictionary) + } else { + if fieldVal.Kind() != reflect.Map { + return nil, fmt.Errorf("want to create dictionary from map, but instead got %s type", fieldVal.Type()) + } + if fieldVal.Type().Key() != reflect.TypeOf("") { + return nil, fmt.Errorf("map key should be string, but instead got %s type", fieldVal.Type().Key()) + } + + sz, err := strconv.ParseUint(settings[1], 10, 64) + if err != nil { + panic(fmt.Sprintf("cannot deserialize field '%s' as dict, bad size '%s'", structField.Name, settings[1])) + } + + dict = cell.NewDict(uint(sz)) + + for _, mapK := range fieldVal.MapKeys() { + mapKI, ok := big.NewInt(0).SetString(mapK.Interface().(string), 10) + if !ok { + return nil, fmt.Errorf("cannot parse '%s' map key to big int of '%s' field", mapK.Interface().(string), structField.Name) + } + + mapKB := cell.BeginCell() + if err := mapKB.StoreBigInt(mapKI, uint(sz)); err != nil { + return nil, fmt.Errorf("store big int of size %d to %s field", sz, structField.Name) + } + + mapV := fieldVal.MapIndex(mapK) + + cellVT := reflect.StructOf([]reflect.StructField{{ + Name: "Value", + Type: mapV.Type(), + Tag: reflect.StructTag(fmt.Sprintf("tlb:%q", strings.Join(settings[3:], " "))), + }}) + cellV := reflect.New(cellVT).Elem() + cellV.Field(0).Set(mapV) + + mapVC, err := ToCell(cellV.Interface()) + if err != nil { + return nil, fmt.Errorf("creating cell for dict value of '%s' field: %w", structField.Name, err) + } + + if err := dict.Set(mapKB.EndCell(), mapVC); err != nil { + return nil, fmt.Errorf("set dict key/value on '%s' field: %w", structField.Name, err) + } + } + } + + err := builder.StoreDict(dict) if err != nil { return nil, fmt.Errorf("failed to store dict for %s, err: %w", structField.Name, err) } diff --git a/tlb/loader_test.go b/tlb/loader_test.go index c9eb1723..84403cd5 100644 --- a/tlb/loader_test.go +++ b/tlb/loader_test.go @@ -3,8 +3,10 @@ package tlb import ( "bytes" "encoding/json" + "fmt" "math/big" "os" + "reflect" "testing" "github.com/xssnick/tonutils-go/address" @@ -47,6 +49,12 @@ type testAny struct { StructAny any `tlb:"^ [StructA,StructB]"` } +type testDict struct { + Dict *cell.Dictionary `tlb:"dict 256"` + DictMapBool map[string]bool `tlb:"dict 55 -> bool"` + DictMapUint map[string]uint64 `tlb:"dict 77 -> ## 43"` +} + type testInner struct { _ Magic `tlb:"$1011"` Val int64 `tlb:"## 34"` @@ -57,7 +65,7 @@ type testInner struct { B bool `tlb:"bool"` Addr *address.Address `tlb:"addr"` Manual manualLoad `tlb:"."` - Dict *cell.Dictionary `tlb:"dict 256"` + Dict testDict `tlb:"^"` StructMaybe *smallStruct `tlb:"maybe ^"` } @@ -97,6 +105,14 @@ func TestLoadAnyRegistered(t *testing.T) { json.NewEncoder(os.Stdout).Encode(v2) } +func mustParseInt(x string) *big.Int { + ret, ok := new(big.Int).SetString(x, 10) + if !ok { + panic(fmt.Errorf("big int from '%s'", ret)) + } + return ret +} + func TestLoadFromCell(t *testing.T) { addr := address.MustParseAddr("EQCD39VS5jcptHL8vMjEXrzGaRcCVYto7HUn4bpAOg8xqB2N") dKey := cell.BeginCell().MustStoreSlice(addr.Data(), 256).EndCell() @@ -108,6 +124,33 @@ func TestLoadFromCell(t *testing.T) { t.Fatal(err) } + dMapBoolKV := map[string]bool{"43": true, "76": false, "79": true} + dMapBool := cell.NewDict(55) + for k, v := range dMapBoolKV { + err := dMapBool.Set( + cell.BeginCell().MustStoreBigInt(mustParseInt(k), 55).EndCell(), + cell.BeginCell().MustStoreBoolBit(v).EndCell()) + if err != nil { + t.Fatal(err) + } + } + dMapIntKV := map[string]uint64{"43": 43, "76": 75, "79": 79} + dMapInt := cell.NewDict(77) + for k, v := range dMapIntKV { + err := dMapInt.Set( + cell.BeginCell().MustStoreBigInt(mustParseInt(k), 77).EndCell(), + cell.BeginCell().MustStoreUInt(v, 43).EndCell()) + if err != nil { + t.Fatal(err) + } + } + + dictC := cell.BeginCell(). + MustStoreDict(d). + MustStoreDict(dMapBool). + MustStoreDict(dMapInt). + EndCell() + mRef := cell.BeginCell().MustStoreUInt('y', 8).EndCell() ref := cell.BeginCell().MustStoreUInt(0b1011, 4). @@ -116,7 +159,10 @@ func TestLoadFromCell(t *testing.T) { MustStoreCoins(700000). MustStoreUInt(5, 10). MustStoreUInt(7126382921832, 176). - MustStoreBoolBit(true).MustStoreAddr(addr).MustStoreUInt('x', 8).MustStoreDict(d). + MustStoreBoolBit(true). + MustStoreAddr(addr). + MustStoreUInt('x', 8). + MustStoreRef(dictC). MustStoreMaybeRef(mRef) a := cell.BeginCell().MustStoreUInt(0xFFAA, 16). @@ -181,9 +227,15 @@ func TestLoadFromCell(t *testing.T) { t.Fatal("manual not eq") } - if !bytes.Equal(x.Part.Dict.Get(dKey).Hash(), dVal.Hash()) { + if !bytes.Equal(x.Part.Dict.Dict.Get(dKey).Hash(), dVal.Hash()) { t.Fatal("dict val not eq") } + if !reflect.DeepEqual(x.Part.Dict.DictMapBool, dMapBoolKV) { + t.Fatal("bool dict val not eq") + } + if !reflect.DeepEqual(x.Part.Dict.DictMapUint, dMapIntKV) { + t.Fatal("uint dict val not eq") + } if x.Var.Uint64() != 999 { t.Fatal("var not eq") From bf36199915c40de9b77d44fccae63d1d96bc8eb1 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Tue, 17 Oct 2023 08:53:24 +0400 Subject: [PATCH 05/23] TLB Map loader tests adjustments --- tlb/loader.go | 12 ++++++------ tlb/loader_test.go | 41 +++++++++++++++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/tlb/loader.go b/tlb/loader.go index 8312ab6d..868ca774 100644 --- a/tlb/loader.go +++ b/tlb/loader.go @@ -341,18 +341,18 @@ func loadFromCell(v any, slice *cell.Slice, skipProofBranches, skipMagic bool) e } mappedDict := reflect.MakeMapWithSize(reflect.MapOf(structField.Type.Key(), structField.Type.Elem()), 0) + dictVT := reflect.StructOf([]reflect.StructField{{ + Name: "Value", + Type: structField.Type.Elem(), + Tag: reflect.StructTag(fmt.Sprintf("tlb:%q", strings.Join(settings[3:], " "))), + }}) + for _, kv := range dict.All() { dictK, err := kv.Key.BeginParse().LoadBigUInt(uint(sz)) if err != nil { return fmt.Errorf("failed to load dict key for %s: %w", structField.Name, err) } - dictVT := reflect.StructOf([]reflect.StructField{{ - Name: "Value", - Type: structField.Type.Elem(), - Tag: reflect.StructTag(fmt.Sprintf("tlb:%q", strings.Join(settings[3:], " "))), - }}) - dictV := reflect.New(dictVT).Interface() err = loadFromCell(dictV, kv.Value.BeginParse(), skipProofBranches, false) diff --git a/tlb/loader_test.go b/tlb/loader_test.go index 84403cd5..a79775c5 100644 --- a/tlb/loader_test.go +++ b/tlb/loader_test.go @@ -50,9 +50,10 @@ type testAny struct { } type testDict struct { - Dict *cell.Dictionary `tlb:"dict 256"` - DictMapBool map[string]bool `tlb:"dict 55 -> bool"` - DictMapUint map[string]uint64 `tlb:"dict 77 -> ## 43"` + Dict *cell.Dictionary `tlb:"dict 256"` + DictMapBool map[string]bool `tlb:"dict 55 -> bool"` + DictMapUint map[string]uint64 `tlb:"dict 77 -> ## 43"` + DictMapStruct map[string]any `tlb:"dict 128 -> ^ [StructA,StructC]"` } type testInner struct { @@ -114,6 +115,9 @@ func mustParseInt(x string) *big.Int { } func TestLoadFromCell(t *testing.T) { + Register(StructA{}) + Register(StructC{}) + addr := address.MustParseAddr("EQCD39VS5jcptHL8vMjEXrzGaRcCVYto7HUn4bpAOg8xqB2N") dKey := cell.BeginCell().MustStoreSlice(addr.Data(), 256).EndCell() dVal := cell.BeginCell().MustStoreAddr(addr).EndCell() @@ -144,11 +148,23 @@ func TestLoadFromCell(t *testing.T) { t.Fatal(err) } } + dMapStructKV := map[string]any{"43": StructA{Val: 1}, "322": StructC{Val: true}} + dMapStruct := cell.NewDict(128) + for k, v := range dMapStructKV { + cl, _ := ToCell(v) + err := dMapStruct.Set( + cell.BeginCell().MustStoreBigInt(mustParseInt(k), 128).EndCell(), + cell.BeginCell().MustStoreRef(cl).EndCell()) + if err != nil { + t.Fatal(err) + } + } dictC := cell.BeginCell(). MustStoreDict(d). MustStoreDict(dMapBool). MustStoreDict(dMapInt). + MustStoreDict(dMapStruct). EndCell() mRef := cell.BeginCell().MustStoreUInt('y', 8).EndCell() @@ -236,6 +252,9 @@ func TestLoadFromCell(t *testing.T) { if !reflect.DeepEqual(x.Part.Dict.DictMapUint, dMapIntKV) { t.Fatal("uint dict val not eq") } + if !reflect.DeepEqual(x.Part.Dict.DictMapStruct, dMapStructKV) { + t.Fatal("struct dict val not eq") + } if x.Var.Uint64() != 999 { t.Fatal("var not eq") @@ -270,7 +289,7 @@ func TestLoadFromCell_MappedDict(t *testing.T) { t.Fatal(err) } - err = dict.SetIntKey(big.NewInt(1), b.EndCell()) + err = dict.SetIntKey(big.NewInt(1), cell.BeginCell().MustStoreRef(b.EndCell()).EndCell()) if err != nil { t.Fatal(err) } @@ -281,10 +300,11 @@ func TestLoadFromCell_MappedDict(t *testing.T) { } var ret struct { - Value map[string]bool `tlb:"dict 3 -> bool"` + Value map[string]bool `tlb:"dict 3 -> ^ bool"` } - err = LoadFromCell(&ret, b.EndCell().BeginParse()) + x := b.EndCell() + err = LoadFromCell(&ret, x.BeginParse()) if err != nil { t.Fatal(err) } @@ -297,4 +317,13 @@ func TestLoadFromCell_MappedDict(t *testing.T) { if string(j) != "{\"Value\":{\"1\":true}}" { t.Fatal("wrong map json") } + + cl, err := ToCell(ret) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(cl.Hash(), x.Hash()) { + t.Fatal("wrong hash") + } } From f75303ca2929bd105078a95446f0e23340475e09 Mon Sep 17 00:00:00 2001 From: Tikhon Slasten Date: Mon, 23 Oct 2023 15:01:10 +0800 Subject: [PATCH 06/23] Measure matrix size --- adnl/rldp/raptorq/discmath/matrix-gf2.go | 2 + adnl/rldp/raptorq/discmath/metrics.go | 94 ++++++++++++++++++++++++ adnl/rldp/raptorq/solver_test.go | 8 ++ 3 files changed, 104 insertions(+) create mode 100644 adnl/rldp/raptorq/discmath/metrics.go diff --git a/adnl/rldp/raptorq/discmath/matrix-gf2.go b/adnl/rldp/raptorq/discmath/matrix-gf2.go index 9eb7f6c6..5e75c8c6 100644 --- a/adnl/rldp/raptorq/discmath/matrix-gf2.go +++ b/adnl/rldp/raptorq/discmath/matrix-gf2.go @@ -10,6 +10,8 @@ type MatrixGF2 struct { } func NewMatrixGF2(rows, cols uint32) *MatrixGF2 { + // defaultMetrics.store(rows, cols) // only for tests + data := make([][]uint8, rows) for i := range data { data[i] = make([]uint8, cols) diff --git a/adnl/rldp/raptorq/discmath/metrics.go b/adnl/rldp/raptorq/discmath/metrics.go new file mode 100644 index 00000000..537a3f7c --- /dev/null +++ b/adnl/rldp/raptorq/discmath/metrics.go @@ -0,0 +1,94 @@ +package discmath + +import "fmt" + +var defaultMetrics *metrics + +func init() { + defaultMetrics = newMetrics() +} + +type metrics struct { + isFirst bool + minRows uint32 + maxRows uint32 + avgRows uint32 + minCols uint32 + maxCols uint32 + avgCols uint32 + minSize uint32 + maxSize uint32 + avgSize uint32 +} + +func newMetrics() *metrics { + return &metrics{} +} + +func (m *metrics) store(rows, cols uint32) { + size := rows * cols + + if m.isFirst { + m.isFirst = false + m.minRows = rows + m.maxRows = rows + m.avgRows = rows + m.minCols = cols + m.maxCols = cols + m.avgCols = cols + m.minSize = size + m.maxSize = size + m.avgSize = size + + return + } + + if m.minRows > rows { + m.minRows = rows + } + + if m.maxRows < rows { + m.maxRows = rows + } + + m.avgRows = (m.avgRows + rows) / 2 + + if m.minCols > cols { + m.minCols = cols + } + + if m.maxCols < cols { + m.maxCols = cols + } + + m.avgCols = (m.avgCols + cols) / 2 + + if m.minSize > size { + m.minSize = size + } + + if m.maxSize < size { + m.maxSize = size + } + + m.avgSize = (m.avgSize + size) / 2 +} + +func (m *metrics) String() string { + return fmt.Sprintf( + "\n--- Rows ---\nmin=%d max=%d avg=%d\n--- Cols ---\nmin=%d max=%d avg=%d\n--- Size ---\nmin=%d max=%d avg=%d\n", + m.minRows, + m.maxRows, + m.avgRows, + m.minCols, + m.maxCols, + m.avgCols, + m.minSize, + m.maxSize, + m.avgSize, + ) +} + +func GetMetrics() string { + return defaultMetrics.String() +} diff --git a/adnl/rldp/raptorq/solver_test.go b/adnl/rldp/raptorq/solver_test.go index e287b1fa..e31e3de1 100644 --- a/adnl/rldp/raptorq/solver_test.go +++ b/adnl/rldp/raptorq/solver_test.go @@ -6,6 +6,8 @@ import ( "encoding/binary" "encoding/hex" "testing" + + "github.com/xssnick/tonutils-go/adnl/rldp/raptorq/discmath" ) func Test_Encode(t *testing.T) { @@ -23,6 +25,8 @@ func Test_Encode(t *testing.T) { if !bytes.Equal(sx, should) { t.Fatal("encoded not eq, got", hex.EncodeToString(sx)) } + + t.Log(discmath.GetMetrics()) } func Test_EncodeDecode(t *testing.T) { @@ -56,6 +60,8 @@ func Test_EncodeDecode(t *testing.T) { if !bytes.Equal(data, str) { t.Fatal("initial data not eq decrypted") } + + t.Log(discmath.GetMetrics()) } func Test_EncodeDecodeFuzz(t *testing.T) { @@ -104,6 +110,8 @@ func Test_EncodeDecodeFuzz(t *testing.T) { t.Fatal("initial data not eq decrypted") } } + + t.Log(discmath.GetMetrics()) } func Benchmark_EncodeDecodeFuzz(b *testing.B) { From 5f638eebf11285c751a62e53d18834df2bb5cddf Mon Sep 17 00:00:00 2001 From: Tikhon Slasten Date: Mon, 30 Oct 2023 11:20:39 +0800 Subject: [PATCH 07/23] Implement matrix GF2 as slice of bytes --- adnl/rldp/raptorq/discmath/matrix-gf2.go | 163 ++++++ adnl/rldp/raptorq/discmath/matrix-gf2_test.go | 513 ++++++++++++++++++ 2 files changed, 676 insertions(+) create mode 100644 adnl/rldp/raptorq/discmath/matrix-gf2_test.go diff --git a/adnl/rldp/raptorq/discmath/matrix-gf2.go b/adnl/rldp/raptorq/discmath/matrix-gf2.go index 5e75c8c6..0b56842e 100644 --- a/adnl/rldp/raptorq/discmath/matrix-gf2.go +++ b/adnl/rldp/raptorq/discmath/matrix-gf2.go @@ -2,6 +2,7 @@ package discmath import ( "fmt" + "math/bits" "strings" ) @@ -83,3 +84,165 @@ func (m *MatrixGF2) String() string { } return strings.Join(rows, "\n") } + +// PlainMatrixGF2 is a matrix of GF2 elements +// Data is stored in a byte array, each byte is contained 8 elements (bits) +type PlainMatrixGF2 struct { + rows, cols uint32 + data []byte +} + +const ( + // elSize is a size of array's element in bits + elSize = 8 + maxEl = 1< 0 { + dataSize++ + } + + return &PlainMatrixGF2{ + rows: rows, + cols: cols, + data: make([]byte, dataSize), + } +} + +func (m *PlainMatrixGF2) RowsNum() uint32 { + return m.rows +} + +func (m *PlainMatrixGF2) ColsNum() uint32 { + return m.cols +} + +func (m *PlainMatrixGF2) Get(row, col uint32) byte { + idx, bit := m.getIdx(row, col) + + res := m.data[idx] & bit + if res > 0 { + return 1 + } + + return 0 +} + +func (m *PlainMatrixGF2) Set(row, col uint32) { + idx, bit := m.getIdx(row, col) + m.data[idx] |= bit +} + +func (m *PlainMatrixGF2) Unset(row, col uint32) { + idx, bit := m.getIdx(row, col) + m.data[idx] &= ^bit +} + +// GetRow returns row data, bit mask for the first byte and bit mask for the last byte +// Bitmask contains 1 for each bit that is part of the row +func (m *PlainMatrixGF2) GetRow(row uint32) ([]byte, byte, byte) { + firstBitIdx := row * m.cols + firstElIdx := firstBitIdx / elSize + + lastBitIdx := (row+1)*m.cols - 1 + lastElIdx := lastBitIdx / elSize + + firstBitmask := byte(maxEl - (1 << (firstBitIdx % elSize)) + 1) + lastBitmask := byte((1 << (lastBitIdx%elSize + 1)) - 1) + + if firstElIdx == lastElIdx { + firstBitmask &= lastBitmask + lastBitmask = firstBitmask + } + + if int(lastElIdx) == len(m.data) { + return m.data[firstElIdx:], firstBitmask, lastBitmask + } + + return m.data[firstElIdx : lastElIdx+1], firstBitmask, lastBitmask +} + +func (m *PlainMatrixGF2) RowAdd(row uint32, what []byte, firstBitmask, lastBitmask byte) { + col := uint32(0) + for i, whatByte := range what { + // compute bitmask of "what" row for current byte + bitmask := byte(maxEl) + if i == 0 { + bitmask = firstBitmask + } else if i == len(what)-1 { + bitmask = lastBitmask + } + + for j := bits.TrailingZeros8(bitmask); j < elSize-bits.LeadingZeros8(bitmask); j++ { + whatBit := (whatByte & (1 << j)) >> j + colBit := m.Get(row, col) + + if whatBit != colBit { + m.Set(row, col) + } else { + m.Unset(row, col) + } + + col++ + } + } +} + +func (m *PlainMatrixGF2) Mul(s *MatrixGF256) *PlainMatrixGF2 { + mg := NewPlainMatrixGF2(s.RowsNum(), m.ColsNum()) + + s.Each(func(row, col uint32) { + rowData, firstBitmask, lastBitmask := m.GetRow(col) + mg.RowAdd(row, rowData, firstBitmask, lastBitmask) + }) + + return mg +} + +func (m *PlainMatrixGF2) ToGF256() *MatrixGF256 { + mg := NewMatrixGF256(m.RowsNum(), m.ColsNum()) + + for i := uint32(0); i < m.rows; i++ { + mg.rows[i] = GF256{data: m.getRowUInt8(i)} + } + + return mg +} + +func (m *PlainMatrixGF2) String() string { + var rows []string + for i := uint32(0); i < m.rows; i++ { + var cols []string + for j := uint32(0); j < m.cols; j++ { + cols = append(cols, fmt.Sprintf("%02x", m.Get(i, j))) + } + + rows = append(rows, strings.Join(cols, " ")) + } + + return strings.Join(rows, "\n") +} + +// getRowUInt8 returns row data as array of uint8 +func (m *PlainMatrixGF2) getRowUInt8(row uint32) []uint8 { + rowUInt8 := make([]uint8, m.cols) + for i := uint32(0); i < m.cols; i++ { + rowUInt8[i] = m.Get(row, i) + } + + return rowUInt8 +} + +// getIdx returns index in array of bytes and offset in this byte +func (m *PlainMatrixGF2) getIdx(row, col uint32) (uint32, byte) { + idx := row*m.cols + col + bitIdx := byte(idx % elSize) + + return idx / elSize, byte(1 << bitIdx) +} diff --git a/adnl/rldp/raptorq/discmath/matrix-gf2_test.go b/adnl/rldp/raptorq/discmath/matrix-gf2_test.go new file mode 100644 index 00000000..45d023ed --- /dev/null +++ b/adnl/rldp/raptorq/discmath/matrix-gf2_test.go @@ -0,0 +1,513 @@ +package discmath + +import ( + "reflect" + "testing" +) + +func TestNewPlainMatrixGF2(t *testing.T) { + type args struct { + rows uint32 + cols uint32 + } + + tests := []struct { + name string + args args + want *PlainMatrixGF2 + }{ + { + name: "1x1", + args: args{ + rows: 1, + cols: 1, + }, + want: &PlainMatrixGF2{ + rows: 1, + cols: 1, + data: []byte{0}, + }, + }, + { + name: "size lower one element", + args: args{ + rows: 2, + cols: 3, + }, + want: &PlainMatrixGF2{ + rows: 2, + cols: 3, + data: []byte{0}, + }, + }, + { + name: "size greater one element", + args: args{ + rows: 2, + cols: 7, + }, + want: &PlainMatrixGF2{ + rows: 2, + cols: 7, + data: []byte{0, 0}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NewPlainMatrixGF2(tt.args.rows, tt.args.cols) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewPlainMatrixGF2() = %v, want %v", got, tt.want) + } + + gotRows := got.RowsNum() + if gotRows != tt.want.rows { + t.Errorf("RowsNum() = %v, want %v", gotRows, tt.want.rows) + } + + gotCols := got.ColsNum() + if gotCols != tt.want.cols { + t.Errorf("ColsNum() = %v, want %v", gotCols, tt.want.cols) + } + }) + } +} + +func TestPlainMatrixGF2_Get(t *testing.T) { + type args struct { + row uint32 + col uint32 + } + + tests := []struct { + name string + m *PlainMatrixGF2 + args args + want byte + }{ + { + name: "get 0", + m: &PlainMatrixGF2{ + rows: 2, + cols: 5, + data: []byte{131, 0}, // 1000 0011 + }, + args: args{ + row: 1, + col: 4, + }, + want: 0, + }, + { + name: "get 1", + m: &PlainMatrixGF2{ + rows: 2, + cols: 5, + data: []byte{0, 2}, // 0000 0010 + }, + args: args{ + row: 1, + col: 4, + }, + want: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.m.Get(tt.args.row, tt.args.col) + if got != tt.want { + t.Errorf("Get() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPlainMatrixGF2_Set(t *testing.T) { + type args struct { + row uint32 + col uint32 + } + + tests := []struct { + name string + m *PlainMatrixGF2 + args args + want []byte + }{ + { + name: "3x3 set [0,1]", + m: &PlainMatrixGF2{ + rows: 3, + cols: 3, + data: []byte{0, 0}, + }, + args: args{ + row: 0, + col: 1, + }, + want: []byte{2, 0}, + }, + { + name: "3x3 set [2,2]", + m: &PlainMatrixGF2{ + rows: 3, + cols: 3, + data: []byte{0, 0}, + }, + args: args{ + row: 2, + col: 2, + }, + want: []byte{0, 1}, + }, + { + name: "2x10, set [1,9]", + m: &PlainMatrixGF2{ + rows: 2, + cols: 10, + data: []byte{0, 0, 0}, + }, + args: args{ + row: 1, + col: 9, + }, + want: []byte{0, 0, 8}, + }, + { + name: "set bit which is already set", + m: &PlainMatrixGF2{ + rows: 3, + cols: 3, + data: []byte{24, 0}, // 1: 1 1 0 + }, + args: args{ + row: 1, + col: 1, + }, + want: []byte{24, 0}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.m.Set(tt.args.row, tt.args.col) + if !reflect.DeepEqual(tt.m.data, tt.want) { + t.Errorf("Set() = %v, want %v", tt.m.data, tt.want) + } + }) + } +} + +func TestPlainMatrixGF2_Unset(t *testing.T) { + type args struct { + row uint32 + col uint32 + } + + tests := []struct { + name string + m *PlainMatrixGF2 + args args + want []byte + }{ + { + name: "3x3 unset [0,1]", + m: &PlainMatrixGF2{ + rows: 3, + cols: 3, + data: []byte{2, 0}, + }, + args: args{ + row: 0, + col: 1, + }, + want: []byte{0, 0}, + }, + { + name: "3x3 unset [1,2]", + m: &PlainMatrixGF2{ + rows: 3, + cols: 3, + data: []byte{32, 0}, + }, + args: args{ + row: 1, + col: 2, + }, + want: []byte{0, 0}, + }, + { + name: "unset bit which is already unset", + m: &PlainMatrixGF2{ + rows: 3, + cols: 3, + data: []byte{0, 0}, + }, + args: args{ + row: 1, + col: 1, + }, + want: []byte{0, 0}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.m.Unset(tt.args.row, tt.args.col) + if !reflect.DeepEqual(tt.m.data, tt.want) { + t.Errorf("Unset() = %v, want %v", tt.m.data, tt.want) + } + }) + } +} + +func TestPlainMatrixGF2_GetRow(t *testing.T) { + tests := []struct { + name string + m *PlainMatrixGF2 + row uint32 + want []byte + wantFirstBitmask byte + wantLastBitmask byte + }{ + { + name: "all row in one element", + m: &PlainMatrixGF2{ + rows: 3, + cols: 2, + data: []byte{8}, // 00 00[10]00 + }, + row: 1, + want: []byte{8}, + wantFirstBitmask: 12, // 00 00[11]00 + wantLastBitmask: 12, // 00 00[11]00 + }, + { + name: "row in few elements", + m: &PlainMatrixGF2{ + rows: 2, + cols: 10, + data: []byte{0, 248, 12}, // 0: 0000 0000 | 1: [1111 10]00 | 2: 0000 [1100] + }, + row: 1, + want: []byte{248, 12}, + wantFirstBitmask: 252, + wantLastBitmask: 15, + }, + { + name: "last row", + m: &PlainMatrixGF2{ + rows: 3, + cols: 8, + data: []byte{0, 0, 0}, + }, + row: 2, + want: []byte{0}, + wantFirstBitmask: 255, + wantLastBitmask: 255, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, gotFirstBitmask, gotLastBitmask := tt.m.GetRow(tt.row) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetRow() got = %v, want %v", got, tt.want) + } + + if gotFirstBitmask != tt.wantFirstBitmask { + t.Errorf("GetRow() gotFirstBitmask = %v, want %v", gotFirstBitmask, tt.wantFirstBitmask) + } + + if gotLastBitmask != tt.wantLastBitmask { + t.Errorf("GetRow() gotLastBitmask = %v, want %v", gotLastBitmask, tt.wantLastBitmask) + } + }) + } +} + +func TestPlainMatrixGF2_RowAdd(t *testing.T) { + type args struct { + row uint32 + what []byte + firstBitmask byte + lastBitmask byte + } + + tests := []struct { + name string + m *PlainMatrixGF2 + args args + want []byte + }{ + { + name: "same rows", + m: &PlainMatrixGF2{ + rows: 3, + cols: 3, + data: []byte{146, 0}, // 0: 10[010]010 + }, + args: args{ + row: 1, + what: []byte{24}, // 00[011]000 + firstBitmask: 56, // 00[111]000 + lastBitmask: 56, // 00[111]000 + }, + want: []byte{138, 0}, // 10[001]010 + }, + { + name: "different rows: #1 + #2", + m: &PlainMatrixGF2{ + rows: 3, + cols: 3, + data: []byte{146, 0}, // 0: 10[010]010 | 1: 0000 0000 + }, + args: args{ + row: 1, + what: []byte{128, 1}, // 0: 10]00 0000 | 1: 0000 000[1 + firstBitmask: 192, + lastBitmask: 1, + }, + want: []byte{162, 0}, // 0: 10[100]010 | 1: 0000 0000 + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.m.RowAdd(tt.args.row, tt.args.what, tt.args.firstBitmask, tt.args.lastBitmask) + if !reflect.DeepEqual(tt.m.data, tt.want) { + t.Errorf("RowAdd() = %v, want %v", tt.m.data, tt.want) + } + }) + } +} + +func TestPlainMatrixGF2_Mul(t *testing.T) { + tests := []struct { + name string + m *PlainMatrixGF2 + s *MatrixGF256 + want *PlainMatrixGF2 + }{ + // TODO: Add test cases. + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.m.Mul(tt.s) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Mul() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPlainMatrixGF2_ToGF256(t *testing.T) { + tests := []struct { + name string + m *PlainMatrixGF2 + want *MatrixGF256 + }{ + { + name: "5x3", + m: &PlainMatrixGF2{ + rows: 5, + cols: 3, + data: []byte{234, 49}, // 0: 11][101][010] | 1: 0[011][000][1 + }, + want: &MatrixGF256{ + rows: []GF256{ + {data: []uint8{0, 1, 0}}, + {data: []uint8{1, 0, 1}}, + {data: []uint8{1, 1, 1}}, + {data: []uint8{0, 0, 0}}, + {data: []uint8{1, 1, 0}}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.m.ToGF256() + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ToGF256() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPlainMatrixGF2_String(t *testing.T) { + tests := []struct { + name string + m *PlainMatrixGF2 + want string + }{ + { + name: "2x3", + m: &PlainMatrixGF2{ + rows: 2, + cols: 3, + data: []byte{42}, // 00 [101][010] + }, + want: "00 01 00\n" + "01 00 01", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.m.String() + if got != tt.want { + t.Errorf("String() = %v, want %v", got, tt.want) + } + }) + } +} + +// Size: 100x100 +// Data size: 10 000 bytes +// BenchmarkMatrixGF2-12: GetRow+RowAdd 114345 10073 ns/op 0 B/op 0 allocs/op +// BenchmarkMatrixGF2-12: GetRow 25560945 43.34 ns/op 0 B/op 0 allocs/op +func BenchmarkMatrixGF2(b *testing.B) { + const dimension = 100 + + m := NewMatrixGF2(dimension, dimension) + for i := uint32(0); i < dimension; i++ { + m.Set(i, i) + } + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for j := uint32(0); j < m.RowsNum()-1; j++ { + what := m.GetRow(j + 1) + m.RowAdd(j, what) + } + } +} + +// Size: 100x100 +// Data size: 1250 bytes +// BenchmarkPlainMatrixGF2-12 GetRow+RowAdd 30560 39130 ns/op 0 B/op 0 allocs/op +// BenchmarkPlainMatrixGF2-12: GetRow 2639367 435.2 ns/op 0 B/op 0 allocs/op +func BenchmarkPlainMatrixGF2(b *testing.B) { + const dimension = 100 + + m := NewPlainMatrixGF2(dimension, dimension) + for i := uint32(0); i < dimension; i++ { + m.Set(i, i) + } + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for j := uint32(0); j < m.RowsNum()-1; j++ { + what, fBitmask, lBitmask := m.GetRow(j + 1) + m.RowAdd(j, what, fBitmask, lBitmask) + } + } +} From d4ced7e6be807d022befbb3b400ef9ab4ccb3920 Mon Sep 17 00:00:00 2001 From: Tikhon Slasten Date: Mon, 30 Oct 2023 11:24:29 +0800 Subject: [PATCH 08/23] Use PlainMatrixGF2 in computations --- adnl/rldp/raptorq/discmath/matrix-gf256.go | 4 ++-- adnl/rldp/raptorq/discmath/sparse-matrix.go | 4 ++-- adnl/rldp/raptorq/solver.go | 5 ++++- adnl/rldp/raptorq/solver_test.go | 15 ++++++++------- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/adnl/rldp/raptorq/discmath/matrix-gf256.go b/adnl/rldp/raptorq/discmath/matrix-gf256.go index a7a89db9..44f41076 100644 --- a/adnl/rldp/raptorq/discmath/matrix-gf256.go +++ b/adnl/rldp/raptorq/discmath/matrix-gf256.go @@ -138,8 +138,8 @@ func (m *MatrixGF256) Each(f func(row, col uint32)) { } } -func (m *MatrixGF256) ToGF2(rowFrom, colFrom, rowSize, colSize uint32) *MatrixGF2 { - mGF2 := NewMatrixGF2(rowSize, colSize) +func (m *MatrixGF256) ToGF2(rowFrom, colFrom, rowSize, colSize uint32) *PlainMatrixGF2 { + mGF2 := NewPlainMatrixGF2(rowSize, colSize) m.Each(func(row, col uint32) { if (row >= rowFrom && row < rowFrom+rowSize) && (col >= colFrom && col < colFrom+colSize) { diff --git a/adnl/rldp/raptorq/discmath/sparse-matrix.go b/adnl/rldp/raptorq/discmath/sparse-matrix.go index 44d2851c..707e0c78 100644 --- a/adnl/rldp/raptorq/discmath/sparse-matrix.go +++ b/adnl/rldp/raptorq/discmath/sparse-matrix.go @@ -101,8 +101,8 @@ func (s *SparseMatrixGF2) ApplyColsPermutation(permutation []uint32) *SparseMatr return res } -func (s *SparseMatrixGF2) ToDense(rowFrom, colFrom, rowSize, colSize uint32) *MatrixGF2 { - m := NewMatrixGF2(rowSize, colSize) +func (s *SparseMatrixGF2) ToDense(rowFrom, colFrom, rowSize, colSize uint32) *PlainMatrixGF2 { + m := NewPlainMatrixGF2(rowSize, colSize) s.Each(func(row, col uint32) { if (row >= rowFrom && row < rowFrom+rowSize) && (col >= colFrom && col < colFrom+colSize) { diff --git a/adnl/rldp/raptorq/solver.go b/adnl/rldp/raptorq/solver.go index 0b23031b..ca08bb48 100644 --- a/adnl/rldp/raptorq/solver.go +++ b/adnl/rldp/raptorq/solver.go @@ -3,6 +3,7 @@ package raptorq import ( "errors" "fmt" + "github.com/xssnick/tonutils-go/adnl/rldp/raptorq/discmath" ) @@ -103,7 +104,9 @@ func (p *raptorParams) Solve(symbols []*Symbol) (*discmath.MatrixGF256, error) { if row >= uSize { break } - e.RowAdd(row, e.GetRow(i)) + + what, fBitmask, lBitmask := e.GetRow(i) + e.RowAdd(row, what, fBitmask, lBitmask) d.RowAdd(row, d.GetRow(i)) } } diff --git a/adnl/rldp/raptorq/solver_test.go b/adnl/rldp/raptorq/solver_test.go index e31e3de1..6453bef7 100644 --- a/adnl/rldp/raptorq/solver_test.go +++ b/adnl/rldp/raptorq/solver_test.go @@ -114,10 +114,15 @@ func Test_EncodeDecodeFuzz(t *testing.T) { t.Log(discmath.GetMetrics()) } +// Benchmark_EncodeDecodeFuzz-12: MatrixGF2 907 1329193 ns/op 653478 B/op 1806 allocs/op +// Benchmark_EncodeDecodeFuzz-12: PlainMatrixGF2 1003 1429374 ns/op 652845 B/op 1809 allocs/op func Benchmark_EncodeDecodeFuzz(b *testing.B) { str := make([]byte, 4096) rand.Read(str) - for n := 0; n < 100; n++ { + + b.ReportAllocs() + + for n := 0; n < b.N; n++ { var symSz uint32 = 768 r := NewRaptorQ(symSz) enc, err := r.CreateEncoder(str) @@ -127,7 +132,7 @@ func Benchmark_EncodeDecodeFuzz(b *testing.B) { dec, err := r.CreateDecoder(uint32(len(str))) if err != nil { - panic(err) + b.Fatal("create decoder err", err) } _, err = dec.AddSymbol(2, enc.GenSymbol(2)) @@ -144,13 +149,9 @@ func Benchmark_EncodeDecodeFuzz(b *testing.B) { } } - _, data, err := dec.Decode() + _, _, err = dec.Decode() if err != nil { b.Fatal("decode err", err) } - - if !bytes.Equal(data, str) { - b.Fatal("initial data not eq decrypted") - } } } From aedac62255028c7253ee9e055c6279d3b9613a26 Mon Sep 17 00:00:00 2001 From: Tikhon Slasten Date: Tue, 31 Oct 2023 13:04:32 +0800 Subject: [PATCH 09/23] Implement PlainOffsetMatrixGF2 --- adnl/rldp/raptorq/discmath/matrix-gf2.go | 113 +++++ adnl/rldp/raptorq/discmath/matrix-gf2_test.go | 456 +++++++++++++++++- 2 files changed, 551 insertions(+), 18 deletions(-) diff --git a/adnl/rldp/raptorq/discmath/matrix-gf2.go b/adnl/rldp/raptorq/discmath/matrix-gf2.go index 0b56842e..98ec0e69 100644 --- a/adnl/rldp/raptorq/discmath/matrix-gf2.go +++ b/adnl/rldp/raptorq/discmath/matrix-gf2.go @@ -246,3 +246,116 @@ func (m *PlainMatrixGF2) getIdx(row, col uint32) (uint32, byte) { return idx / elSize, byte(1 << bitIdx) } + +type PlainOffsetMatrixGF2 struct { + rows, cols uint32 + data [][]byte +} + +func NewPlainOffsetMatrixGF2(rows, cols uint32) *PlainOffsetMatrixGF2 { + // defaultMetrics.store(rows, cols) // only for tests + + rowSize := cols / elSize + if cols%elSize > 0 { + rowSize++ + } + + data := make([][]byte, rows) + for i := range data { + data[i] = make([]byte, rowSize) + } + + return &PlainOffsetMatrixGF2{ + rows: rows, + cols: cols, + data: data, + } +} + +func (m *PlainOffsetMatrixGF2) RowsNum() uint32 { + return m.rows +} + +func (m *PlainOffsetMatrixGF2) ColsNum() uint32 { + return m.cols +} + +func (m *PlainOffsetMatrixGF2) Get(row, col uint32) byte { + return m.getElement(row, col) +} + +func (m *PlainOffsetMatrixGF2) Set(row, col uint32) { + elIdx, colIdx := m.getColPosition(col) + m.data[row][elIdx] |= 1 << colIdx +} + +func (m *PlainOffsetMatrixGF2) Unset(row, col uint32) { + elIdx, colIdx := m.getColPosition(col) + m.data[row][elIdx] &= ^(1 << colIdx) +} + +func (m *PlainOffsetMatrixGF2) GetRow(row uint32) []byte { + return m.data[row] +} + +func (m *PlainOffsetMatrixGF2) RowAdd(row uint32, what []byte) { + for i, whatByte := range what { + m.data[row][i] ^= whatByte + } +} + +func (m *PlainOffsetMatrixGF2) Mul(s *MatrixGF256) *PlainOffsetMatrixGF2 { + mg := NewPlainOffsetMatrixGF2(s.RowsNum(), m.ColsNum()) + + s.Each(func(row, col uint32) { + mRow := m.GetRow(col) + mg.RowAdd(row, mRow) + }) + + return mg +} + +func (m *PlainOffsetMatrixGF2) ToGF256() *MatrixGF256 { + mg := NewMatrixGF256(m.RowsNum(), m.ColsNum()) + + for i := uint32(0); i < m.rows; i++ { + mg.rows[i] = GF256{data: m.getRowUInt8(i)} + } + + return mg +} + +func (m *PlainOffsetMatrixGF2) String() string { + var rows []string + for row := uint32(0); row < m.rows; row++ { + var cols []string + for col := uint32(0); col < m.cols; col++ { + cols = append(cols, fmt.Sprintf("%02x", m.getElement(row, col))) + } + + rows = append(rows, strings.Join(cols, " ")) + } + + return strings.Join(rows, "\n") +} + +func (m *PlainOffsetMatrixGF2) getRowUInt8(row uint32) []uint8 { + result := make([]uint8, m.cols) + for col := uint32(0); col < m.cols; col++ { + result[col] = m.getElement(row, col) + } + + return result +} + +// getElement returns element in matrix by row and col. Possible values: 0 or 1 +func (m *PlainOffsetMatrixGF2) getElement(row, col uint32) byte { + elIdx, colIdx := m.getColPosition(col) + + return (m.data[row][elIdx] & (1 << colIdx)) >> colIdx +} + +// getColPosition returns index of element in array and offset in this element +func (m *PlainOffsetMatrixGF2) getColPosition(col uint32) (uint32, byte) { + return col / elSize, byte(col % elSize) +} diff --git a/adnl/rldp/raptorq/discmath/matrix-gf2_test.go b/adnl/rldp/raptorq/discmath/matrix-gf2_test.go index 45d023ed..ada5ea09 100644 --- a/adnl/rldp/raptorq/discmath/matrix-gf2_test.go +++ b/adnl/rldp/raptorq/discmath/matrix-gf2_test.go @@ -384,46 +384,441 @@ func TestPlainMatrixGF2_RowAdd(t *testing.T) { } } -func TestPlainMatrixGF2_Mul(t *testing.T) { +func TestPlainMatrixGF2_ToGF256(t *testing.T) { tests := []struct { name string m *PlainMatrixGF2 - s *MatrixGF256 - want *PlainMatrixGF2 + want *MatrixGF256 }{ - // TODO: Add test cases. + { + name: "5x3", + m: &PlainMatrixGF2{ + rows: 5, + cols: 3, + data: []byte{234, 49}, // 0: 11][101][010] | 1: 0[011][000][1 + }, + want: &MatrixGF256{ + rows: []GF256{ + {data: []uint8{0, 1, 0}}, + {data: []uint8{1, 0, 1}}, + {data: []uint8{1, 1, 1}}, + {data: []uint8{0, 0, 0}}, + {data: []uint8{1, 1, 0}}, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := tt.m.Mul(tt.s) + got := tt.m.ToGF256() if !reflect.DeepEqual(got, tt.want) { - t.Errorf("Mul() = %v, want %v", got, tt.want) + t.Errorf("ToGF256() = %v, want %v", got, tt.want) } }) } } -func TestPlainMatrixGF2_ToGF256(t *testing.T) { +func TestPlainMatrixGF2_String(t *testing.T) { tests := []struct { name string m *PlainMatrixGF2 - want *MatrixGF256 + want string }{ { - name: "5x3", + name: "2x3", m: &PlainMatrixGF2{ - rows: 5, + rows: 2, cols: 3, - data: []byte{234, 49}, // 0: 11][101][010] | 1: 0[011][000][1 + data: []byte{42}, // 00 [101][010] + }, + want: "00 01 00\n" + "01 00 01", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.m.String() + if got != tt.want { + t.Errorf("String() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNewPlainOffsetMatrixGF2(t *testing.T) { + type args struct { + rows uint32 + cols uint32 + } + + tests := []struct { + name string + args args + want *PlainOffsetMatrixGF2 + }{ + { + name: "1x1", + args: args{ + rows: 1, + cols: 1, + }, + want: &PlainOffsetMatrixGF2{ + rows: 1, + cols: 1, + data: [][]byte{{0}}, + }, + }, + { + name: "column size lower one element", + args: args{ + rows: 3, + cols: 5, + }, + want: &PlainOffsetMatrixGF2{ + rows: 3, + cols: 5, + data: [][]byte{{0}, {0}, {0}}, + }, + }, + { + name: "column size greater one element", + args: args{ + rows: 3, + cols: 10, + }, + want: &PlainOffsetMatrixGF2{ + rows: 3, + cols: 10, + data: [][]byte{{0, 0}, {0, 0}, {0, 0}}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NewPlainOffsetMatrixGF2(tt.args.rows, tt.args.cols) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewPlainOffsetMatrixGF2() = %v, want %v", got, tt.want) + } + + gotRows := got.RowsNum() + if gotRows != tt.want.rows { + t.Errorf("RowsNum() = %v, want %v", gotRows, tt.want.rows) + } + + gotCols := got.ColsNum() + if gotCols != tt.want.cols { + t.Errorf("ColsNum() = %v, want %v", gotCols, tt.want.cols) + } + }) + } +} + +func TestPlainOffsetMatrixGF2_Get(t *testing.T) { + type args struct { + row uint32 + col uint32 + } + + tests := []struct { + name string + m *PlainOffsetMatrixGF2 + args args + want byte + }{ + { + name: "get 0", + m: &PlainOffsetMatrixGF2{ + rows: 2, + cols: 5, + data: [][]byte{{19}, {0}}, // 000[10011] + }, + args: args{ + row: 0, + col: 2, + }, + want: 0, + }, + { + name: "get 1", + m: &PlainOffsetMatrixGF2{ + rows: 2, + cols: 5, + data: [][]byte{{19}, {0}}, // 000[10011] + }, + args: args{ + row: 0, + col: 4, + }, + want: 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.m.Get(tt.args.row, tt.args.col) + if got != tt.want { + t.Errorf("Get() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPlainOffsetMatrixGF2_Set(t *testing.T) { + type args struct { + row uint32 + col uint32 + } + + tests := []struct { + name string + m *PlainOffsetMatrixGF2 + args args + want [][]byte + }{ + { + name: "3x3 set [0,1]", + m: &PlainOffsetMatrixGF2{ + rows: 3, + cols: 3, + data: [][]byte{{0}, {0}, {0}}, + }, + args: args{ + row: 0, + col: 1, + }, + want: [][]byte{{2}, {0}, {0}}, + }, + { + name: "3x3 set [2,2]", + m: &PlainOffsetMatrixGF2{ + rows: 3, + cols: 3, + data: [][]byte{{0}, {0}, {0}}, + }, + args: args{ + row: 2, + col: 2, + }, + want: [][]byte{{0}, {0}, {4}}, + }, + { + name: "2x10, set [1,9]", + m: &PlainOffsetMatrixGF2{ + rows: 2, + cols: 10, + data: [][]byte{{0, 0}, {0, 0}}, + }, + args: args{ + row: 1, + col: 9, + }, + want: [][]byte{{0, 0}, {0, 2}}, + }, + { + name: "set bit which is already set", + m: &PlainOffsetMatrixGF2{ + rows: 3, + cols: 3, + data: [][]byte{{0}, {2}, {0}}, + }, + args: args{ + row: 1, + col: 1, + }, + want: [][]byte{{0}, {2}, {0}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.m.Set(tt.args.row, tt.args.col) + if !reflect.DeepEqual(tt.m.data, tt.want) { + t.Errorf("Set() = %v, want %v", tt.m.data, tt.want) + } + }) + } +} + +func TestPlainOffsetMatrixGF2_Unset(t *testing.T) { + type args struct { + row uint32 + col uint32 + } + + tests := []struct { + name string + m *PlainOffsetMatrixGF2 + args args + want [][]byte + }{ + { + name: "3x3 unset [0,1]", + m: &PlainOffsetMatrixGF2{ + rows: 3, + cols: 3, + data: [][]byte{{2}, {0}, {0}}, + }, + args: args{ + row: 0, + col: 1, + }, + want: [][]byte{{0}, {0}, {0}}, + }, + { + name: "3x3 unset [1,2]", + m: &PlainOffsetMatrixGF2{ + rows: 3, + cols: 3, + data: [][]byte{{0}, {4}, {0}}, + }, + args: args{ + row: 1, + col: 2, + }, + want: [][]byte{{0}, {0}, {0}}, + }, + { + name: "unset bit which is already unset", + m: &PlainOffsetMatrixGF2{ + rows: 3, + cols: 3, + data: [][]byte{{0}, {0}, {0}}, + }, + args: args{ + row: 1, + col: 1, + }, + want: [][]byte{{0}, {0}, {0}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.m.Unset(tt.args.row, tt.args.col) + if !reflect.DeepEqual(tt.m.data, tt.want) { + t.Errorf("Unset() = %v, want %v", tt.m.data, tt.want) + } + }) + } +} + +func TestPlainOffsetMatrixGF2_GetRow(t *testing.T) { + tests := []struct { + name string + m *PlainOffsetMatrixGF2 + row uint32 + want []byte + }{ + { + name: "all row in one element", + m: &PlainOffsetMatrixGF2{ + rows: 3, + cols: 2, + data: [][]byte{{0}, {8}, {0}}, + }, + row: 1, + want: []byte{8}, + }, + { + name: "row in few elements", + m: &PlainOffsetMatrixGF2{ + rows: 2, + cols: 10, + data: [][]byte{{0, 0}, {17, 3}}, // 1,0: 00010001] 1,1: 000000[11 + }, + row: 1, + want: []byte{17, 3}, + }, + { + name: "last row", + m: &PlainOffsetMatrixGF2{ + rows: 3, + cols: 8, + data: [][]byte{{0}, {0}, {0}}, + }, + row: 2, + want: []byte{0}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.m.GetRow(tt.row) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetRow() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPlainOffsetMatrixGF2_RowAdd(t *testing.T) { + type args struct { + row uint32 + what []byte + } + + tests := []struct { + name string + m *PlainOffsetMatrixGF2 + args args + want [][]byte + }{ + { + name: "row in one element", + m: &PlainOffsetMatrixGF2{ + rows: 3, + cols: 3, + data: [][]byte{{0}, {3}, {0}}, + }, + args: args{ + row: 1, + what: []byte{6}, + }, + want: [][]byte{{0}, {5}, {0}}, + }, + { + name: "row in few elements", + m: &PlainOffsetMatrixGF2{ + rows: 2, + cols: 10, + data: [][]byte{{0, 0}, {129, 2}}, // 1,0: 10000001] | 1,1: 000000[10 + }, + args: args{ + row: 1, + what: []byte{1, 3}, + }, + want: [][]byte{{0, 0}, {128, 1}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.m.RowAdd(tt.args.row, tt.args.what) + if !reflect.DeepEqual(tt.m.data, tt.want) { + t.Errorf("RowAdd() = %v, want %v", tt.m.data, tt.want) + } + }) + } +} + +func TestPlainOffsetMatrixGF2_ToGF256(t *testing.T) { + tests := []struct { + name string + m *PlainOffsetMatrixGF2 + want *MatrixGF256 + }{ + { + name: "2x3", + m: &PlainOffsetMatrixGF2{ + rows: 2, + cols: 3, + data: [][]byte{{2}, {5}}, }, want: &MatrixGF256{ rows: []GF256{ {data: []uint8{0, 1, 0}}, {data: []uint8{1, 0, 1}}, - {data: []uint8{1, 1, 1}}, - {data: []uint8{0, 0, 0}}, - {data: []uint8{1, 1, 0}}, }, }, }, @@ -439,18 +834,18 @@ func TestPlainMatrixGF2_ToGF256(t *testing.T) { } } -func TestPlainMatrixGF2_String(t *testing.T) { +func TestPlainOffsetMatrixGF2_String(t *testing.T) { tests := []struct { name string - m *PlainMatrixGF2 + m *PlainOffsetMatrixGF2 want string }{ { name: "2x3", - m: &PlainMatrixGF2{ + m: &PlainOffsetMatrixGF2{ rows: 2, cols: 3, - data: []byte{42}, // 00 [101][010] + data: [][]byte{{2}, {5}}, }, want: "00 01 00\n" + "01 00 01", }, @@ -466,6 +861,8 @@ func TestPlainMatrixGF2_String(t *testing.T) { } } +// --- Benchmarks --- + // Size: 100x100 // Data size: 10 000 bytes // BenchmarkMatrixGF2-12: GetRow+RowAdd 114345 10073 ns/op 0 B/op 0 allocs/op @@ -511,3 +908,26 @@ func BenchmarkPlainMatrixGF2(b *testing.B) { } } } + +// Size: 100x100 +// Data size: 1300 bytes +// BenchmarkPlainOffsetMatrixGF2-12: GetRow+RowAdd 857792 1253 ns/op 0 B/op 0 allocs/op +// BenchmarkPlainOffsetMatrixGF2-12: GetRow 23977622 50.21 ns/op 0 B/op 0 allocs/op +func BenchmarkPlainOffsetMatrixGF2(b *testing.B) { + const dimension = 100 + + m := NewPlainOffsetMatrixGF2(dimension, dimension) + for i := uint32(0); i < dimension; i++ { + m.Set(i, i) + } + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for j := uint32(0); j < m.RowsNum()-1; j++ { + what := m.GetRow(j + 1) + m.RowAdd(j, what) + } + } +} From b28ac00e8c78df5d941606dd6aaa898f0e36dae1 Mon Sep 17 00:00:00 2001 From: Tikhon Slasten Date: Tue, 31 Oct 2023 13:07:50 +0800 Subject: [PATCH 10/23] Use PlainOffsetMatrixGF2 in computations --- adnl/rldp/raptorq/discmath/matrix-gf256.go | 4 ++-- adnl/rldp/raptorq/discmath/sparse-matrix.go | 4 ++-- adnl/rldp/raptorq/solver.go | 3 +-- adnl/rldp/raptorq/solver_test.go | 5 +++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/adnl/rldp/raptorq/discmath/matrix-gf256.go b/adnl/rldp/raptorq/discmath/matrix-gf256.go index 44f41076..198e552f 100644 --- a/adnl/rldp/raptorq/discmath/matrix-gf256.go +++ b/adnl/rldp/raptorq/discmath/matrix-gf256.go @@ -138,8 +138,8 @@ func (m *MatrixGF256) Each(f func(row, col uint32)) { } } -func (m *MatrixGF256) ToGF2(rowFrom, colFrom, rowSize, colSize uint32) *PlainMatrixGF2 { - mGF2 := NewPlainMatrixGF2(rowSize, colSize) +func (m *MatrixGF256) ToGF2(rowFrom, colFrom, rowSize, colSize uint32) *PlainOffsetMatrixGF2 { + mGF2 := NewPlainOffsetMatrixGF2(rowSize, colSize) m.Each(func(row, col uint32) { if (row >= rowFrom && row < rowFrom+rowSize) && (col >= colFrom && col < colFrom+colSize) { diff --git a/adnl/rldp/raptorq/discmath/sparse-matrix.go b/adnl/rldp/raptorq/discmath/sparse-matrix.go index 707e0c78..c08cd700 100644 --- a/adnl/rldp/raptorq/discmath/sparse-matrix.go +++ b/adnl/rldp/raptorq/discmath/sparse-matrix.go @@ -101,8 +101,8 @@ func (s *SparseMatrixGF2) ApplyColsPermutation(permutation []uint32) *SparseMatr return res } -func (s *SparseMatrixGF2) ToDense(rowFrom, colFrom, rowSize, colSize uint32) *PlainMatrixGF2 { - m := NewPlainMatrixGF2(rowSize, colSize) +func (s *SparseMatrixGF2) ToDense(rowFrom, colFrom, rowSize, colSize uint32) *PlainOffsetMatrixGF2 { + m := NewPlainOffsetMatrixGF2(rowSize, colSize) s.Each(func(row, col uint32) { if (row >= rowFrom && row < rowFrom+rowSize) && (col >= colFrom && col < colFrom+colSize) { diff --git a/adnl/rldp/raptorq/solver.go b/adnl/rldp/raptorq/solver.go index ca08bb48..36da6368 100644 --- a/adnl/rldp/raptorq/solver.go +++ b/adnl/rldp/raptorq/solver.go @@ -105,8 +105,7 @@ func (p *raptorParams) Solve(symbols []*Symbol) (*discmath.MatrixGF256, error) { break } - what, fBitmask, lBitmask := e.GetRow(i) - e.RowAdd(row, what, fBitmask, lBitmask) + e.RowAdd(row, e.GetRow(i)) d.RowAdd(row, d.GetRow(i)) } } diff --git a/adnl/rldp/raptorq/solver_test.go b/adnl/rldp/raptorq/solver_test.go index 6453bef7..c0d2849b 100644 --- a/adnl/rldp/raptorq/solver_test.go +++ b/adnl/rldp/raptorq/solver_test.go @@ -114,8 +114,9 @@ func Test_EncodeDecodeFuzz(t *testing.T) { t.Log(discmath.GetMetrics()) } -// Benchmark_EncodeDecodeFuzz-12: MatrixGF2 907 1329193 ns/op 653478 B/op 1806 allocs/op -// Benchmark_EncodeDecodeFuzz-12: PlainMatrixGF2 1003 1429374 ns/op 652845 B/op 1809 allocs/op +// Benchmark_EncodeDecodeFuzz-12: MatrixGF2 907 1329193 ns/op 653478 B/op 1806 allocs/op +// Benchmark_EncodeDecodeFuzz-12: PlainMatrixGF2 1003 1429374 ns/op 652845 B/op 1809 allocs/op +// Benchmark_EncodeDecodeFuzz-12: PlainOffsetMatrixGF2 1093 1096029 ns/op 653680 B/op 1844 allocs/op func Benchmark_EncodeDecodeFuzz(b *testing.B) { str := make([]byte, 4096) rand.Read(str) From 20baa1482fcf023d75faeb401cb7bdce581f8674 Mon Sep 17 00:00:00 2001 From: Tikhon Slasten Date: Wed, 1 Nov 2023 08:26:10 +0800 Subject: [PATCH 11/23] PlainOffsetMatrixGF2 store data in one dimension slice --- adnl/rldp/raptorq/discmath/matrix-gf2.go | 41 ++--- adnl/rldp/raptorq/discmath/matrix-gf2_test.go | 161 ++++++++++-------- adnl/rldp/raptorq/solver_test.go | 2 +- 3 files changed, 113 insertions(+), 91 deletions(-) diff --git a/adnl/rldp/raptorq/discmath/matrix-gf2.go b/adnl/rldp/raptorq/discmath/matrix-gf2.go index 98ec0e69..bc25c84e 100644 --- a/adnl/rldp/raptorq/discmath/matrix-gf2.go +++ b/adnl/rldp/raptorq/discmath/matrix-gf2.go @@ -249,7 +249,8 @@ func (m *PlainMatrixGF2) getIdx(row, col uint32) (uint32, byte) { type PlainOffsetMatrixGF2 struct { rows, cols uint32 - data [][]byte + rowSize uint32 + data []byte } func NewPlainOffsetMatrixGF2(rows, cols uint32) *PlainOffsetMatrixGF2 { @@ -260,15 +261,13 @@ func NewPlainOffsetMatrixGF2(rows, cols uint32) *PlainOffsetMatrixGF2 { rowSize++ } - data := make([][]byte, rows) - for i := range data { - data[i] = make([]byte, rowSize) - } + data := make([]byte, rows*rowSize) return &PlainOffsetMatrixGF2{ - rows: rows, - cols: cols, - data: data, + rows: rows, + cols: cols, + rowSize: rowSize, + data: data, } } @@ -285,22 +284,26 @@ func (m *PlainOffsetMatrixGF2) Get(row, col uint32) byte { } func (m *PlainOffsetMatrixGF2) Set(row, col uint32) { - elIdx, colIdx := m.getColPosition(col) - m.data[row][elIdx] |= 1 << colIdx + elIdx, colIdx := m.getElementPosition(row, col) + m.data[elIdx] |= 1 << colIdx } func (m *PlainOffsetMatrixGF2) Unset(row, col uint32) { - elIdx, colIdx := m.getColPosition(col) - m.data[row][elIdx] &= ^(1 << colIdx) + elIdx, colIdx := m.getElementPosition(row, col) + m.data[elIdx] &= ^(1 << colIdx) } func (m *PlainOffsetMatrixGF2) GetRow(row uint32) []byte { - return m.data[row] + firstElIdx, _ := m.getElementPosition(row, 0) + lastElIdx := firstElIdx + (m.cols-1)/elSize + 1 + + return m.data[firstElIdx:lastElIdx] } func (m *PlainOffsetMatrixGF2) RowAdd(row uint32, what []byte) { + firstElIdx, _ := m.getElementPosition(row, 0) for i, whatByte := range what { - m.data[row][i] ^= whatByte + m.data[firstElIdx+uint32(i)] ^= whatByte } } @@ -350,12 +353,12 @@ func (m *PlainOffsetMatrixGF2) getRowUInt8(row uint32) []uint8 { // getElement returns element in matrix by row and col. Possible values: 0 or 1 func (m *PlainOffsetMatrixGF2) getElement(row, col uint32) byte { - elIdx, colIdx := m.getColPosition(col) + elIdx, colIdx := m.getElementPosition(row, col) - return (m.data[row][elIdx] & (1 << colIdx)) >> colIdx + return (m.data[elIdx] & (1 << colIdx)) >> colIdx } -// getColPosition returns index of element in array and offset in this element -func (m *PlainOffsetMatrixGF2) getColPosition(col uint32) (uint32, byte) { - return col / elSize, byte(col % elSize) +// getElementPosition returns index of element in array and offset in this element +func (m *PlainOffsetMatrixGF2) getElementPosition(row, col uint32) (uint32, byte) { + return (row * m.rowSize) + col/elSize, byte(col % elSize) } diff --git a/adnl/rldp/raptorq/discmath/matrix-gf2_test.go b/adnl/rldp/raptorq/discmath/matrix-gf2_test.go index ada5ea09..43199f86 100644 --- a/adnl/rldp/raptorq/discmath/matrix-gf2_test.go +++ b/adnl/rldp/raptorq/discmath/matrix-gf2_test.go @@ -464,9 +464,10 @@ func TestNewPlainOffsetMatrixGF2(t *testing.T) { cols: 1, }, want: &PlainOffsetMatrixGF2{ - rows: 1, - cols: 1, - data: [][]byte{{0}}, + rows: 1, + cols: 1, + rowSize: 1, + data: []byte{0}, }, }, { @@ -476,9 +477,10 @@ func TestNewPlainOffsetMatrixGF2(t *testing.T) { cols: 5, }, want: &PlainOffsetMatrixGF2{ - rows: 3, - cols: 5, - data: [][]byte{{0}, {0}, {0}}, + rows: 3, + cols: 5, + rowSize: 1, + data: []byte{0, 0, 0}, }, }, { @@ -488,9 +490,10 @@ func TestNewPlainOffsetMatrixGF2(t *testing.T) { cols: 10, }, want: &PlainOffsetMatrixGF2{ - rows: 3, - cols: 10, - data: [][]byte{{0, 0}, {0, 0}, {0, 0}}, + rows: 3, + cols: 10, + rowSize: 2, + data: []byte{0, 0, 0, 0, 0, 0}, }, }, } @@ -530,9 +533,10 @@ func TestPlainOffsetMatrixGF2_Get(t *testing.T) { { name: "get 0", m: &PlainOffsetMatrixGF2{ - rows: 2, - cols: 5, - data: [][]byte{{19}, {0}}, // 000[10011] + rows: 2, + cols: 5, + rowSize: 1, + data: []byte{19, 0}, // 000[10011] }, args: args{ row: 0, @@ -543,9 +547,10 @@ func TestPlainOffsetMatrixGF2_Get(t *testing.T) { { name: "get 1", m: &PlainOffsetMatrixGF2{ - rows: 2, - cols: 5, - data: [][]byte{{19}, {0}}, // 000[10011] + rows: 2, + cols: 5, + rowSize: 1, + data: []byte{19, 0}, // 000[10011] }, args: args{ row: 0, @@ -574,59 +579,63 @@ func TestPlainOffsetMatrixGF2_Set(t *testing.T) { name string m *PlainOffsetMatrixGF2 args args - want [][]byte + want []byte }{ { name: "3x3 set [0,1]", m: &PlainOffsetMatrixGF2{ - rows: 3, - cols: 3, - data: [][]byte{{0}, {0}, {0}}, + rows: 3, + cols: 3, + rowSize: 1, + data: []byte{0, 0, 0}, }, args: args{ row: 0, col: 1, }, - want: [][]byte{{2}, {0}, {0}}, + want: []byte{2, 0, 0}, }, { name: "3x3 set [2,2]", m: &PlainOffsetMatrixGF2{ - rows: 3, - cols: 3, - data: [][]byte{{0}, {0}, {0}}, + rows: 3, + cols: 3, + rowSize: 1, + data: []byte{0, 0, 0}, }, args: args{ row: 2, col: 2, }, - want: [][]byte{{0}, {0}, {4}}, + want: []byte{0, 0, 4}, }, { name: "2x10, set [1,9]", m: &PlainOffsetMatrixGF2{ - rows: 2, - cols: 10, - data: [][]byte{{0, 0}, {0, 0}}, + rows: 2, + cols: 10, + rowSize: 2, + data: []byte{0, 0, 0, 0}, }, args: args{ row: 1, col: 9, }, - want: [][]byte{{0, 0}, {0, 2}}, + want: []byte{0, 0, 0, 2}, }, { name: "set bit which is already set", m: &PlainOffsetMatrixGF2{ - rows: 3, - cols: 3, - data: [][]byte{{0}, {2}, {0}}, + rows: 3, + cols: 3, + rowSize: 1, + data: []byte{0, 2, 0}, }, args: args{ row: 1, col: 1, }, - want: [][]byte{{0}, {2}, {0}}, + want: []byte{0, 2, 0}, }, } @@ -650,46 +659,49 @@ func TestPlainOffsetMatrixGF2_Unset(t *testing.T) { name string m *PlainOffsetMatrixGF2 args args - want [][]byte + want []byte }{ { name: "3x3 unset [0,1]", m: &PlainOffsetMatrixGF2{ - rows: 3, - cols: 3, - data: [][]byte{{2}, {0}, {0}}, + rows: 3, + cols: 3, + rowSize: 1, + data: []byte{2, 0, 0}, }, args: args{ row: 0, col: 1, }, - want: [][]byte{{0}, {0}, {0}}, + want: []byte{0, 0, 0}, }, { name: "3x3 unset [1,2]", m: &PlainOffsetMatrixGF2{ - rows: 3, - cols: 3, - data: [][]byte{{0}, {4}, {0}}, + rows: 3, + cols: 3, + rowSize: 1, + data: []byte{0, 4, 0}, }, args: args{ row: 1, col: 2, }, - want: [][]byte{{0}, {0}, {0}}, + want: []byte{0, 0, 0}, }, { name: "unset bit which is already unset", m: &PlainOffsetMatrixGF2{ - rows: 3, - cols: 3, - data: [][]byte{{0}, {0}, {0}}, + rows: 3, + cols: 3, + rowSize: 1, + data: []byte{0, 0, 0}, }, args: args{ row: 1, col: 1, }, - want: [][]byte{{0}, {0}, {0}}, + want: []byte{0, 0, 0}, }, } @@ -713,9 +725,10 @@ func TestPlainOffsetMatrixGF2_GetRow(t *testing.T) { { name: "all row in one element", m: &PlainOffsetMatrixGF2{ - rows: 3, - cols: 2, - data: [][]byte{{0}, {8}, {0}}, + rows: 3, + cols: 2, + rowSize: 1, + data: []byte{0, 8, 0}, }, row: 1, want: []byte{8}, @@ -723,9 +736,10 @@ func TestPlainOffsetMatrixGF2_GetRow(t *testing.T) { { name: "row in few elements", m: &PlainOffsetMatrixGF2{ - rows: 2, - cols: 10, - data: [][]byte{{0, 0}, {17, 3}}, // 1,0: 00010001] 1,1: 000000[11 + rows: 2, + cols: 10, + rowSize: 2, + data: []byte{0, 0, 17, 3}, // 1,0: 00010001] 1,1: 000000[11 }, row: 1, want: []byte{17, 3}, @@ -733,9 +747,10 @@ func TestPlainOffsetMatrixGF2_GetRow(t *testing.T) { { name: "last row", m: &PlainOffsetMatrixGF2{ - rows: 3, - cols: 8, - data: [][]byte{{0}, {0}, {0}}, + rows: 3, + cols: 8, + rowSize: 1, + data: []byte{0, 0, 0}, }, row: 2, want: []byte{0}, @@ -762,33 +777,35 @@ func TestPlainOffsetMatrixGF2_RowAdd(t *testing.T) { name string m *PlainOffsetMatrixGF2 args args - want [][]byte + want []byte }{ { name: "row in one element", m: &PlainOffsetMatrixGF2{ - rows: 3, - cols: 3, - data: [][]byte{{0}, {3}, {0}}, + rows: 3, + cols: 3, + rowSize: 1, + data: []byte{0, 3, 0}, }, args: args{ row: 1, what: []byte{6}, }, - want: [][]byte{{0}, {5}, {0}}, + want: []byte{0, 5, 0}, }, { name: "row in few elements", m: &PlainOffsetMatrixGF2{ - rows: 2, - cols: 10, - data: [][]byte{{0, 0}, {129, 2}}, // 1,0: 10000001] | 1,1: 000000[10 + rows: 2, + cols: 10, + rowSize: 2, + data: []byte{0, 0, 129, 2}, // 1,0: 10000001] | 1,1: 000000[10 }, args: args{ row: 1, what: []byte{1, 3}, }, - want: [][]byte{{0, 0}, {128, 1}}, + want: []byte{0, 0, 128, 1}, }, } @@ -811,9 +828,10 @@ func TestPlainOffsetMatrixGF2_ToGF256(t *testing.T) { { name: "2x3", m: &PlainOffsetMatrixGF2{ - rows: 2, - cols: 3, - data: [][]byte{{2}, {5}}, + rows: 2, + cols: 3, + rowSize: 1, + data: []byte{2, 5}, }, want: &MatrixGF256{ rows: []GF256{ @@ -843,9 +861,10 @@ func TestPlainOffsetMatrixGF2_String(t *testing.T) { { name: "2x3", m: &PlainOffsetMatrixGF2{ - rows: 2, - cols: 3, - data: [][]byte{{2}, {5}}, + rows: 2, + cols: 3, + rowSize: 1, + data: []byte{2, 5}, }, want: "00 01 00\n" + "01 00 01", }, @@ -911,8 +930,8 @@ func BenchmarkPlainMatrixGF2(b *testing.B) { // Size: 100x100 // Data size: 1300 bytes -// BenchmarkPlainOffsetMatrixGF2-12: GetRow+RowAdd 857792 1253 ns/op 0 B/op 0 allocs/op -// BenchmarkPlainOffsetMatrixGF2-12: GetRow 23977622 50.21 ns/op 0 B/op 0 allocs/op +// BenchmarkPlainOffsetMatrixGF2-12: GetRow+RowAdd 1000000 1096 ns/op 0 B/op 0 allocs/op +// BenchmarkPlainOffsetMatrixGF2-12: GetRow 10408053 109.8 ns/op 0 B/op 0 allocs/op func BenchmarkPlainOffsetMatrixGF2(b *testing.B) { const dimension = 100 diff --git a/adnl/rldp/raptorq/solver_test.go b/adnl/rldp/raptorq/solver_test.go index c0d2849b..d3ef9a70 100644 --- a/adnl/rldp/raptorq/solver_test.go +++ b/adnl/rldp/raptorq/solver_test.go @@ -116,7 +116,7 @@ func Test_EncodeDecodeFuzz(t *testing.T) { // Benchmark_EncodeDecodeFuzz-12: MatrixGF2 907 1329193 ns/op 653478 B/op 1806 allocs/op // Benchmark_EncodeDecodeFuzz-12: PlainMatrixGF2 1003 1429374 ns/op 652845 B/op 1809 allocs/op -// Benchmark_EncodeDecodeFuzz-12: PlainOffsetMatrixGF2 1093 1096029 ns/op 653680 B/op 1844 allocs/op +// Benchmark_EncodeDecodeFuzz-12: PlainOffsetMatrixGF2 1082 1114249 ns/op 652923 B/op 1810 allocs/op func Benchmark_EncodeDecodeFuzz(b *testing.B) { str := make([]byte, 4096) rand.Read(str) From 37320fbe720711221daa4654b36860f86d69b7ba Mon Sep 17 00:00:00 2001 From: Tikhon Slasten Date: Wed, 1 Nov 2023 14:28:47 +0800 Subject: [PATCH 12/23] Refactor --- adnl/rldp/raptorq/discmath/matrix-gf2.go | 202 +------ adnl/rldp/raptorq/discmath/matrix-gf256.go | 4 +- adnl/rldp/raptorq/discmath/matrix-gf2_test.go | 538 ++---------------- adnl/rldp/raptorq/discmath/metrics.go | 94 --- adnl/rldp/raptorq/discmath/sparse-matrix.go | 132 ----- adnl/rldp/raptorq/discmath/utils.go | 9 + adnl/rldp/raptorq/solver_test.go | 11 +- 7 files changed, 68 insertions(+), 922 deletions(-) delete mode 100644 adnl/rldp/raptorq/discmath/metrics.go delete mode 100644 adnl/rldp/raptorq/discmath/sparse-matrix.go create mode 100644 adnl/rldp/raptorq/discmath/utils.go diff --git a/adnl/rldp/raptorq/discmath/matrix-gf2.go b/adnl/rldp/raptorq/discmath/matrix-gf2.go index bc25c84e..71134bd8 100644 --- a/adnl/rldp/raptorq/discmath/matrix-gf2.go +++ b/adnl/rldp/raptorq/discmath/matrix-gf2.go @@ -2,7 +2,6 @@ package discmath import ( "fmt" - "math/bits" "strings" ) @@ -11,8 +10,6 @@ type MatrixGF2 struct { } func NewMatrixGF2(rows, cols uint32) *MatrixGF2 { - // defaultMetrics.store(rows, cols) // only for tests - data := make([][]uint8, rows) for i := range data { data[i] = make([]uint8, cols) @@ -85,177 +82,16 @@ func (m *MatrixGF2) String() string { return strings.Join(rows, "\n") } -// PlainMatrixGF2 is a matrix of GF2 elements -// Data is stored in a byte array, each byte is contained 8 elements (bits) -type PlainMatrixGF2 struct { - rows, cols uint32 - data []byte -} - -const ( - // elSize is a size of array's element in bits - elSize = 8 - maxEl = 1< 0 { - dataSize++ - } - - return &PlainMatrixGF2{ - rows: rows, - cols: cols, - data: make([]byte, dataSize), - } -} - -func (m *PlainMatrixGF2) RowsNum() uint32 { - return m.rows -} - -func (m *PlainMatrixGF2) ColsNum() uint32 { - return m.cols -} - -func (m *PlainMatrixGF2) Get(row, col uint32) byte { - idx, bit := m.getIdx(row, col) - - res := m.data[idx] & bit - if res > 0 { - return 1 - } - - return 0 -} - -func (m *PlainMatrixGF2) Set(row, col uint32) { - idx, bit := m.getIdx(row, col) - m.data[idx] |= bit -} - -func (m *PlainMatrixGF2) Unset(row, col uint32) { - idx, bit := m.getIdx(row, col) - m.data[idx] &= ^bit -} - -// GetRow returns row data, bit mask for the first byte and bit mask for the last byte -// Bitmask contains 1 for each bit that is part of the row -func (m *PlainMatrixGF2) GetRow(row uint32) ([]byte, byte, byte) { - firstBitIdx := row * m.cols - firstElIdx := firstBitIdx / elSize - - lastBitIdx := (row+1)*m.cols - 1 - lastElIdx := lastBitIdx / elSize - - firstBitmask := byte(maxEl - (1 << (firstBitIdx % elSize)) + 1) - lastBitmask := byte((1 << (lastBitIdx%elSize + 1)) - 1) - - if firstElIdx == lastElIdx { - firstBitmask &= lastBitmask - lastBitmask = firstBitmask - } - - if int(lastElIdx) == len(m.data) { - return m.data[firstElIdx:], firstBitmask, lastBitmask - } - - return m.data[firstElIdx : lastElIdx+1], firstBitmask, lastBitmask -} - -func (m *PlainMatrixGF2) RowAdd(row uint32, what []byte, firstBitmask, lastBitmask byte) { - col := uint32(0) - for i, whatByte := range what { - // compute bitmask of "what" row for current byte - bitmask := byte(maxEl) - if i == 0 { - bitmask = firstBitmask - } else if i == len(what)-1 { - bitmask = lastBitmask - } - - for j := bits.TrailingZeros8(bitmask); j < elSize-bits.LeadingZeros8(bitmask); j++ { - whatBit := (whatByte & (1 << j)) >> j - colBit := m.Get(row, col) - - if whatBit != colBit { - m.Set(row, col) - } else { - m.Unset(row, col) - } - - col++ - } - } -} +// elSize is a size of array's element in bits +const elSize = 8 -func (m *PlainMatrixGF2) Mul(s *MatrixGF256) *PlainMatrixGF2 { - mg := NewPlainMatrixGF2(s.RowsNum(), m.ColsNum()) - - s.Each(func(row, col uint32) { - rowData, firstBitmask, lastBitmask := m.GetRow(col) - mg.RowAdd(row, rowData, firstBitmask, lastBitmask) - }) - - return mg -} - -func (m *PlainMatrixGF2) ToGF256() *MatrixGF256 { - mg := NewMatrixGF256(m.RowsNum(), m.ColsNum()) - - for i := uint32(0); i < m.rows; i++ { - mg.rows[i] = GF256{data: m.getRowUInt8(i)} - } - - return mg -} - -func (m *PlainMatrixGF2) String() string { - var rows []string - for i := uint32(0); i < m.rows; i++ { - var cols []string - for j := uint32(0); j < m.cols; j++ { - cols = append(cols, fmt.Sprintf("%02x", m.Get(i, j))) - } - - rows = append(rows, strings.Join(cols, " ")) - } - - return strings.Join(rows, "\n") -} - -// getRowUInt8 returns row data as array of uint8 -func (m *PlainMatrixGF2) getRowUInt8(row uint32) []uint8 { - rowUInt8 := make([]uint8, m.cols) - for i := uint32(0); i < m.cols; i++ { - rowUInt8[i] = m.Get(row, i) - } - - return rowUInt8 -} - -// getIdx returns index in array of bytes and offset in this byte -func (m *PlainMatrixGF2) getIdx(row, col uint32) (uint32, byte) { - idx := row*m.cols + col - bitIdx := byte(idx % elSize) - - return idx / elSize, byte(1 << bitIdx) -} - -type PlainOffsetMatrixGF2 struct { +type PlainMatrixGF2 struct { rows, cols uint32 rowSize uint32 data []byte } -func NewPlainOffsetMatrixGF2(rows, cols uint32) *PlainOffsetMatrixGF2 { - // defaultMetrics.store(rows, cols) // only for tests - +func NewPlainMatrixGF2(rows, cols uint32) *PlainMatrixGF2 { rowSize := cols / elSize if cols%elSize > 0 { rowSize++ @@ -263,7 +99,7 @@ func NewPlainOffsetMatrixGF2(rows, cols uint32) *PlainOffsetMatrixGF2 { data := make([]byte, rows*rowSize) - return &PlainOffsetMatrixGF2{ + return &PlainMatrixGF2{ rows: rows, cols: cols, rowSize: rowSize, @@ -271,44 +107,44 @@ func NewPlainOffsetMatrixGF2(rows, cols uint32) *PlainOffsetMatrixGF2 { } } -func (m *PlainOffsetMatrixGF2) RowsNum() uint32 { +func (m *PlainMatrixGF2) RowsNum() uint32 { return m.rows } -func (m *PlainOffsetMatrixGF2) ColsNum() uint32 { +func (m *PlainMatrixGF2) ColsNum() uint32 { return m.cols } -func (m *PlainOffsetMatrixGF2) Get(row, col uint32) byte { +func (m *PlainMatrixGF2) Get(row, col uint32) byte { return m.getElement(row, col) } -func (m *PlainOffsetMatrixGF2) Set(row, col uint32) { +func (m *PlainMatrixGF2) Set(row, col uint32) { elIdx, colIdx := m.getElementPosition(row, col) m.data[elIdx] |= 1 << colIdx } -func (m *PlainOffsetMatrixGF2) Unset(row, col uint32) { +func (m *PlainMatrixGF2) Unset(row, col uint32) { elIdx, colIdx := m.getElementPosition(row, col) m.data[elIdx] &= ^(1 << colIdx) } -func (m *PlainOffsetMatrixGF2) GetRow(row uint32) []byte { +func (m *PlainMatrixGF2) GetRow(row uint32) []byte { firstElIdx, _ := m.getElementPosition(row, 0) lastElIdx := firstElIdx + (m.cols-1)/elSize + 1 return m.data[firstElIdx:lastElIdx] } -func (m *PlainOffsetMatrixGF2) RowAdd(row uint32, what []byte) { +func (m *PlainMatrixGF2) RowAdd(row uint32, what []byte) { firstElIdx, _ := m.getElementPosition(row, 0) for i, whatByte := range what { m.data[firstElIdx+uint32(i)] ^= whatByte } } -func (m *PlainOffsetMatrixGF2) Mul(s *MatrixGF256) *PlainOffsetMatrixGF2 { - mg := NewPlainOffsetMatrixGF2(s.RowsNum(), m.ColsNum()) +func (m *PlainMatrixGF2) Mul(s *MatrixGF256) *PlainMatrixGF2 { + mg := NewPlainMatrixGF2(s.RowsNum(), m.ColsNum()) s.Each(func(row, col uint32) { mRow := m.GetRow(col) @@ -318,7 +154,7 @@ func (m *PlainOffsetMatrixGF2) Mul(s *MatrixGF256) *PlainOffsetMatrixGF2 { return mg } -func (m *PlainOffsetMatrixGF2) ToGF256() *MatrixGF256 { +func (m *PlainMatrixGF2) ToGF256() *MatrixGF256 { mg := NewMatrixGF256(m.RowsNum(), m.ColsNum()) for i := uint32(0); i < m.rows; i++ { @@ -328,7 +164,7 @@ func (m *PlainOffsetMatrixGF2) ToGF256() *MatrixGF256 { return mg } -func (m *PlainOffsetMatrixGF2) String() string { +func (m *PlainMatrixGF2) String() string { var rows []string for row := uint32(0); row < m.rows; row++ { var cols []string @@ -342,7 +178,7 @@ func (m *PlainOffsetMatrixGF2) String() string { return strings.Join(rows, "\n") } -func (m *PlainOffsetMatrixGF2) getRowUInt8(row uint32) []uint8 { +func (m *PlainMatrixGF2) getRowUInt8(row uint32) []uint8 { result := make([]uint8, m.cols) for col := uint32(0); col < m.cols; col++ { result[col] = m.getElement(row, col) @@ -352,13 +188,13 @@ func (m *PlainOffsetMatrixGF2) getRowUInt8(row uint32) []uint8 { } // getElement returns element in matrix by row and col. Possible values: 0 or 1 -func (m *PlainOffsetMatrixGF2) getElement(row, col uint32) byte { +func (m *PlainMatrixGF2) getElement(row, col uint32) byte { elIdx, colIdx := m.getElementPosition(row, col) return (m.data[elIdx] & (1 << colIdx)) >> colIdx } // getElementPosition returns index of element in array and offset in this element -func (m *PlainOffsetMatrixGF2) getElementPosition(row, col uint32) (uint32, byte) { +func (m *PlainMatrixGF2) getElementPosition(row, col uint32) (uint32, byte) { return (row * m.rowSize) + col/elSize, byte(col % elSize) } diff --git a/adnl/rldp/raptorq/discmath/matrix-gf256.go b/adnl/rldp/raptorq/discmath/matrix-gf256.go index 198e552f..44f41076 100644 --- a/adnl/rldp/raptorq/discmath/matrix-gf256.go +++ b/adnl/rldp/raptorq/discmath/matrix-gf256.go @@ -138,8 +138,8 @@ func (m *MatrixGF256) Each(f func(row, col uint32)) { } } -func (m *MatrixGF256) ToGF2(rowFrom, colFrom, rowSize, colSize uint32) *PlainOffsetMatrixGF2 { - mGF2 := NewPlainOffsetMatrixGF2(rowSize, colSize) +func (m *MatrixGF256) ToGF2(rowFrom, colFrom, rowSize, colSize uint32) *PlainMatrixGF2 { + mGF2 := NewPlainMatrixGF2(rowSize, colSize) m.Each(func(row, col uint32) { if (row >= rowFrom && row < rowFrom+rowSize) && (col >= colFrom && col < colFrom+colSize) { diff --git a/adnl/rldp/raptorq/discmath/matrix-gf2_test.go b/adnl/rldp/raptorq/discmath/matrix-gf2_test.go index 43199f86..305f6eba 100644 --- a/adnl/rldp/raptorq/discmath/matrix-gf2_test.go +++ b/adnl/rldp/raptorq/discmath/matrix-gf2_test.go @@ -23,447 +23,6 @@ func TestNewPlainMatrixGF2(t *testing.T) { cols: 1, }, want: &PlainMatrixGF2{ - rows: 1, - cols: 1, - data: []byte{0}, - }, - }, - { - name: "size lower one element", - args: args{ - rows: 2, - cols: 3, - }, - want: &PlainMatrixGF2{ - rows: 2, - cols: 3, - data: []byte{0}, - }, - }, - { - name: "size greater one element", - args: args{ - rows: 2, - cols: 7, - }, - want: &PlainMatrixGF2{ - rows: 2, - cols: 7, - data: []byte{0, 0}, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := NewPlainMatrixGF2(tt.args.rows, tt.args.cols) - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("NewPlainMatrixGF2() = %v, want %v", got, tt.want) - } - - gotRows := got.RowsNum() - if gotRows != tt.want.rows { - t.Errorf("RowsNum() = %v, want %v", gotRows, tt.want.rows) - } - - gotCols := got.ColsNum() - if gotCols != tt.want.cols { - t.Errorf("ColsNum() = %v, want %v", gotCols, tt.want.cols) - } - }) - } -} - -func TestPlainMatrixGF2_Get(t *testing.T) { - type args struct { - row uint32 - col uint32 - } - - tests := []struct { - name string - m *PlainMatrixGF2 - args args - want byte - }{ - { - name: "get 0", - m: &PlainMatrixGF2{ - rows: 2, - cols: 5, - data: []byte{131, 0}, // 1000 0011 - }, - args: args{ - row: 1, - col: 4, - }, - want: 0, - }, - { - name: "get 1", - m: &PlainMatrixGF2{ - rows: 2, - cols: 5, - data: []byte{0, 2}, // 0000 0010 - }, - args: args{ - row: 1, - col: 4, - }, - want: 1, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := tt.m.Get(tt.args.row, tt.args.col) - if got != tt.want { - t.Errorf("Get() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestPlainMatrixGF2_Set(t *testing.T) { - type args struct { - row uint32 - col uint32 - } - - tests := []struct { - name string - m *PlainMatrixGF2 - args args - want []byte - }{ - { - name: "3x3 set [0,1]", - m: &PlainMatrixGF2{ - rows: 3, - cols: 3, - data: []byte{0, 0}, - }, - args: args{ - row: 0, - col: 1, - }, - want: []byte{2, 0}, - }, - { - name: "3x3 set [2,2]", - m: &PlainMatrixGF2{ - rows: 3, - cols: 3, - data: []byte{0, 0}, - }, - args: args{ - row: 2, - col: 2, - }, - want: []byte{0, 1}, - }, - { - name: "2x10, set [1,9]", - m: &PlainMatrixGF2{ - rows: 2, - cols: 10, - data: []byte{0, 0, 0}, - }, - args: args{ - row: 1, - col: 9, - }, - want: []byte{0, 0, 8}, - }, - { - name: "set bit which is already set", - m: &PlainMatrixGF2{ - rows: 3, - cols: 3, - data: []byte{24, 0}, // 1: 1 1 0 - }, - args: args{ - row: 1, - col: 1, - }, - want: []byte{24, 0}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tt.m.Set(tt.args.row, tt.args.col) - if !reflect.DeepEqual(tt.m.data, tt.want) { - t.Errorf("Set() = %v, want %v", tt.m.data, tt.want) - } - }) - } -} - -func TestPlainMatrixGF2_Unset(t *testing.T) { - type args struct { - row uint32 - col uint32 - } - - tests := []struct { - name string - m *PlainMatrixGF2 - args args - want []byte - }{ - { - name: "3x3 unset [0,1]", - m: &PlainMatrixGF2{ - rows: 3, - cols: 3, - data: []byte{2, 0}, - }, - args: args{ - row: 0, - col: 1, - }, - want: []byte{0, 0}, - }, - { - name: "3x3 unset [1,2]", - m: &PlainMatrixGF2{ - rows: 3, - cols: 3, - data: []byte{32, 0}, - }, - args: args{ - row: 1, - col: 2, - }, - want: []byte{0, 0}, - }, - { - name: "unset bit which is already unset", - m: &PlainMatrixGF2{ - rows: 3, - cols: 3, - data: []byte{0, 0}, - }, - args: args{ - row: 1, - col: 1, - }, - want: []byte{0, 0}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tt.m.Unset(tt.args.row, tt.args.col) - if !reflect.DeepEqual(tt.m.data, tt.want) { - t.Errorf("Unset() = %v, want %v", tt.m.data, tt.want) - } - }) - } -} - -func TestPlainMatrixGF2_GetRow(t *testing.T) { - tests := []struct { - name string - m *PlainMatrixGF2 - row uint32 - want []byte - wantFirstBitmask byte - wantLastBitmask byte - }{ - { - name: "all row in one element", - m: &PlainMatrixGF2{ - rows: 3, - cols: 2, - data: []byte{8}, // 00 00[10]00 - }, - row: 1, - want: []byte{8}, - wantFirstBitmask: 12, // 00 00[11]00 - wantLastBitmask: 12, // 00 00[11]00 - }, - { - name: "row in few elements", - m: &PlainMatrixGF2{ - rows: 2, - cols: 10, - data: []byte{0, 248, 12}, // 0: 0000 0000 | 1: [1111 10]00 | 2: 0000 [1100] - }, - row: 1, - want: []byte{248, 12}, - wantFirstBitmask: 252, - wantLastBitmask: 15, - }, - { - name: "last row", - m: &PlainMatrixGF2{ - rows: 3, - cols: 8, - data: []byte{0, 0, 0}, - }, - row: 2, - want: []byte{0}, - wantFirstBitmask: 255, - wantLastBitmask: 255, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, gotFirstBitmask, gotLastBitmask := tt.m.GetRow(tt.row) - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("GetRow() got = %v, want %v", got, tt.want) - } - - if gotFirstBitmask != tt.wantFirstBitmask { - t.Errorf("GetRow() gotFirstBitmask = %v, want %v", gotFirstBitmask, tt.wantFirstBitmask) - } - - if gotLastBitmask != tt.wantLastBitmask { - t.Errorf("GetRow() gotLastBitmask = %v, want %v", gotLastBitmask, tt.wantLastBitmask) - } - }) - } -} - -func TestPlainMatrixGF2_RowAdd(t *testing.T) { - type args struct { - row uint32 - what []byte - firstBitmask byte - lastBitmask byte - } - - tests := []struct { - name string - m *PlainMatrixGF2 - args args - want []byte - }{ - { - name: "same rows", - m: &PlainMatrixGF2{ - rows: 3, - cols: 3, - data: []byte{146, 0}, // 0: 10[010]010 - }, - args: args{ - row: 1, - what: []byte{24}, // 00[011]000 - firstBitmask: 56, // 00[111]000 - lastBitmask: 56, // 00[111]000 - }, - want: []byte{138, 0}, // 10[001]010 - }, - { - name: "different rows: #1 + #2", - m: &PlainMatrixGF2{ - rows: 3, - cols: 3, - data: []byte{146, 0}, // 0: 10[010]010 | 1: 0000 0000 - }, - args: args{ - row: 1, - what: []byte{128, 1}, // 0: 10]00 0000 | 1: 0000 000[1 - firstBitmask: 192, - lastBitmask: 1, - }, - want: []byte{162, 0}, // 0: 10[100]010 | 1: 0000 0000 - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tt.m.RowAdd(tt.args.row, tt.args.what, tt.args.firstBitmask, tt.args.lastBitmask) - if !reflect.DeepEqual(tt.m.data, tt.want) { - t.Errorf("RowAdd() = %v, want %v", tt.m.data, tt.want) - } - }) - } -} - -func TestPlainMatrixGF2_ToGF256(t *testing.T) { - tests := []struct { - name string - m *PlainMatrixGF2 - want *MatrixGF256 - }{ - { - name: "5x3", - m: &PlainMatrixGF2{ - rows: 5, - cols: 3, - data: []byte{234, 49}, // 0: 11][101][010] | 1: 0[011][000][1 - }, - want: &MatrixGF256{ - rows: []GF256{ - {data: []uint8{0, 1, 0}}, - {data: []uint8{1, 0, 1}}, - {data: []uint8{1, 1, 1}}, - {data: []uint8{0, 0, 0}}, - {data: []uint8{1, 1, 0}}, - }, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := tt.m.ToGF256() - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("ToGF256() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestPlainMatrixGF2_String(t *testing.T) { - tests := []struct { - name string - m *PlainMatrixGF2 - want string - }{ - { - name: "2x3", - m: &PlainMatrixGF2{ - rows: 2, - cols: 3, - data: []byte{42}, // 00 [101][010] - }, - want: "00 01 00\n" + "01 00 01", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := tt.m.String() - if got != tt.want { - t.Errorf("String() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestNewPlainOffsetMatrixGF2(t *testing.T) { - type args struct { - rows uint32 - cols uint32 - } - - tests := []struct { - name string - args args - want *PlainOffsetMatrixGF2 - }{ - { - name: "1x1", - args: args{ - rows: 1, - cols: 1, - }, - want: &PlainOffsetMatrixGF2{ rows: 1, cols: 1, rowSize: 1, @@ -476,7 +35,7 @@ func TestNewPlainOffsetMatrixGF2(t *testing.T) { rows: 3, cols: 5, }, - want: &PlainOffsetMatrixGF2{ + want: &PlainMatrixGF2{ rows: 3, cols: 5, rowSize: 1, @@ -489,7 +48,7 @@ func TestNewPlainOffsetMatrixGF2(t *testing.T) { rows: 3, cols: 10, }, - want: &PlainOffsetMatrixGF2{ + want: &PlainMatrixGF2{ rows: 3, cols: 10, rowSize: 2, @@ -500,9 +59,9 @@ func TestNewPlainOffsetMatrixGF2(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := NewPlainOffsetMatrixGF2(tt.args.rows, tt.args.cols) + got := NewPlainMatrixGF2(tt.args.rows, tt.args.cols) if !reflect.DeepEqual(got, tt.want) { - t.Errorf("NewPlainOffsetMatrixGF2() = %v, want %v", got, tt.want) + t.Errorf("NewPlainMatrixGF2() = %v, want %v", got, tt.want) } gotRows := got.RowsNum() @@ -518,7 +77,7 @@ func TestNewPlainOffsetMatrixGF2(t *testing.T) { } } -func TestPlainOffsetMatrixGF2_Get(t *testing.T) { +func TestPlainMatrixGF2_Get(t *testing.T) { type args struct { row uint32 col uint32 @@ -526,13 +85,13 @@ func TestPlainOffsetMatrixGF2_Get(t *testing.T) { tests := []struct { name string - m *PlainOffsetMatrixGF2 + m *PlainMatrixGF2 args args want byte }{ { name: "get 0", - m: &PlainOffsetMatrixGF2{ + m: &PlainMatrixGF2{ rows: 2, cols: 5, rowSize: 1, @@ -546,7 +105,7 @@ func TestPlainOffsetMatrixGF2_Get(t *testing.T) { }, { name: "get 1", - m: &PlainOffsetMatrixGF2{ + m: &PlainMatrixGF2{ rows: 2, cols: 5, rowSize: 1, @@ -569,7 +128,7 @@ func TestPlainOffsetMatrixGF2_Get(t *testing.T) { } } -func TestPlainOffsetMatrixGF2_Set(t *testing.T) { +func TestPlainMatrixGF2_Set(t *testing.T) { type args struct { row uint32 col uint32 @@ -577,13 +136,13 @@ func TestPlainOffsetMatrixGF2_Set(t *testing.T) { tests := []struct { name string - m *PlainOffsetMatrixGF2 + m *PlainMatrixGF2 args args want []byte }{ { name: "3x3 set [0,1]", - m: &PlainOffsetMatrixGF2{ + m: &PlainMatrixGF2{ rows: 3, cols: 3, rowSize: 1, @@ -597,7 +156,7 @@ func TestPlainOffsetMatrixGF2_Set(t *testing.T) { }, { name: "3x3 set [2,2]", - m: &PlainOffsetMatrixGF2{ + m: &PlainMatrixGF2{ rows: 3, cols: 3, rowSize: 1, @@ -611,7 +170,7 @@ func TestPlainOffsetMatrixGF2_Set(t *testing.T) { }, { name: "2x10, set [1,9]", - m: &PlainOffsetMatrixGF2{ + m: &PlainMatrixGF2{ rows: 2, cols: 10, rowSize: 2, @@ -625,7 +184,7 @@ func TestPlainOffsetMatrixGF2_Set(t *testing.T) { }, { name: "set bit which is already set", - m: &PlainOffsetMatrixGF2{ + m: &PlainMatrixGF2{ rows: 3, cols: 3, rowSize: 1, @@ -649,7 +208,7 @@ func TestPlainOffsetMatrixGF2_Set(t *testing.T) { } } -func TestPlainOffsetMatrixGF2_Unset(t *testing.T) { +func TestPlainMatrixGF2_Unset(t *testing.T) { type args struct { row uint32 col uint32 @@ -657,13 +216,13 @@ func TestPlainOffsetMatrixGF2_Unset(t *testing.T) { tests := []struct { name string - m *PlainOffsetMatrixGF2 + m *PlainMatrixGF2 args args want []byte }{ { name: "3x3 unset [0,1]", - m: &PlainOffsetMatrixGF2{ + m: &PlainMatrixGF2{ rows: 3, cols: 3, rowSize: 1, @@ -677,7 +236,7 @@ func TestPlainOffsetMatrixGF2_Unset(t *testing.T) { }, { name: "3x3 unset [1,2]", - m: &PlainOffsetMatrixGF2{ + m: &PlainMatrixGF2{ rows: 3, cols: 3, rowSize: 1, @@ -691,7 +250,7 @@ func TestPlainOffsetMatrixGF2_Unset(t *testing.T) { }, { name: "unset bit which is already unset", - m: &PlainOffsetMatrixGF2{ + m: &PlainMatrixGF2{ rows: 3, cols: 3, rowSize: 1, @@ -715,16 +274,16 @@ func TestPlainOffsetMatrixGF2_Unset(t *testing.T) { } } -func TestPlainOffsetMatrixGF2_GetRow(t *testing.T) { +func TestPlainMatrixGF2_GetRow(t *testing.T) { tests := []struct { name string - m *PlainOffsetMatrixGF2 + m *PlainMatrixGF2 row uint32 want []byte }{ { name: "all row in one element", - m: &PlainOffsetMatrixGF2{ + m: &PlainMatrixGF2{ rows: 3, cols: 2, rowSize: 1, @@ -735,7 +294,7 @@ func TestPlainOffsetMatrixGF2_GetRow(t *testing.T) { }, { name: "row in few elements", - m: &PlainOffsetMatrixGF2{ + m: &PlainMatrixGF2{ rows: 2, cols: 10, rowSize: 2, @@ -746,7 +305,7 @@ func TestPlainOffsetMatrixGF2_GetRow(t *testing.T) { }, { name: "last row", - m: &PlainOffsetMatrixGF2{ + m: &PlainMatrixGF2{ rows: 3, cols: 8, rowSize: 1, @@ -767,7 +326,7 @@ func TestPlainOffsetMatrixGF2_GetRow(t *testing.T) { } } -func TestPlainOffsetMatrixGF2_RowAdd(t *testing.T) { +func TestPlainMatrixGF2_RowAdd(t *testing.T) { type args struct { row uint32 what []byte @@ -775,13 +334,13 @@ func TestPlainOffsetMatrixGF2_RowAdd(t *testing.T) { tests := []struct { name string - m *PlainOffsetMatrixGF2 + m *PlainMatrixGF2 args args want []byte }{ { name: "row in one element", - m: &PlainOffsetMatrixGF2{ + m: &PlainMatrixGF2{ rows: 3, cols: 3, rowSize: 1, @@ -795,7 +354,7 @@ func TestPlainOffsetMatrixGF2_RowAdd(t *testing.T) { }, { name: "row in few elements", - m: &PlainOffsetMatrixGF2{ + m: &PlainMatrixGF2{ rows: 2, cols: 10, rowSize: 2, @@ -819,15 +378,15 @@ func TestPlainOffsetMatrixGF2_RowAdd(t *testing.T) { } } -func TestPlainOffsetMatrixGF2_ToGF256(t *testing.T) { +func TestPlainMatrixGF2_ToGF256(t *testing.T) { tests := []struct { name string - m *PlainOffsetMatrixGF2 + m *PlainMatrixGF2 want *MatrixGF256 }{ { name: "2x3", - m: &PlainOffsetMatrixGF2{ + m: &PlainMatrixGF2{ rows: 2, cols: 3, rowSize: 1, @@ -852,15 +411,15 @@ func TestPlainOffsetMatrixGF2_ToGF256(t *testing.T) { } } -func TestPlainOffsetMatrixGF2_String(t *testing.T) { +func TestPlainMatrixGF2_String(t *testing.T) { tests := []struct { name string - m *PlainOffsetMatrixGF2 + m *PlainMatrixGF2 want string }{ { name: "2x3", - m: &PlainOffsetMatrixGF2{ + m: &PlainMatrixGF2{ rows: 2, cols: 3, rowSize: 1, @@ -906,9 +465,9 @@ func BenchmarkMatrixGF2(b *testing.B) { } // Size: 100x100 -// Data size: 1250 bytes -// BenchmarkPlainMatrixGF2-12 GetRow+RowAdd 30560 39130 ns/op 0 B/op 0 allocs/op -// BenchmarkPlainMatrixGF2-12: GetRow 2639367 435.2 ns/op 0 B/op 0 allocs/op +// Data size: 1300 bytes +// BenchmarkPlainMatrixGF2-12: GetRow+RowAdd 1000000 1096 ns/op 0 B/op 0 allocs/op +// BenchmarkPlainMatrixGF2-12: GetRow 10408053 109.8 ns/op 0 B/op 0 allocs/op func BenchmarkPlainMatrixGF2(b *testing.B) { const dimension = 100 @@ -920,29 +479,6 @@ func BenchmarkPlainMatrixGF2(b *testing.B) { b.ReportAllocs() b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := uint32(0); j < m.RowsNum()-1; j++ { - what, fBitmask, lBitmask := m.GetRow(j + 1) - m.RowAdd(j, what, fBitmask, lBitmask) - } - } -} - -// Size: 100x100 -// Data size: 1300 bytes -// BenchmarkPlainOffsetMatrixGF2-12: GetRow+RowAdd 1000000 1096 ns/op 0 B/op 0 allocs/op -// BenchmarkPlainOffsetMatrixGF2-12: GetRow 10408053 109.8 ns/op 0 B/op 0 allocs/op -func BenchmarkPlainOffsetMatrixGF2(b *testing.B) { - const dimension = 100 - - m := NewPlainOffsetMatrixGF2(dimension, dimension) - for i := uint32(0); i < dimension; i++ { - m.Set(i, i) - } - - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { for j := uint32(0); j < m.RowsNum()-1; j++ { what := m.GetRow(j + 1) diff --git a/adnl/rldp/raptorq/discmath/metrics.go b/adnl/rldp/raptorq/discmath/metrics.go deleted file mode 100644 index 537a3f7c..00000000 --- a/adnl/rldp/raptorq/discmath/metrics.go +++ /dev/null @@ -1,94 +0,0 @@ -package discmath - -import "fmt" - -var defaultMetrics *metrics - -func init() { - defaultMetrics = newMetrics() -} - -type metrics struct { - isFirst bool - minRows uint32 - maxRows uint32 - avgRows uint32 - minCols uint32 - maxCols uint32 - avgCols uint32 - minSize uint32 - maxSize uint32 - avgSize uint32 -} - -func newMetrics() *metrics { - return &metrics{} -} - -func (m *metrics) store(rows, cols uint32) { - size := rows * cols - - if m.isFirst { - m.isFirst = false - m.minRows = rows - m.maxRows = rows - m.avgRows = rows - m.minCols = cols - m.maxCols = cols - m.avgCols = cols - m.minSize = size - m.maxSize = size - m.avgSize = size - - return - } - - if m.minRows > rows { - m.minRows = rows - } - - if m.maxRows < rows { - m.maxRows = rows - } - - m.avgRows = (m.avgRows + rows) / 2 - - if m.minCols > cols { - m.minCols = cols - } - - if m.maxCols < cols { - m.maxCols = cols - } - - m.avgCols = (m.avgCols + cols) / 2 - - if m.minSize > size { - m.minSize = size - } - - if m.maxSize < size { - m.maxSize = size - } - - m.avgSize = (m.avgSize + size) / 2 -} - -func (m *metrics) String() string { - return fmt.Sprintf( - "\n--- Rows ---\nmin=%d max=%d avg=%d\n--- Cols ---\nmin=%d max=%d avg=%d\n--- Size ---\nmin=%d max=%d avg=%d\n", - m.minRows, - m.maxRows, - m.avgRows, - m.minCols, - m.maxCols, - m.avgCols, - m.minSize, - m.maxSize, - m.avgSize, - ) -} - -func GetMetrics() string { - return defaultMetrics.String() -} diff --git a/adnl/rldp/raptorq/discmath/sparse-matrix.go b/adnl/rldp/raptorq/discmath/sparse-matrix.go deleted file mode 100644 index c08cd700..00000000 --- a/adnl/rldp/raptorq/discmath/sparse-matrix.go +++ /dev/null @@ -1,132 +0,0 @@ -package discmath - -import "sort" - -type exists struct{} - -type SparseMatrixGF2 struct { - data map[uint64]exists - rows uint32 - cols uint32 -} - -func NewSparseMatrixGF2(rows, cols uint32) *SparseMatrixGF2 { - return &SparseMatrixGF2{ - data: map[uint64]exists{}, - rows: rows, - cols: cols, - } -} - -func (s *SparseMatrixGF2) Set(row, col uint32) { - if row >= s.rows { - panic("row is out of range") - } - if col >= s.cols { - panic("col is out of range") - } - - s.data[(uint64(row)<<32)|uint64(col)] = exists{} -} - -func (s *SparseMatrixGF2) NonZeroes() int { - return len(s.data) -} - -func (s *SparseMatrixGF2) ColsNum() uint32 { - return s.cols -} - -func (s *SparseMatrixGF2) RowsNum() uint32 { - return s.rows -} - -func (s *SparseMatrixGF2) Transpose() *SparseMatrixGF2 { - m := NewSparseMatrixGF2(s.cols, s.rows) - for k := range s.data { - row := k >> 32 - col := (k << 32) >> 32 - m.data[(row<<32)|col] = exists{} - } - return m -} - -func (s *SparseMatrixGF2) Each(f func(row, col uint32)) { - for k := range s.data { - row := k >> 32 - col := (k << 32) >> 32 - f(uint32(row), uint32(col)) - } -} - -func (s *SparseMatrixGF2) GetCols(row uint32) (cols []uint32) { - for k := range s.data { - if ((k << 32) >> 32) == uint64(row) { - col := k >> 32 - cols = append(cols, uint32(col)) - } - } - sort.Slice(cols, func(i, j int) bool { return cols[i] < cols[j] }) - return cols -} - -func (s *SparseMatrixGF2) GetRows(col uint32) (rows []uint32) { - for k := range s.data { - if (k >> 32) == uint64(col) { - row := (k << 32) >> 32 - rows = append(rows, uint32(row)) - } - } - sort.Slice(rows, func(i, j int) bool { return rows[i] < rows[j] }) - return rows -} - -func (s *SparseMatrixGF2) ApplyRowsPermutation(permutation []uint32) *SparseMatrixGF2 { - permutation = InversePermutation(permutation) - - res := NewSparseMatrixGF2(s.rows, s.cols) - s.Each(func(row, col uint32) { - res.Set(permutation[row], col) - }) - return res -} - -func (s *SparseMatrixGF2) ApplyColsPermutation(permutation []uint32) *SparseMatrixGF2 { - permutation = InversePermutation(permutation) - - res := NewSparseMatrixGF2(s.rows, s.cols) - s.Each(func(row, col uint32) { - res.Set(row, permutation[col]) - }) - return res -} - -func (s *SparseMatrixGF2) ToDense(rowFrom, colFrom, rowSize, colSize uint32) *PlainOffsetMatrixGF2 { - m := NewPlainOffsetMatrixGF2(rowSize, colSize) - s.Each(func(row, col uint32) { - if (row >= rowFrom && row < rowFrom+rowSize) && - (col >= colFrom && col < colFrom+colSize) { - m.Set(row-rowFrom, col-colFrom) - } - }) - return m -} - -func (s *SparseMatrixGF2) GetBlock(rowOffset, colOffset, rowSize, colSize uint32) *SparseMatrixGF2 { - res := NewSparseMatrixGF2(rowSize, colSize) - s.Each(func(row, col uint32) { - if (row >= rowOffset && row < rowSize+rowOffset) && - (col >= colOffset && col < colSize+colOffset) { - res.Set(row-rowOffset, col-colOffset) - } - }) - return res -} - -func InversePermutation(mut []uint32) []uint32 { - res := make([]uint32, len(mut)) - for i, u := range mut { - res[u] = uint32(i) - } - return res -} diff --git a/adnl/rldp/raptorq/discmath/utils.go b/adnl/rldp/raptorq/discmath/utils.go new file mode 100644 index 00000000..debf64bd --- /dev/null +++ b/adnl/rldp/raptorq/discmath/utils.go @@ -0,0 +1,9 @@ +package discmath + +func InversePermutation(mut []uint32) []uint32 { + res := make([]uint32, len(mut)) + for i, u := range mut { + res[u] = uint32(i) + } + return res +} diff --git a/adnl/rldp/raptorq/solver_test.go b/adnl/rldp/raptorq/solver_test.go index d3ef9a70..e474f01d 100644 --- a/adnl/rldp/raptorq/solver_test.go +++ b/adnl/rldp/raptorq/solver_test.go @@ -6,8 +6,6 @@ import ( "encoding/binary" "encoding/hex" "testing" - - "github.com/xssnick/tonutils-go/adnl/rldp/raptorq/discmath" ) func Test_Encode(t *testing.T) { @@ -25,8 +23,6 @@ func Test_Encode(t *testing.T) { if !bytes.Equal(sx, should) { t.Fatal("encoded not eq, got", hex.EncodeToString(sx)) } - - t.Log(discmath.GetMetrics()) } func Test_EncodeDecode(t *testing.T) { @@ -60,8 +56,6 @@ func Test_EncodeDecode(t *testing.T) { if !bytes.Equal(data, str) { t.Fatal("initial data not eq decrypted") } - - t.Log(discmath.GetMetrics()) } func Test_EncodeDecodeFuzz(t *testing.T) { @@ -110,13 +104,10 @@ func Test_EncodeDecodeFuzz(t *testing.T) { t.Fatal("initial data not eq decrypted") } } - - t.Log(discmath.GetMetrics()) } // Benchmark_EncodeDecodeFuzz-12: MatrixGF2 907 1329193 ns/op 653478 B/op 1806 allocs/op -// Benchmark_EncodeDecodeFuzz-12: PlainMatrixGF2 1003 1429374 ns/op 652845 B/op 1809 allocs/op -// Benchmark_EncodeDecodeFuzz-12: PlainOffsetMatrixGF2 1082 1114249 ns/op 652923 B/op 1810 allocs/op +// Benchmark_EncodeDecodeFuzz-12: PlainMatrixGF2 1082 1114249 ns/op 652923 B/op 1810 allocs/op func Benchmark_EncodeDecodeFuzz(b *testing.B) { str := make([]byte, 4096) rand.Read(str) From 1e22488367c78997000ee291b94e3caa1c6c20e1 Mon Sep 17 00:00:00 2001 From: Tikhon Slasten Date: Tue, 7 Nov 2023 12:06:52 +0800 Subject: [PATCH 13/23] Get next node on update --- adnl/dht/client.go | 58 +++++++++++++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/adnl/dht/client.go b/adnl/dht/client.go index 4532717d..d54eae8a 100644 --- a/adnl/dht/client.go +++ b/adnl/dht/client.go @@ -10,7 +10,6 @@ import ( "fmt" "net" "reflect" - "runtime" "sync" "sync/atomic" "time" @@ -431,10 +430,13 @@ func (c *Client) FindValue(ctx context.Context, key *Key, continuation ...*Conti const threads = 8 result := make(chan *foundResult, threads) - var numNoTasks int64 + + var numWaitingNextNode int + var foundValue bool + cond := sync.NewCond(&sync.Mutex{}) + for i := 0; i < threads; i++ { go func() { - noTasks := false for { select { case <-threadCtx.Done(): @@ -442,27 +444,36 @@ func (c *Client) FindValue(ctx context.Context, key *Key, continuation ...*Conti default: } - // we get most prioritized node, priority depends on depth node, _ := plist.getNode() if node == nil { - if !noTasks { - noTasks = true - atomic.AddInt64(&numNoTasks, 1) - } + cond.L.Lock() + numWaitingNextNode++ - if atomic.LoadInt64(&numNoTasks) < threads { - // something is pending - runtime.Gosched() - continue - } + for { + if foundValue { + cond.L.Unlock() - result <- nil - return - } + return + } + + if numWaitingNextNode == threads { + cond.L.Unlock() + + result <- nil - if noTasks { - noTasks = false - atomic.AddInt64(&numNoTasks, -1) + return + } + + node, _ = plist.getNode() + if node != nil { + break + } + + cond.Wait() + } + + numWaitingNextNode-- + cond.L.Unlock() } findCtx, findCancel := context.WithTimeout(threadCtx, queryTimeout) @@ -476,6 +487,12 @@ func (c *Client) FindValue(ctx context.Context, key *Key, continuation ...*Conti switch v := val.(type) { case *Value: result <- &foundResult{value: v, node: node} + cond.L.Lock() + foundValue = true + cond.Broadcast() + cond.L.Unlock() + + return case []*Node: if len(v) > 24 { // max 24 nodes to add @@ -489,6 +506,7 @@ func (c *Client) FindValue(ctx context.Context, key *Key, continuation ...*Conti } plist.addNode(newNode) + cond.Signal() } } } @@ -502,7 +520,9 @@ func (c *Client) FindValue(ctx context.Context, key *Key, continuation ...*Conti if val == nil { return nil, cont, ErrDHTValueIsNotFound } + cont.checkedNodes = append(cont.checkedNodes, val.node) + return val.value, cont, nil } } From 46204ba561743107d5d1ad79c4cc760e9991aaf3 Mon Sep 17 00:00:00 2001 From: Tikhon Slasten Date: Wed, 22 Nov 2023 16:54:27 +0800 Subject: [PATCH 14/23] JSON unmarshal of Address --- address/addr.go | 17 +++++++ address/addr_test.go | 110 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+) diff --git a/address/addr.go b/address/addr.go index 9e0fba40..d79b9c20 100644 --- a/address/addr.go +++ b/address/addr.go @@ -128,6 +128,23 @@ func (a *Address) MarshalJSON() ([]byte, error) { return []byte(fmt.Sprintf("%q", a.String())), nil } +func (a *Address) UnmarshalJSON(data []byte) error { + if len(data) < 2 || data[0] != '"' || data[len(data)-1] != '"' { + return fmt.Errorf("invalid data") + } + + data = data[1 : len(data)-1] + + addr, err := ParseAddr(string(data)) + if err != nil { + return err + } + + *a = *addr + + return nil +} + func MustParseAddr(addr string) *Address { a, err := ParseAddr(addr) if err != nil { diff --git a/address/addr_test.go b/address/addr_test.go index 444be29d..3e196e16 100644 --- a/address/addr_test.go +++ b/address/addr_test.go @@ -461,3 +461,113 @@ func TestParseflags(t *testing.T) { }) } } + +func TestAddress_MarshalJSON(t *testing.T) { + tests := []struct { + name string + address *Address + want string + wantErr bool + }{ + { + name: "none", + address: NewAddressNone(), + want: "\"NONE\"", + wantErr: false, + }, + { + name: "std address", + address: MustParseAddr("EQCTDVUzmAq6EfzYGEWpVOv16yo-H5Vw3B0rktcidz_ULOUj"), + want: "\"EQCTDVUzmAq6EfzYGEWpVOv16yo-H5Vw3B0rktcidz_ULOUj\"", + wantErr: false, + }, + { + name: "ext address", + address: NewAddressExt(0, 256, []byte{}), + want: "\"EXT_ADDRESS\"", + wantErr: false, + }, + { + name: "var address", + address: NewAddressVar(0, 0, 256, []byte{}), + want: "\"VAR_ADDRESS\"", + wantErr: false, + }, + { + name: "not supported type", + address: &Address{addrType: 5}, + want: "\"NOT_SUPPORTED\"", + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.address.MarshalJSON() + if (err != nil) != tt.wantErr { + t.Errorf("MarshalJSON() error = %v, wantErr %v", err, tt.wantErr) + return + } + + wantBytes := []byte(tt.want) + if !reflect.DeepEqual(got, wantBytes) { + t.Errorf("MarshalJSON() got = %v, want %v", string(got), tt.want) + } + }) + } +} + +func TestAddress_UnmarshalJSON(t *testing.T) { + tests := []struct { + name string + address string + want Address + wantErr bool + }{ + { + name: "invalid empty", + address: "", + want: Address{}, + wantErr: true, + }, + { + name: "empty", + address: "\"\"", + want: Address{}, + wantErr: true, + }, + { + name: "valid", + address: "\"EQC6KV4zs8TJtSZapOrRFmqSkxzpq-oSCoxekQRKElf4nC1I\"", + want: Address{ + addrType: StdAddress, + bitsLen: 256, + flags: flags{bounceable: true, testnet: false}, + workchain: 0, + data: []byte{186, 41, 94, 51, 179, 196, 201, 181, 38, 90, 164, 234, 209, 22, 106, 146, 147, 28, 233, 171, 234, 18, 10, 140, 94, 145, 4, 74, 18, 87, 248, 156}, + }, + wantErr: false, + }, + { + name: "invalid", + address: "\"AQCTDVUzmAq6EfzYGEWpVOv16yo-H5Vw3B0rktcidz_ULOUj\"", + want: Address{}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var a Address + + err := a.UnmarshalJSON([]byte(tt.address)) + if (err != nil) != tt.wantErr { + t.Errorf("UnmarshalJSON() error = %v, wantErr %v", err, tt.wantErr) + } + + if !reflect.DeepEqual(a, tt.want) { + t.Errorf("UnmarshalJSON() got = %v, want %v", a, tt.want) + } + }) + } +} From f463181899526642fe91ec049480884f99dc41da Mon Sep 17 00:00:00 2001 From: Tikhon Slasten Date: Wed, 22 Nov 2023 17:50:36 +0800 Subject: [PATCH 15/23] JSON unmarshal of Coins --- tlb/coins.go | 34 +++++++++++++- tlb/coins_test.go | 117 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+), 2 deletions(-) diff --git a/tlb/coins.go b/tlb/coins.go index bdd599a1..ce941db2 100644 --- a/tlb/coins.go +++ b/tlb/coins.go @@ -9,6 +9,8 @@ import ( "github.com/xssnick/tonutils-go/tvm/cell" ) +var errInvalid = errors.New("invalid string") + type Coins struct { decimals int val *big.Int @@ -116,15 +118,26 @@ func FromNanoTONU(val uint64) Coins { } } +func FromNanoTONStr(val string) (Coins, error) { + v, ok := new(big.Int).SetString(val, 10) + if !ok { + return Coins{}, errInvalid + } + + return Coins{ + decimals: 9, + val: v, + }, nil +} + func FromTON(val string) (Coins, error) { return FromDecimal(val, 9) } func FromDecimal(val string, decimals int) (Coins, error) { if decimals < 0 || decimals >= 128 { - return Coins{}, fmt.Errorf("invalid decmals") + return Coins{}, fmt.Errorf("invalid decimals") } - errInvalid := errors.New("invalid string") s := strings.SplitN(val, ".", 2) @@ -192,3 +205,20 @@ func (g Coins) ToCell() (*cell.Cell, error) { func (g Coins) MarshalJSON() ([]byte, error) { return []byte(fmt.Sprintf("%q", g.Nano().String())), nil } + +func (g *Coins) UnmarshalJSON(data []byte) error { + if len(data) < 2 || data[0] != '"' || data[len(data)-1] != '"' { + return fmt.Errorf("invalid data") + } + + data = data[1 : len(data)-1] + + coins, err := FromNanoTONStr(string(data)) + if err != nil { + return err + } + + *g = coins + + return nil +} diff --git a/tlb/coins_test.go b/tlb/coins_test.go index b2a0dfa2..b4084dc3 100644 --- a/tlb/coins_test.go +++ b/tlb/coins_test.go @@ -4,6 +4,7 @@ import ( "crypto/rand" "fmt" "math/big" + "reflect" "strings" "testing" ) @@ -157,3 +158,119 @@ func TestCoins_Decimals(t *testing.T) { }) } } + +func TestCoins_MarshalJSON(t *testing.T) { + tests := []struct { + name string + coins Coins + want string + wantErr bool + }{ + { + name: "0.123456789 TON", + coins: Coins{ + decimals: 9, + val: big.NewInt(123_456_789), + }, + want: "\"123456789\"", + wantErr: false, + }, + { + name: "1 TON", + coins: Coins{ + decimals: 9, + val: big.NewInt(1_000_000_000), + }, + want: "\"1000000000\"", + wantErr: false, + }, + { + name: "123 TON", + coins: Coins{ + decimals: 9, + val: big.NewInt(123_000_000_000), + }, + want: "\"123000000000\"", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.coins.MarshalJSON() + if (err != nil) != tt.wantErr { + t.Errorf("MarshalJSON() error = %v, wantErr %v", err, tt.wantErr) + return + } + + wantBytes := []byte(tt.want) + if !reflect.DeepEqual(got, wantBytes) { + t.Errorf("MarshalJSON() got = %v, want %v", string(got), tt.want) + } + }) + } +} + +func TestCoins_UnmarshalJSON(t *testing.T) { + tests := []struct { + name string + data string + want Coins + wantErr bool + }{ + { + name: "empty invalid", + data: "", + wantErr: true, + }, + { + name: "empty", + data: "\"\"", + wantErr: true, + }, + { + name: "invalid", + data: "\"123a\"", + wantErr: true, + }, + { + name: "0.123456789 TON", + data: "\"123456789\"", + want: Coins{ + decimals: 9, + val: big.NewInt(123_456_789), + }, + }, + { + name: "1 TON", + data: "\"1000000000\"", + want: Coins{ + decimals: 9, + val: big.NewInt(1_000_000_000), + }, + }, + { + name: "123 TON", + data: "\"123000000000\"", + want: Coins{ + decimals: 9, + val: big.NewInt(123_000_000_000), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var coins Coins + + err := coins.UnmarshalJSON([]byte(tt.data)) + if (err != nil) != tt.wantErr { + t.Errorf("UnmarshalJSON() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if !reflect.DeepEqual(coins, tt.want) { + t.Errorf("UnmarshalJSON() got = %v, want %v", coins, tt.want) + } + }) + } +} From 5acd2f076b396801edafdd439b9b5f8bff4581df Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Tue, 28 Nov 2023 10:03:54 +0400 Subject: [PATCH 16/23] Better TL-B Either serialization logic & fixed transaction interface tlb tags --- example/block-scan/main.go | 2 +- tlb/loader.go | 457 ++++++++++++++++++++++--------------- tlb/loader_test.go | 24 +- tlb/message.go | 95 +------- tlb/message_test.go | 6 +- ton/sendmessage.go | 2 +- ton/wallet/highloadv2r2.go | 3 +- ton/wallet/v3.go | 3 +- ton/wallet/v4r2.go | 3 +- ton/wallet/wallet_test.go | 6 +- 10 files changed, 319 insertions(+), 282 deletions(-) diff --git a/example/block-scan/main.go b/example/block-scan/main.go index f0bb2ca0..efde13ee 100644 --- a/example/block-scan/main.go +++ b/example/block-scan/main.go @@ -160,7 +160,7 @@ func main() { log.Printf("no transactions in %d block\n", master.SeqNo) } - master, err = api.WaitForBlock(master.SeqNo + 1).GetMasterchainInfo(ctx) + master, err = api.WaitForBlock(master.SeqNo+1).LookupBlock(ctx, master.Workchain, master.Shard, master.SeqNo+1) if err != nil { log.Fatalln("get masterchain info err: ", err.Error()) return diff --git a/tlb/loader.go b/tlb/loader.go index 868ca774..043f6c0b 100644 --- a/tlb/loader.go +++ b/tlb/loader.go @@ -30,7 +30,12 @@ type Marshaller interface { // bool - loads 1 bit boolean // addr - loads ton address // maybe - reads 1 bit, and loads rest if its 1, can be used in combination with others only -// either X Y - reads 1 bit, if its 0 - loads X, if 1 - loads Y +// either [leave {bits},{refs}] X Y - reads 1 bit, if its 0 - loads X, if 1 - loads Y, +// +// tries to serialize first condition, if not succeed (not enough free bits or refs), then second. +// if 'leave' is specified, then after write it will additionally check specified +// number of free bits and refs in cell. +// // ?FieldName - Conditional field loading depending on boolean value of specified field. // / Specified field must be declared before tag usage, or it will be always false during loading // Some tags can be combined, for example "dict 256", "maybe ^" @@ -106,18 +111,30 @@ func loadFromCell(v any, slice *cell.Slice, skipProofBranches, skipMagic bool) e } if settings[0] == "either" { - if len(settings) < 3 { + settings = settings[1:] + if len(settings) < 2 { panic("either tag should have 2 args") } + + if settings[0] == "leave" { + settings = settings[1:] + + if len(settings) < 3 { + panic("either tag should have 2 args and leave tag should have 1 arg") + } + // skip leave + settings = settings[1:] + } + isSecond, err := loader.LoadBoolBit() if err != nil { return fmt.Errorf("failed to load maybe for %s, err: %w", structField.Name, err) } if !isSecond { - settings = []string{settings[1]} + settings = []string{settings[0]} } else { - settings = []string{settings[2]} + settings = []string{settings[1]} } } @@ -160,7 +177,7 @@ func loadFromCell(v any, slice *cell.Slice, skipProofBranches, skipMagic bool) e if structField.Type.Kind() == reflect.Interface { allowed := strings.Join(settings, "") if !strings.HasPrefix(allowed, "[") || !strings.HasSuffix(allowed, "]") { - panic("corrupted allowed list tag, should be [a,b,c], got " + allowed) + panic("corrupted allowed list tag of field " + structField.Name + ", should be [a,b,c], got " + allowed) } // cut brackets @@ -436,8 +453,8 @@ func ToCell(v any) (*cell.Cell, error) { root := cell.BeginCell() +next: for i := 0; i < rv.NumField(); i++ { - builder := root structField := rv.Type().Field(i) parseType := structField.Type fieldVal := rv.Field(i) @@ -466,249 +483,323 @@ func ToCell(v any) (*cell.Cell, error) { } if fieldVal.IsNil() { - if err := builder.StoreBoolBit(false); err != nil { + if err := root.StoreBoolBit(false); err != nil { return nil, fmt.Errorf("cannot store maybe bit: %w", err) } continue } - if err := builder.StoreBoolBit(true); err != nil { + if err := root.StoreBoolBit(true); err != nil { return nil, fmt.Errorf("cannot store maybe bit: %w", err) } settings = settings[1:] } + if structField.Type.Kind() == reflect.Pointer && structField.Type.Elem().Kind() != reflect.Struct { + // to same process both pointers and types + parseType = parseType.Elem() + fieldVal = fieldVal.Elem() + } + if settings[0] == "either" { - if len(settings) < 3 { + settings = settings[1:] + + if len(settings) < 2 { panic("either tag should have 2 args") } - // currently, if one of the options is ref - we choose it - second := strings.HasPrefix(settings[2], "^") - if err := builder.StoreBoolBit(second); err != nil { - return nil, fmt.Errorf("cannot store maybe bit: %w", err) + leaveBits, leaveRefs := 0, 0 + if settings[0] == "leave" { + settings = settings[1:] + + if len(settings) < 3 { + panic("either tag should have 2 args and leave tag should have 1 arg") + } + + spl := strings.Split(settings[0], ",") + settings = settings[1:] + + val, err := strconv.ParseUint(spl[0], 10, 10) + if err != nil { + panic("invalid argument for either leave bits") + } + // set how many free bits we need to have after either written + leaveBits = int(val) + + if len(spl) > 1 { + val, err = strconv.ParseUint(spl[1], 10, 10) + if err != nil { + panic("invalid argument for either leave refs") + } + // set how many free efs we need to have after either written + leaveRefs = int(val) + } } - if second { - settings = []string{settings[2]} - } else { - settings = []string{settings[1]} + // we try first option, if it is overflows then we try second + for x := 0; x < 2; x++ { + builder := cell.BeginCell() + if err := storeField([]string{settings[x]}, builder, structField, fieldVal, parseType); err != nil { + return nil, fmt.Errorf("failed to serialize field %s to cell as either %d: %w", structField.Name, x, err) + } + + // check if we have enough free bits + if x == 0 && (int(root.BitsLeft())-int(builder.BitsUsed()+1) < leaveBits || int(root.RefsLeft())-int(builder.RefsUsed()) < leaveRefs) { + // if not, then we try second option + continue + } + + if err := root.StoreUInt(uint64(x), 1); err != nil { + return nil, fmt.Errorf("cannot store either bit: %w", err) + } + if err := root.StoreBuilder(builder); err != nil { + return nil, fmt.Errorf("failed to concat builder of field %s to cell as either %d: %w", structField.Name, x, err) + } + + continue next } + + return nil, fmt.Errorf("failed to serialize either field %s to cell: no valid options", structField.Name) } - if structField.Type.Kind() == reflect.Pointer && structField.Type.Elem().Kind() != reflect.Struct { - // to same process both pointers and types - parseType = parseType.Elem() - fieldVal = fieldVal.Elem() + if err := storeField(settings, root, structField, fieldVal, parseType); err != nil { + return nil, fmt.Errorf("failed to serialize field %s to cell: %w", structField.Name, err) } + } - asRef := false - if settings[0] == "^" { - asRef = true - settings = settings[1:] - builder = cell.BeginCell() + return root.EndCell(), nil +} + +func storeField(settings []string, root *cell.Builder, structField reflect.StructField, fieldVal reflect.Value, parseType reflect.Type) error { + builder := root + + asRef := false + if settings[0] == "^" { + asRef = true + settings = settings[1:] + builder = cell.BeginCell() + } + + if structField.Type.Kind() == reflect.Interface { + allowed := strings.Join(settings, "") + if !strings.HasPrefix(allowed, "[") || !strings.HasSuffix(allowed, "]") { + panic("corrupted allowed list tag of field " + structField.Name + ", should be [a,b,c], got " + allowed) } - if structField.Type.Kind() == reflect.Interface { - allowed := strings.Join(settings, "") - if !strings.HasPrefix(allowed, "[") || !strings.HasSuffix(allowed, "]") { - panic("corrupted allowed list tag, should be [a,b,c], got " + allowed) - } + // cut brackets + allowed = allowed[1 : len(allowed)-1] + types := strings.Split(allowed, ",") - // cut brackets - allowed = allowed[1 : len(allowed)-1] - types := strings.Split(allowed, ",") + t := fieldVal.Elem().Type() + if t.Kind() == reflect.Pointer { + t = t.Elem() + } - t := fieldVal.Elem().Type() - found := false - for _, typ := range types { - if t.Name() == typ { - found = true - break - } + found := false + for _, typ := range types { + if t.Name() == typ { + found = true + break } + } - if !found { - return nil, fmt.Errorf("unexpected data to serialize, not registered magic in tag") - } - settings = settings[:0] + if !found { + return fmt.Errorf("unexpected data to serialize, not registered magic in tag for %s", t.String()) } + settings = settings[:0] + } - if len(settings) == 0 || settings[0] == "." { - c, err := structStore(fieldVal, structField.Type.Name()) - if err != nil { - return nil, err - } + if len(settings) == 0 || settings[0] == "." { + c, err := structStore(fieldVal, structField.Type.Name()) + if err != nil { + return err + } - err = builder.StoreBuilder(c.ToBuilder()) - if err != nil { - return nil, fmt.Errorf("failed to store cell to builder for %s, err: %w", structField.Name, err) - } - } else if settings[0] == "##" { - num, err := strconv.ParseUint(settings[1], 10, 64) - if err != nil { - // we panic, because its developer's issue, need to fix tag - panic("corrupted num bits in ## tag") - } + err = builder.StoreBuilder(c.ToBuilder()) + if err != nil { + return fmt.Errorf("failed to store cell to builder for %s, err: %w", structField.Name, err) + } + } else if settings[0] == "##" { + num, err := strconv.ParseUint(settings[1], 10, 64) + if err != nil { + // we panic, because its developer's issue, need to fix tag + panic("corrupted num bits in ## tag") + } - switch { - case num <= 64: - switch parseType.Kind() { - case reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8, reflect.Int: - err = builder.StoreInt(fieldVal.Int(), uint(num)) - if err != nil { - return nil, fmt.Errorf("failed to store int %d, err: %w", num, err) - } - case reflect.Uint64, reflect.Uint32, reflect.Uint16, reflect.Uint8, reflect.Uint: - err = builder.StoreUInt(fieldVal.Uint(), uint(num)) - if err != nil { - return nil, fmt.Errorf("failed to store int %d, err: %w", num, err) - } - default: - if parseType == reflect.TypeOf(&big.Int{}) { - err = builder.StoreBigInt(fieldVal.Interface().(*big.Int), uint(num)) - if err != nil { - return nil, fmt.Errorf("failed to store bigint %d, err: %w", num, err) - } - } else { - panic("unexpected field type for tag ## - " + parseType.String()) - } + switch { + case num <= 64: + switch parseType.Kind() { + case reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8, reflect.Int: + err = builder.StoreInt(fieldVal.Int(), uint(num)) + if err != nil { + return fmt.Errorf("failed to store int %d, err: %w", num, err) } - case num <= 256: - err := builder.StoreBigInt(fieldVal.Interface().(*big.Int), uint(num)) + case reflect.Uint64, reflect.Uint32, reflect.Uint16, reflect.Uint8, reflect.Uint: + err = builder.StoreUInt(fieldVal.Uint(), uint(num)) if err != nil { - return nil, fmt.Errorf("failed to store bigint %d, err: %w", num, err) + return fmt.Errorf("failed to store int %d, err: %w", num, err) + } + default: + if parseType == reflect.TypeOf(&big.Int{}) { + err = builder.StoreBigInt(fieldVal.Interface().(*big.Int), uint(num)) + if err != nil { + return fmt.Errorf("failed to store bigint %d, err: %w", num, err) + } + } else { + panic("unexpected field type for tag ## - " + parseType.String()) } } - } else if settings[0] == "addr" { - err := builder.StoreAddr(fieldVal.Interface().(*address.Address)) - if err != nil { - return nil, fmt.Errorf("failed to store address, err: %w", err) - } - } else if settings[0] == "bool" { - err := builder.StoreBoolBit(fieldVal.Bool()) - if err != nil { - return nil, fmt.Errorf("failed to store bool, err: %w", err) - } - } else if settings[0] == "bits" { - num, err := strconv.Atoi(settings[1]) + case num <= 256: + err := builder.StoreBigInt(fieldVal.Interface().(*big.Int), uint(num)) if err != nil { - // we panic, because its developer's issue, need to fix tag - panic("corrupted num bits in bits tag") + return fmt.Errorf("failed to store bigint %d, err: %w", num, err) } + } + } else if settings[0] == "addr" { + err := builder.StoreAddr(fieldVal.Interface().(*address.Address)) + if err != nil { + return fmt.Errorf("failed to store address, err: %w", err) + } + } else if settings[0] == "bool" { + err := builder.StoreBoolBit(fieldVal.Bool()) + if err != nil { + return fmt.Errorf("failed to store bool, err: %w", err) + } + } else if settings[0] == "bits" { + num, err := strconv.Atoi(settings[1]) + if err != nil { + // we panic, because its developer's issue, need to fix tag + panic("corrupted num bits in bits tag") + } - err = builder.StoreSlice(fieldVal.Bytes(), uint(num)) - if err != nil { - return nil, fmt.Errorf("failed to store bits %d, err: %w", num, err) - } - } else if parseType == reflect.TypeOf(Magic{}) { - var sz, base int - if strings.HasPrefix(settings[0], "#") { - base = 16 - sz = (len(settings[0]) - 1) * 4 - } else if strings.HasPrefix(settings[0], "$") { - base = 2 - sz = len(settings[0]) - 1 - } else { - panic("unknown magic value type in tag") - } + err = builder.StoreSlice(fieldVal.Bytes(), uint(num)) + if err != nil { + return fmt.Errorf("failed to store bits %d, err: %w", num, err) + } + } else if parseType == reflect.TypeOf(Magic{}) { + var sz, base int + if strings.HasPrefix(settings[0], "#") { + base = 16 + sz = (len(settings[0]) - 1) * 4 + } else if strings.HasPrefix(settings[0], "$") { + base = 2 + sz = len(settings[0]) - 1 + } else { + panic("unknown magic value type in tag") + } - if sz > 64 { - panic("too big magic value type in tag") - } + if sz > 64 { + panic("too big magic value type in tag") + } - magic, err := strconv.ParseInt(settings[0][1:], base, 64) - if err != nil { - panic("corrupted magic value in tag") + magic, err := strconv.ParseInt(settings[0][1:], base, 64) + if err != nil { + panic("corrupted magic value in tag") + } + + err = builder.StoreUInt(uint64(magic), uint(sz)) + if err != nil { + return fmt.Errorf("failed to store magic: %w", err) + } + } else if settings[0] == "dict" { + var dict *cell.Dictionary + + settings = settings[1:] + + isInline := len(settings) > 0 && settings[0] == "inline" + if isInline { + settings = settings[1:] + } + + if len(settings) < 3 || settings[1] != "->" { + dict = fieldVal.Interface().(*cell.Dictionary) + } else { + if fieldVal.Kind() != reflect.Map { + return fmt.Errorf("want to create dictionary from map, but instead got %s type", fieldVal.Type()) + } + if fieldVal.Type().Key() != reflect.TypeOf("") { + return fmt.Errorf("map key should be string, but instead got %s type", fieldVal.Type().Key()) } - err = builder.StoreUInt(uint64(magic), uint(sz)) + sz, err := strconv.ParseUint(settings[0], 10, 64) if err != nil { - return nil, fmt.Errorf("failed to store magic: %w", err) + panic(fmt.Sprintf("cannot deserialize field '%s' as dict, bad size '%s'", structField.Name, settings[0])) } - } else if settings[0] == "dict" { - var dict *cell.Dictionary - if len(settings) < 4 || settings[2] != "->" { - dict = fieldVal.Interface().(*cell.Dictionary) - } else { - if fieldVal.Kind() != reflect.Map { - return nil, fmt.Errorf("want to create dictionary from map, but instead got %s type", fieldVal.Type()) - } - if fieldVal.Type().Key() != reflect.TypeOf("") { - return nil, fmt.Errorf("map key should be string, but instead got %s type", fieldVal.Type().Key()) - } + dict = cell.NewDict(uint(sz)) - sz, err := strconv.ParseUint(settings[1], 10, 64) - if err != nil { - panic(fmt.Sprintf("cannot deserialize field '%s' as dict, bad size '%s'", structField.Name, settings[1])) + for _, mapK := range fieldVal.MapKeys() { + mapKI, ok := big.NewInt(0).SetString(mapK.Interface().(string), 10) + if !ok { + return fmt.Errorf("cannot parse '%s' map key to big int of '%s' field", mapK.Interface().(string), structField.Name) } - dict = cell.NewDict(uint(sz)) - - for _, mapK := range fieldVal.MapKeys() { - mapKI, ok := big.NewInt(0).SetString(mapK.Interface().(string), 10) - if !ok { - return nil, fmt.Errorf("cannot parse '%s' map key to big int of '%s' field", mapK.Interface().(string), structField.Name) - } - - mapKB := cell.BeginCell() - if err := mapKB.StoreBigInt(mapKI, uint(sz)); err != nil { - return nil, fmt.Errorf("store big int of size %d to %s field", sz, structField.Name) - } + mapKB := cell.BeginCell() + if err := mapKB.StoreBigInt(mapKI, uint(sz)); err != nil { + return fmt.Errorf("store big int of size %d to %s field", sz, structField.Name) + } - mapV := fieldVal.MapIndex(mapK) + mapV := fieldVal.MapIndex(mapK) - cellVT := reflect.StructOf([]reflect.StructField{{ - Name: "Value", - Type: mapV.Type(), - Tag: reflect.StructTag(fmt.Sprintf("tlb:%q", strings.Join(settings[3:], " "))), - }}) - cellV := reflect.New(cellVT).Elem() - cellV.Field(0).Set(mapV) + cellVT := reflect.StructOf([]reflect.StructField{{ + Name: "Value", + Type: mapV.Type(), + Tag: reflect.StructTag(fmt.Sprintf("tlb:%q", strings.Join(settings[2:], " "))), + }}) + cellV := reflect.New(cellVT).Elem() + cellV.Field(0).Set(mapV) - mapVC, err := ToCell(cellV.Interface()) - if err != nil { - return nil, fmt.Errorf("creating cell for dict value of '%s' field: %w", structField.Name, err) - } + mapVC, err := ToCell(cellV.Interface()) + if err != nil { + return fmt.Errorf("creating cell for dict value of '%s' field: %w", structField.Name, err) + } - if err := dict.Set(mapKB.EndCell(), mapVC); err != nil { - return nil, fmt.Errorf("set dict key/value on '%s' field: %w", structField.Name, err) - } + if err := dict.Set(mapKB.EndCell(), mapVC); err != nil { + return fmt.Errorf("set dict key/value on '%s' field: %w", structField.Name, err) } } + } - err := builder.StoreDict(dict) + if isInline { + dCell, err := dict.ToCell() if err != nil { - return nil, fmt.Errorf("failed to store dict for %s, err: %w", structField.Name, err) + return fmt.Errorf("failed to serialize inline dict to cell for %s, err: %w", structField.Name, err) } - } else if settings[0] == "var" { - if settings[1] == "uint" { - sz, err := strconv.Atoi(settings[2]) - if err != nil { - panic(err.Error()) - } - err = builder.StoreBigVarUInt(fieldVal.Interface().(*big.Int), uint(sz)) - if err != nil { - return nil, fmt.Errorf("failed to store var uint: %w", err) - } - } else { - panic("var of type " + settings[1] + " is not supported") + if err = builder.StoreBuilder(dCell.ToBuilder()); err != nil { + return fmt.Errorf("failed to store inline dict for %s, err: %w", structField.Name, err) } } else { - panic(fmt.Sprintf("cannot serialize field '%s' as tag '%s' of struct '%s', use manual serialization", structField.Name, tag, rv.Type().String())) + if err := builder.StoreDict(dict); err != nil { + return fmt.Errorf("failed to store dict for %s, err: %w", structField.Name, err) + } } + } else if settings[0] == "var" { + if settings[1] == "uint" { + sz, err := strconv.Atoi(settings[2]) + if err != nil { + panic(err.Error()) + } - if asRef { - err := root.StoreRef(builder.EndCell()) + err = builder.StoreBigVarUInt(fieldVal.Interface().(*big.Int), uint(sz)) if err != nil { - return nil, fmt.Errorf("failed to store cell to ref for %s, err: %w", structField.Name, err) + return fmt.Errorf("failed to store var uint: %w", err) } + } else { + panic("var of type " + settings[1] + " is not supported") } + } else { + panic(fmt.Sprintf("cannot serialize field '%s' as tag '%s', use manual serialization", structField.Name, structField.Tag.Get("tlb"))) } - return root.EndCell(), nil + if asRef { + err := root.StoreRef(builder.EndCell()) + if err != nil { + return fmt.Errorf("failed to store cell to ref for %s, err: %w", structField.Name, err) + } + } + + return nil } var cellType = reflect.TypeOf(&cell.Cell{}) @@ -753,7 +844,7 @@ func structStore(field reflect.Value, name string) (*cell.Cell, error) { c, err := ToCell(inf) if err != nil { - return nil, fmt.Errorf("failed to store to cell for %s, err: %w", name, err) + return nil, fmt.Errorf("failed to store to cell for %s of type %s, err: %w", name, field.Type().String(), err) } return c, nil } diff --git a/tlb/loader_test.go b/tlb/loader_test.go index a79775c5..0123577e 100644 --- a/tlb/loader_test.go +++ b/tlb/loader_test.go @@ -14,7 +14,8 @@ import ( ) type smallStruct struct { - Sz uint32 `tlb:"## 8"` + Sz uint32 `tlb:"## 8"` + DictMapInlineInt32 map[string]int64 `tlb:"dict inline 5 -> ## 32"` } type manualLoad struct { @@ -77,7 +78,7 @@ type testTLB struct { Inside testInner `tlb:"^"` InsideMaybe *testInner `tlb:"maybe ^"` Part testInner `tlb:"."` - InsideMaybeEither *testInner `tlb:"maybe either ^ ."` + InsideMaybeEither *testInner `tlb:"maybe either leave 20,0 ^ ."` Bits []byte `tlb:"bits 20"` Var *big.Int `tlb:"var uint 3"` EndCell *cell.Cell `tlb:"."` @@ -160,6 +161,17 @@ func TestLoadFromCell(t *testing.T) { } } + dMapIntInlKV := map[string]int64{"2": 43, "8": -75} + dMapInnerInlineInt := cell.NewDict(5) + for k, v := range dMapIntInlKV { + err := dMapInnerInlineInt.Set( + cell.BeginCell().MustStoreBigInt(mustParseInt(k), 5).EndCell(), + cell.BeginCell().MustStoreInt(v, 32).EndCell()) + if err != nil { + t.Fatal(err) + } + } + dictC := cell.BeginCell(). MustStoreDict(d). MustStoreDict(dMapBool). @@ -167,7 +179,10 @@ func TestLoadFromCell(t *testing.T) { MustStoreDict(dMapStruct). EndCell() - mRef := cell.BeginCell().MustStoreUInt('y', 8).EndCell() + mRef := cell.BeginCell(). + MustStoreUInt('y', 8). + MustStoreBuilder(dMapInnerInlineInt.MustToCell().ToBuilder()). + EndCell() ref := cell.BeginCell().MustStoreUInt(0b1011, 4). MustStoreInt(-7172, 34). @@ -255,6 +270,9 @@ func TestLoadFromCell(t *testing.T) { if !reflect.DeepEqual(x.Part.Dict.DictMapStruct, dMapStructKV) { t.Fatal("struct dict val not eq") } + if !reflect.DeepEqual(x.Part.StructMaybe.DictMapInlineInt32, dMapIntInlKV) { + t.Fatal("struct dict val not eq") + } if x.Var.Uint64() != 999 { t.Fatal("var not eq") diff --git a/tlb/message.go b/tlb/message.go index 8000240e..7df06671 100644 --- a/tlb/message.go +++ b/tlb/message.go @@ -16,6 +16,12 @@ const ( MsgTypeExternalOut MsgType = "EXTERNAL_OUT" ) +func init() { + Register(ExternalMessage{}) + Register(ExternalMessageOut{}) + Register(InternalMessage{}) +} + type AnyMessage interface { Payload() *cell.Cell SenderAddr() *address.Address @@ -24,7 +30,7 @@ type AnyMessage interface { type Message struct { MsgType MsgType `tlb:"-"` - Msg AnyMessage `tlb:"."` + Msg AnyMessage `tlb:"[ExternalMessage,ExternalMessageOut,InternalMessage]"` } type MessagesList struct { @@ -45,7 +51,7 @@ type InternalMessage struct { CreatedLT uint64 `tlb:"## 64"` CreatedAt uint32 `tlb:"## 32"` - StateInit *StateInit `tlb:"maybe either . ^"` + StateInit *StateInit `tlb:"maybe either leave 1,1 . ^"` Body *cell.Cell `tlb:"either . ^"` } @@ -55,7 +61,7 @@ type ExternalMessage struct { DstAddr *address.Address `tlb:"addr"` ImportFee Coins `tlb:"."` - StateInit *StateInit `tlb:"maybe either . ^"` + StateInit *StateInit `tlb:"maybe either leave 1,1 . ^"` Body *cell.Cell `tlb:"either . ^"` } @@ -66,7 +72,7 @@ type ExternalMessageOut struct { CreatedLT uint64 `tlb:"## 64"` CreatedAt uint32 `tlb:"## 32"` - StateInit *StateInit `tlb:"maybe either . ^"` + StateInit *StateInit `tlb:"maybe either leave 1,1 . ^"` Body *cell.Cell `tlb:"either . ^"` } @@ -181,92 +187,11 @@ func (m *Message) AsExternalOut() *ExternalMessageOut { return m.Msg.(*ExternalMessageOut) } -func (m *InternalMessage) ToCell() (*cell.Cell, error) { - b := cell.BeginCell() - b.MustStoreUInt(0, 1) // identification of int msg - b.MustStoreBoolBit(m.IHRDisabled) - b.MustStoreBoolBit(m.Bounce) - b.MustStoreBoolBit(m.Bounced) - b.MustStoreAddr(m.SrcAddr) - b.MustStoreAddr(m.DstAddr) - b.MustStoreBigCoins(m.Amount.Nano()) - - b.MustStoreDict(m.ExtraCurrencies) - - b.MustStoreBigCoins(m.IHRFee.Nano()) - b.MustStoreBigCoins(m.FwdFee.Nano()) - - b.MustStoreUInt(m.CreatedLT, 64) - b.MustStoreUInt(uint64(m.CreatedAt), 32) - b.MustStoreBoolBit(m.StateInit != nil) - if m.StateInit != nil { - stateCell, err := ToCell(m.StateInit) - if err != nil { - return nil, err - } - - if int(b.BitsLeft())-2 < int(stateCell.BitsSize()) || int(b.RefsLeft())-1 < int(m.Body.RefsNum()) { - b.MustStoreBoolBit(true) - b.MustStoreRef(stateCell) - } else { - b.MustStoreBoolBit(false) - b.MustStoreBuilder(stateCell.ToBuilder()) - } - } - - if m.Body != nil { - if int(b.BitsLeft())-1 < int(m.Body.BitsSize()) || b.RefsLeft() < m.Body.RefsNum() { - b.MustStoreBoolBit(true) - b.MustStoreRef(m.Body) - } else { - b.MustStoreBoolBit(false) - b.MustStoreBuilder(m.Body.ToBuilder()) - } - } else { - b.MustStoreBoolBit(false) - } - - return b.EndCell(), nil -} - func (m *InternalMessage) Dump() string { return fmt.Sprintf("Amount %s TON, Created at: %d, Created lt %d\nBounce: %t, Bounced %t, IHRDisabled %t\nSrcAddr: %s\nDstAddr: %s\nPayload: %s", m.Amount.String(), m.CreatedAt, m.CreatedLT, m.Bounce, m.Bounced, m.IHRDisabled, m.SrcAddr, m.DstAddr, m.Body.Dump()) } -func (m *ExternalMessage) ToCell() (*cell.Cell, error) { - builder := cell.BeginCell().MustStoreUInt(0b10, 2). - MustStoreAddr(m.SrcAddr). - MustStoreAddr(m.DstAddr). - MustStoreBigCoins(m.ImportFee.Nano()) - - builder.MustStoreBoolBit(m.StateInit != nil) // has state init - if m.StateInit != nil { - stateCell, err := ToCell(m.StateInit) - if err != nil { - return nil, fmt.Errorf("failed to serialize state init: %w", err) - } - - if int(builder.BitsLeft())-2 < int(stateCell.BitsSize()) || int(builder.RefsLeft())-1 < int(m.Body.RefsNum()) { - builder.MustStoreBoolBit(true) // state as ref - builder.MustStoreRef(stateCell) - } else { - builder.MustStoreBoolBit(false) // state as slice - builder.MustStoreBuilder(stateCell.ToBuilder()) - } - } - - if int(builder.BitsLeft())-1 < int(m.Body.BitsSize()) || builder.RefsLeft() < m.Body.RefsNum() { - builder.MustStoreBoolBit(true) // body as ref - builder.MustStoreRef(m.Body) - } else { - builder.MustStoreBoolBit(false) // body as slice - builder.MustStoreBuilder(m.Body.ToBuilder()) - } - - return builder.EndCell(), nil -} - func (m *MessagesList) ToSlice() ([]Message, error) { if m.List == nil { return nil, nil diff --git a/tlb/message_test.go b/tlb/message_test.go index ed98f099..93354c21 100644 --- a/tlb/message_test.go +++ b/tlb/message_test.go @@ -28,7 +28,7 @@ func TestInternalMessage_ToCell(t *testing.T) { // need to deploy contract on te Body: cell.BeginCell().EndCell(), } - c, err := intMsg.ToCell() + c, err := ToCell(intMsg) if err != nil { t.Fatal("to cell err", err) } @@ -90,7 +90,7 @@ func TestMessage_LoadFromCell(t *testing.T) { StateInit: nil, Body: cell.BeginCell().MustStoreUInt(777, 27).EndCell(), } - _cell, err := tIntMsg.ToCell() + _cell, err := ToCell(tIntMsg) if err != nil { t.Fatal(err) } @@ -112,7 +112,7 @@ func TestMessage_LoadFromCell(t *testing.T) { StateInit: nil, Body: cell.BeginCell().MustStoreUInt(777, 27).EndCell(), } - _cell, err := tExMsg.ToCell() + _cell, err := ToCell(tExMsg) if err != nil { t.Fatal(err) } diff --git a/ton/sendmessage.go b/ton/sendmessage.go index 3b18ae2a..0cb3c694 100644 --- a/ton/sendmessage.go +++ b/ton/sendmessage.go @@ -26,7 +26,7 @@ var ErrMessageNotAccepted = errors.New("message was not accepted by the contract var ErrNoTransactionsWereFound = errors.New("no transactions were found") func (c *APIClient) SendExternalMessage(ctx context.Context, msg *tlb.ExternalMessage) error { - req, err := msg.ToCell() + req, err := tlb.ToCell(msg) if err != nil { return fmt.Errorf("failed to serialize external message, err: %w", err) } diff --git a/ton/wallet/highloadv2r2.go b/ton/wallet/highloadv2r2.go index c1e5b9b3..4078d6cd 100644 --- a/ton/wallet/highloadv2r2.go +++ b/ton/wallet/highloadv2r2.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/xssnick/tonutils-go/tlb" "math/big" "time" @@ -29,7 +30,7 @@ func (s *SpecHighloadV2R2) BuildMessage(_ context.Context, messages []*Message) dict := cell.NewDict(16) for i, message := range messages { - msg, err := message.InternalMessage.ToCell() + msg, err := tlb.ToCell(message.InternalMessage) if err != nil { return nil, fmt.Errorf("failed to convert msg to cell: %w", err) } diff --git a/ton/wallet/v3.go b/ton/wallet/v3.go index b1b7f0d6..3d800a2c 100644 --- a/ton/wallet/v3.go +++ b/ton/wallet/v3.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/xssnick/tonutils-go/tlb" "github.com/xssnick/tonutils-go/ton" "time" @@ -50,7 +51,7 @@ func (s *SpecV3) BuildMessage(ctx context.Context, isInitialized bool, block *to MustStoreUInt(seq, 32) for i, message := range messages { - intMsg, err := message.InternalMessage.ToCell() + intMsg, err := tlb.ToCell(message.InternalMessage) if err != nil { return nil, fmt.Errorf("failed to convert internal message %d to cell: %w", i, err) } diff --git a/ton/wallet/v4r2.go b/ton/wallet/v4r2.go index edecfcec..05288d0c 100644 --- a/ton/wallet/v4r2.go +++ b/ton/wallet/v4r2.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/xssnick/tonutils-go/tlb" "github.com/xssnick/tonutils-go/ton" "time" @@ -51,7 +52,7 @@ func (s *SpecV4R2) BuildMessage(ctx context.Context, isInitialized bool, block * MustStoreInt(0, 8) // op for i, message := range messages { - intMsg, err := message.InternalMessage.ToCell() + intMsg, err := tlb.ToCell(message.InternalMessage) if err != nil { return nil, fmt.Errorf("failed to convert internal message %d to cell: %w", i, err) } diff --git a/ton/wallet/wallet_test.go b/ton/wallet/wallet_test.go index 69739108..8da44655 100644 --- a/ton/wallet/wallet_test.go +++ b/ton/wallet/wallet_test.go @@ -348,7 +348,7 @@ func checkV4R2(t *testing.T, p *cell.Slice, w *Wallet, flow int, intMsg *tlb.Int t.Fatal("mode incorrect") } - intMsgRef, _ := intMsg.ToCell() + intMsgRef, _ := tlb.ToCell(intMsg) payload := cell.BeginCell().MustStoreUInt(DefaultSubwallet, 32). MustStoreUInt(exp, 32). MustStoreUInt(seq, 32) @@ -392,7 +392,7 @@ func checkV3(t *testing.T, p *cell.Slice, w *Wallet, flow int, intMsg *tlb.Inter t.Fatal("mode incorrect") } - intMsgRef, _ := intMsg.ToCell() + intMsgRef, _ := tlb.ToCell(intMsg) payload := cell.BeginCell().MustStoreUInt(DefaultSubwallet, 32). MustStoreUInt(exp, 32). MustStoreUInt(seq, 32) @@ -426,7 +426,7 @@ func checkHighloadV2R2(t *testing.T, p *cell.Slice, w *Wallet, intMsg *tlb.Inter t.Fatal("dict incorrect") } - intMsgRef, _ := intMsg.ToCell() + intMsgRef, _ := tlb.ToCell(intMsg) dict := cell.NewDict(16) err := dict.SetIntKey(big.NewInt(0), cell.BeginCell(). From df992fc6665ba02990b4a6df70d77a10257c541b Mon Sep 17 00:00:00 2001 From: Andrey Pfau Date: Wed, 29 Nov 2023 02:44:23 +0700 Subject: [PATCH 17/23] Octet AddMul optimization --- adnl/rldp/raptorq/discmath/gf256.go | 4 +-- adnl/rldp/raptorq/discmath/oct.go | 42 ++++++++++++++--------------- adnl/rldp/raptorq/solver_test.go | 2 +- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/adnl/rldp/raptorq/discmath/gf256.go b/adnl/rldp/raptorq/discmath/gf256.go index 4369c229..c1ee38dd 100644 --- a/adnl/rldp/raptorq/discmath/gf256.go +++ b/adnl/rldp/raptorq/discmath/gf256.go @@ -17,13 +17,13 @@ func (g *GF256) Add(g2 *GF256) { func (g *GF256) Mul(x uint8) { for i := 0; i < len(g.data); i++ { - g.data[i] = octMul(g.data[i], x) + g.data[i] = OctMul(g.data[i], x) } } func (g *GF256) AddMul(g2 *GF256, x uint8) { for i := 0; i < len(g.data); i++ { - g.data[i] = octAdd(g.data[i], octMul(x, g2.data[i])) + g.data[i] ^= OctAddMul(x, g2.data[i]) } } diff --git a/adnl/rldp/raptorq/discmath/oct.go b/adnl/rldp/raptorq/discmath/oct.go index 79dd0620..da6c2921 100644 --- a/adnl/rldp/raptorq/discmath/oct.go +++ b/adnl/rldp/raptorq/discmath/oct.go @@ -1,6 +1,6 @@ package discmath -var _LogPreCalc = [...]uint8{0, 1, 25, 2, 50, 26, 198, 3, 223, 51, 238, 27, 104, 199, 75, 4, 100, 224, 14, 52, 141, 239, +var _LogPreCalc = [...]uint8{0, 0, 1, 25, 2, 50, 26, 198, 3, 223, 51, 238, 27, 104, 199, 75, 4, 100, 224, 14, 52, 141, 239, 129, 28, 193, 105, 248, 200, 8, 76, 113, 5, 138, 101, 47, 225, 36, 15, 33, 53, 147, 142, 218, 240, 18, 130, 69, 29, 181, 194, 125, 106, 39, 249, 185, 201, 154, 9, 120, 77, 228, 114, 166, 6, 191, 139, 98, 102, 221, 48, 253, 226, 152, 37, 179, 16, 145, 34, 136, 54, 208, 148, 206, 143, 150, 219, 189, 241, @@ -38,8 +38,22 @@ var _ExpPreCalc = [...]uint8{1, 2, 4, 8, 16, 32, 64, 128, 29, 58, 116, 232, 205, 122, 244, 245, 247, 243, 251, 235, 203, 139, 11, 22, 44, 88, 176, 125, 250, 233, 207, 131, 27, 54, 108, 216, 173, 71, 142} -func OctLog(x uint8) uint8 { - return _LogPreCalc[x] +var _MulPreCalc = calcOctMulTable() + +func calcOctMulTable() [256][256]uint8 { + var result [256][256]uint8 + + for i := 1; i < 256; i++ { + for j := 1; j < 256; j++ { + result[i][j] = OctMul(uint8(i), uint8(j)) + } + } + + return result +} + +func OctAddMul(x, y uint8) uint8 { + return _MulPreCalc[x][y] } func OctExp(x uint32) uint8 { @@ -47,29 +61,13 @@ func OctExp(x uint32) uint8 { } func OctInverse(x uint8) uint8 { - return OctExp(uint32(255 - OctLog(x-1))) + return OctExp(uint32(255 - _LogPreCalc[x])) } -func OctDiv(x, y uint8) uint8 { - if x == 0 || y == 0 { - return x - } - - return OctExp(uint32(OctLog(x-1)) - uint32(OctLog(y-1)+255)) -} - -func octMul(x, y uint8) uint8 { +func OctMul(x, y uint8) uint8 { if x == 0 || y == 0 { return 0 } - return OctExp(uint32(OctLog(x-1)) + uint32(OctLog(y-1))) -} - -func octAdd(x, y uint8) uint8 { - return x ^ y -} - -func OctSub(x, y uint8) uint8 { - return octAdd(x, y) + return _ExpPreCalc[uint32(_LogPreCalc[x])+uint32(_LogPreCalc[y])] } diff --git a/adnl/rldp/raptorq/solver_test.go b/adnl/rldp/raptorq/solver_test.go index e287b1fa..48502bc3 100644 --- a/adnl/rldp/raptorq/solver_test.go +++ b/adnl/rldp/raptorq/solver_test.go @@ -109,7 +109,7 @@ func Test_EncodeDecodeFuzz(t *testing.T) { func Benchmark_EncodeDecodeFuzz(b *testing.B) { str := make([]byte, 4096) rand.Read(str) - for n := 0; n < 100; n++ { + for n := 0; n < 1000; n++ { var symSz uint32 = 768 r := NewRaptorQ(symSz) enc, err := r.CreateEncoder(str) From 314105416103f663201f90a56c553473a55b2998 Mon Sep 17 00:00:00 2001 From: Andrey Pfau Date: Wed, 29 Nov 2023 12:23:09 +0700 Subject: [PATCH 18/23] Heavy add/mul/addmul optimizations --- adnl/rldp/raptorq/discmath/gf256.go | 12 ++----- adnl/rldp/raptorq/discmath/oct.go | 52 ++++++++++++++++++++++++----- 2 files changed, 46 insertions(+), 18 deletions(-) diff --git a/adnl/rldp/raptorq/discmath/gf256.go b/adnl/rldp/raptorq/discmath/gf256.go index c1ee38dd..53205987 100644 --- a/adnl/rldp/raptorq/discmath/gf256.go +++ b/adnl/rldp/raptorq/discmath/gf256.go @@ -10,21 +10,15 @@ type GF256 struct { } func (g *GF256) Add(g2 *GF256) { - for i := 0; i < len(g.data); i++ { - g.data[i] ^= g2.data[i] - } + OctVecAdd(g.data, g2.data) } func (g *GF256) Mul(x uint8) { - for i := 0; i < len(g.data); i++ { - g.data[i] = OctMul(g.data[i], x) - } + OctVecMul(g.data, x) } func (g *GF256) AddMul(g2 *GF256, x uint8) { - for i := 0; i < len(g.data); i++ { - g.data[i] ^= OctAddMul(x, g2.data[i]) - } + OctVecMulAdd(g.data, g2.data, x) } func (g *GF256) Bytes() []byte { diff --git a/adnl/rldp/raptorq/discmath/oct.go b/adnl/rldp/raptorq/discmath/oct.go index da6c2921..6cf1d81c 100644 --- a/adnl/rldp/raptorq/discmath/oct.go +++ b/adnl/rldp/raptorq/discmath/oct.go @@ -1,5 +1,7 @@ package discmath +import "unsafe" + var _LogPreCalc = [...]uint8{0, 0, 1, 25, 2, 50, 26, 198, 3, 223, 51, 238, 27, 104, 199, 75, 4, 100, 224, 14, 52, 141, 239, 129, 28, 193, 105, 248, 200, 8, 76, 113, 5, 138, 101, 47, 225, 36, 15, 33, 53, 147, 142, 218, 240, 18, 130, 69, 29, 181, 194, 125, 106, 39, 249, 185, 201, 154, 9, 120, 77, 228, 114, 166, 6, 191, 139, @@ -45,17 +47,13 @@ func calcOctMulTable() [256][256]uint8 { for i := 1; i < 256; i++ { for j := 1; j < 256; j++ { - result[i][j] = OctMul(uint8(i), uint8(j)) + result[i][j] = _ExpPreCalc[uint32(_LogPreCalc[i])+uint32(_LogPreCalc[j])] } } return result } -func OctAddMul(x, y uint8) uint8 { - return _MulPreCalc[x][y] -} - func OctExp(x uint32) uint8 { return _ExpPreCalc[x] } @@ -64,10 +62,46 @@ func OctInverse(x uint8) uint8 { return OctExp(uint32(255 - _LogPreCalc[x])) } -func OctMul(x, y uint8) uint8 { - if x == 0 || y == 0 { - return 0 +func OctVecAdd(x, y []byte) { + xUint64 := *(*[]uint64)(unsafe.Pointer(&x)) + yUint64 := *(*[]uint64)(unsafe.Pointer(&y)) + + for i := 0; i < len(x)/8; i++ { + xUint64[i] ^= yUint64[i] + } + + for i := len(x) - len(x)%8; i < len(x); i++ { + x[i] ^= y[i] + } +} + +func OctVecMul(vector []byte, multiplier uint8) { + table := _MulPreCalc[multiplier] + for i := 0; i < len(vector); i++ { + vector[i] = table[vector[i]] } +} + +func OctVecMulAdd(x, y []byte, multiplier uint8) { + table := _MulPreCalc[multiplier] + xUint64 := *(*[]uint64)(unsafe.Pointer(&x)) + pos := 0 + for i := 0; i < len(x)/8; i++ { + var prod uint64 + prod |= uint64(table[y[pos]]) + prod |= uint64(table[y[pos+1]]) << 8 + prod |= uint64(table[y[pos+2]]) << 16 + prod |= uint64(table[y[pos+3]]) << 24 + prod |= uint64(table[y[pos+4]]) << 32 + prod |= uint64(table[y[pos+5]]) << 40 + prod |= uint64(table[y[pos+6]]) << 48 + prod |= uint64(table[y[pos+7]]) << 56 - return _ExpPreCalc[uint32(_LogPreCalc[x])+uint32(_LogPreCalc[y])] + pos += 8 + xUint64[i] ^= prod + } + + for i := len(x) - len(x)%8; i < len(x); i++ { + x[i] ^= table[y[i]] + } } From 17dded5399208a8d5a7093064cc7fd4d710fd874 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Fri, 1 Dec 2023 14:33:08 +0400 Subject: [PATCH 19/23] DHT weight logic changed to distance --- adnl/dht/node.go | 8 ++++---- adnl/dht/node_test.go | 44 +++++++++++++++++++++++++++++++++++++++++-- adnl/dht/priority.go | 8 ++++---- 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/adnl/dht/node.go b/adnl/dht/node.go index d69c34be..364caa61 100644 --- a/adnl/dht/node.go +++ b/adnl/dht/node.go @@ -250,19 +250,19 @@ func (n *dhtNode) id() string { return hex.EncodeToString(n.adnlId) } -func (n *dhtNode) weight(id []byte) int { +func (n *dhtNode) distance(id []byte) int { n.mx.Lock() defer n.mx.Unlock() - w := leadingZeroBits(xor(id, n.adnlId)) + w := 256 - leadingZeroBits(xor(id, n.adnlId)) if n.currentState == _StateFail { - w -= 3 // less priority for failed + w += 16 // less priority for failed if w < 0 { w = 0 } } ping := time.Duration(atomic.LoadInt64(&n.ping)) - return (1 << 30) + ((w << 20) - int(ping/(time.Millisecond*5))) + return (w << 20) + int(ping/(time.Millisecond*5)) } func xor(a, b []byte) []byte { diff --git a/adnl/dht/node_test.go b/adnl/dht/node_test.go index 0c0a9570..d5b2eae6 100644 --- a/adnl/dht/node_test.go +++ b/adnl/dht/node_test.go @@ -477,8 +477,8 @@ func TestNode_weight(t *testing.T) { {tNode3, []byte{0b00100100, 0b10100100, 0b00100101}, 1<<30 + 0}, } for _, test := range tests { - t.Run("weight test", func(t *testing.T) { - res := test.testNode.weight(test.testId) + t.Run("distance test", func(t *testing.T) { + res := test.testNode.distance(test.testId) if res != test.want { t.Errorf("got '%d', want '%d'", res, test.want) } @@ -486,6 +486,46 @@ func TestNode_weight(t *testing.T) { } } +func TestNode_weight2(t *testing.T) { + key, err := hex.DecodeString("75b9507dc58a931ea6e860d444987e82d8501e09191264c35b95f6952d8debe4") + if err != nil { + t.Fatal("failed to prepare test public key, err: ", err) + } + + tPubKey, err := hex.DecodeString("75b9507dc58a931ea6e860d444987e82d8501e09191264c35b95f6956d8debe4") + if err != nil { + t.Fatal("failed to prepare test public key, err: ", err) + } + + kId, err := tl.Hash(adnl.PublicKeyED25519{Key: tPubKey}) + if err != nil { + t.Fatal("failed to prepare test key id, err: ", err) + } + tNode1 := &dhtNode{ + adnlId: kId, + ping: 100000, + addr: net.IPv4(1, 2, 3, 4).To4().String() + ":" + "35465", + serverKey: tPubKey, + currentState: _StateActive, + } + tNode2 := &dhtNode{ + adnlId: kId, + ping: 5000000, + addr: net.IPv4(1, 2, 3, 4).To4().String() + ":" + "35465", + serverKey: tPubKey, + currentState: _StateActive, + } + tNode3 := &dhtNode{ + adnlId: kId, + ping: 100000, + addr: net.IPv4(1, 2, 3, 4).To4().String() + ":" + "35465", + serverKey: tPubKey, + currentState: _StateFail, + } + + println(tNode1.distance(key), tNode2.distance(key), tNode3.distance(key)) +} + func TestNode_xor(t *testing.T) { tests := []struct { give1 []byte diff --git a/adnl/dht/priority.go b/adnl/dht/priority.go index 2867a59f..4536ed18 100644 --- a/adnl/dht/priority.go +++ b/adnl/dht/priority.go @@ -7,7 +7,7 @@ import ( type nodePriority struct { id string node *dhtNode - priority int + distance int next *nodePriority used bool } @@ -33,7 +33,7 @@ func (p *priorityList) addNode(node *dhtNode) bool { item := &nodePriority{ id: id, node: node, - priority: node.weight(p.targetId), + distance: node.distance(p.targetId), } p.mx.Lock() @@ -58,7 +58,7 @@ func (p *priorityList) addNode(node *dhtNode) bool { return false } - if item.priority > cur.priority { + if item.distance < cur.distance { item.next = cur if prev != nil { prev.next = item @@ -109,7 +109,7 @@ func (p *priorityList) getNode() (*dhtNode, int) { } res.used = true - return res.node, res.priority + return res.node, res.distance } func (p *priorityList) markUsed(node *dhtNode, used bool) { From 8c9f56ea3be2d47236007aaa4786fdee64f4607e Mon Sep 17 00:00:00 2001 From: Tikhon Slasten Date: Sat, 2 Dec 2023 16:44:02 +0800 Subject: [PATCH 20/23] Fix address unmarshal JSON --- address/addr.go | 25 +++++++++++++++++++++---- address/addr_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/address/addr.go b/address/addr.go index d79b9c20..c342d335 100644 --- a/address/addr.go +++ b/address/addr.go @@ -134,10 +134,27 @@ func (a *Address) UnmarshalJSON(data []byte) error { } data = data[1 : len(data)-1] - - addr, err := ParseAddr(string(data)) - if err != nil { - return err + strData := string(data) + + var ( + addr *Address + err error + ) + + switch strData { + case "NONE": + addr = NewAddressNone() + case "EXT_ADDRESS": + addr = NewAddressExt(0, 0, nil) + case "VAR_ADDRESS": + addr = NewAddressVar(0, 0, 0, nil) + case "NOT_SUPPORTED": + return fmt.Errorf("not supported address") + default: + addr, err = ParseAddr(strData) + if err != nil { + return err + } } *a = *addr diff --git a/address/addr_test.go b/address/addr_test.go index 3e196e16..e6a6f702 100644 --- a/address/addr_test.go +++ b/address/addr_test.go @@ -554,6 +554,38 @@ func TestAddress_UnmarshalJSON(t *testing.T) { want: Address{}, wantErr: true, }, + { + name: "none address", + address: "\"NONE\"", + want: Address{ + addrType: NoneAddress, + }, + wantErr: false, + }, + { + name: "ext address", + address: "\"EXT_ADDRESS\"", + want: Address{ + flags: flags{bounceable: true, testnet: false}, + addrType: ExtAddress, + workchain: 0, + bitsLen: 0, + data: nil, + }, + wantErr: false, + }, + { + name: "var address", + address: "\"VAR_ADDRESS\"", + want: Address{ + flags: flags{bounceable: true, testnet: false}, + addrType: VarAddress, + workchain: 0, + bitsLen: 0, + data: nil, + }, + wantErr: false, + }, } for _, tt := range tests { From 39872ea7b254875ea36627d79733c8fde45e4362 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Mon, 4 Dec 2023 15:47:49 +0400 Subject: [PATCH 21/23] Fixed no active connections panic in liteclient --- liteclient/pool.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/liteclient/pool.go b/liteclient/pool.go index 9312bb6a..b08ab280 100644 --- a/liteclient/pool.go +++ b/liteclient/pool.go @@ -250,6 +250,10 @@ func (c *ConnectionPool) queryWithSmartBalancer(req *ADNLRequest) (*connection, } c.nodesMx.RUnlock() + if reqNode == nil { + return nil, ErrNoActiveConnections + } + atomic.AddInt64(&reqNode.weight, -1) _, err := reqNode.queryAdnl(req.QueryID, req.Data) From 418f05923b8cc406dc81605f7ea77acdf68bc64e Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Mon, 4 Dec 2023 23:48:53 +0400 Subject: [PATCH 22/23] Cell index serialization improved to visit only unique cells --- tvm/cell/cell_test.go | 4 ++++ tvm/cell/flattenIndex.go | 31 ++++++++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/tvm/cell/cell_test.go b/tvm/cell/cell_test.go index bd77b499..4a8448f7 100644 --- a/tvm/cell/cell_test.go +++ b/tvm/cell/cell_test.go @@ -212,6 +212,10 @@ func TestBOCBomb(t *testing.T) { if !bytes.Equal(c.Hash(), hash) { t.Fatal("incorrect hash", hex.EncodeToString(c.Hash()), hex.EncodeToString(hash)) } + + if len(c.ToBOCWithFlags(false)) != len(boc) { + t.Fatal("len", len(c.ToBOC()), len(boc)) + } } func TestCell_TxWithMerkleBody(t *testing.T) { diff --git a/tvm/cell/flattenIndex.go b/tvm/cell/flattenIndex.go index 9e4ae929..6f3abf27 100644 --- a/tvm/cell/flattenIndex.go +++ b/tvm/cell/flattenIndex.go @@ -16,13 +16,18 @@ func flattenIndex(cells []*Cell) ([]*idxItem, map[string]*idxItem) { for len(cells) > 0 { next := make([]*Cell, 0, len(cells)*4) for _, p := range cells { + hash := string(p.Hash()) + + if _, ok := index[hash]; ok { + continue + } + // move cell forward in boc, because behind reference is not allowed - index[string(p.Hash())] = &idxItem{ - index: idx, + index[hash] = &idxItem{ cell: p, + index: idx, } idx++ - next = append(next, p.refs...) } cells = next @@ -33,6 +38,26 @@ func flattenIndex(cells []*Cell) ([]*idxItem, map[string]*idxItem) { idxSlice = append(idxSlice, id) } + for verifyOrder := true; verifyOrder; { + verifyOrder = false + + for _, id := range idxSlice { + for _, ref := range id.cell.refs { + idRef := index[string(ref.Hash())] + + if idRef.index < id.index { + // if we found that ref index is behind parent, + // move ref index forward + idRef.index = idx + idx++ + + // we changed index, so we need to verify order again + verifyOrder = true + } + } + } + } + sort.Slice(idxSlice, func(i, j int) bool { return idxSlice[i].index < idxSlice[j].index }) From 9687272a32c9b566ec2200429abc266d5a074531 Mon Sep 17 00:00:00 2001 From: Tikhon Slasten Date: Tue, 5 Dec 2023 13:36:01 +0800 Subject: [PATCH 23/23] Marshal Ext and Var addresses to hex --- address/addr.go | 58 +++++++++++++++++----- address/addr_test.go | 114 +++++++++++++++++++++++++++++++++++++------ 2 files changed, 146 insertions(+), 26 deletions(-) diff --git a/address/addr.go b/address/addr.go index c342d335..bd114f66 100644 --- a/address/addr.go +++ b/address/addr.go @@ -3,6 +3,7 @@ package address import ( "encoding/base64" "encoding/binary" + "encoding/hex" "errors" "fmt" @@ -93,10 +94,22 @@ func (a *Address) String() string { binary.BigEndian.PutUint16(address[34:], crc16.Checksum(address[:34], crcTable)) return base64.RawURLEncoding.EncodeToString(address[:]) case ExtAddress: - // TODO support readable serialization - return "EXT_ADDRESS" + address := make([]byte, 1+4+len(a.data)) + + address[0] = a.FlagsToByte() + binary.BigEndian.PutUint32(address[1:], uint32(a.bitsLen)) + copy(address[5:], a.data) + + return fmt.Sprintf("EXT:%s", hex.EncodeToString(address)) case VarAddress: - return "VAR_ADDRESS" + address := make([]byte, 1+4+4+len(a.data)) + + address[0] = a.FlagsToByte() + binary.BigEndian.PutUint32(address[1:], uint32(a.workchain)) + binary.BigEndian.PutUint32(address[5:], uint32(a.bitsLen)) + copy(address[9:], a.data) + + return fmt.Sprintf("VAR:%s", hex.EncodeToString(address)) default: return "NOT_SUPPORTED" } @@ -141,16 +154,39 @@ func (a *Address) UnmarshalJSON(data []byte) error { err error ) - switch strData { - case "NONE": + if strData == "NONE" { addr = NewAddressNone() - case "EXT_ADDRESS": - addr = NewAddressExt(0, 0, nil) - case "VAR_ADDRESS": - addr = NewAddressVar(0, 0, 0, nil) - case "NOT_SUPPORTED": + } else if strData == "NOT_SUPPORTED" { return fmt.Errorf("not supported address") - default: + } else if len(strData) >= 9 && strData[:4] == "EXT:" { + strData = strData[4:] + + b, err := hex.DecodeString(strData) + if err != nil { + return err + } + + addr = NewAddressExt( + b[0], + uint(binary.BigEndian.Uint32(b[1:5])), + b[5:], + ) + + } else if len(strData) >= 13 && strData[:4] == "VAR:" { + strData = strData[4:] + + b, err := hex.DecodeString(strData) + if err != nil { + return err + } + + addr = NewAddressVar( + b[0], + int32(binary.BigEndian.Uint32(b[1:5])), + uint(binary.BigEndian.Uint32(b[5:9])), + b[9:], + ) + } else { addr, err = ParseAddr(strData) if err != nil { return err diff --git a/address/addr_test.go b/address/addr_test.go index e6a6f702..acbcd0d0 100644 --- a/address/addr_test.go +++ b/address/addr_test.go @@ -482,15 +482,63 @@ func TestAddress_MarshalJSON(t *testing.T) { wantErr: false, }, { - name: "ext address", - address: NewAddressExt(0, 256, []byte{}), - want: "\"EXT_ADDRESS\"", + name: "ext address", + address: &Address{ + flags: flags{ + bounceable: true, + testnet: false, + }, + addrType: ExtAddress, + workchain: 0, + bitsLen: 256, + data: []byte{1, 2, 3}, + }, + want: "\"EXT:1100000100010203\"", wantErr: false, }, { - name: "var address", - address: NewAddressVar(0, 0, 256, []byte{}), - want: "\"VAR_ADDRESS\"", + name: "ext address with empty data", + address: &Address{ + flags: flags{ + bounceable: true, + testnet: false, + }, + addrType: ExtAddress, + workchain: 0, + bitsLen: 256, + data: nil, + }, + want: "\"EXT:1100000100\"", + wantErr: false, + }, + { + name: "var address", + address: &Address{ + flags: flags{ + bounceable: true, + testnet: true, + }, + addrType: VarAddress, + workchain: -1, + bitsLen: 256, + data: []byte{4, 5, 6}, + }, + want: "\"VAR:91ffffffff00000100040506\"", + wantErr: false, + }, + { + name: "var address with empty data", + address: &Address{ + flags: flags{ + bounceable: true, + testnet: true, + }, + addrType: VarAddress, + workchain: -1, + bitsLen: 256, + data: nil, + }, + want: "\"VAR:91ffffffff00000100\"", wantErr: false, }, { @@ -564,25 +612,61 @@ func TestAddress_UnmarshalJSON(t *testing.T) { }, { name: "ext address", - address: "\"EXT_ADDRESS\"", + address: "\"EXT:1100000100010203\"", want: Address{ - flags: flags{bounceable: true, testnet: false}, + flags: flags{ + bounceable: true, + testnet: false, + }, addrType: ExtAddress, workchain: 0, - bitsLen: 0, - data: nil, + bitsLen: 256, + data: []byte{1, 2, 3}, + }, + wantErr: false, + }, + { + name: "ext address with empty data", + address: "\"EXT:1100000100\"", + want: Address{ + flags: flags{ + bounceable: true, + testnet: false, + }, + addrType: ExtAddress, + workchain: 0, + bitsLen: 256, + data: []byte{}, }, wantErr: false, }, { name: "var address", - address: "\"VAR_ADDRESS\"", + address: "\"VAR:91ffffffff00000100040506\"", want: Address{ - flags: flags{bounceable: true, testnet: false}, + flags: flags{ + bounceable: true, + testnet: true, + }, addrType: VarAddress, - workchain: 0, - bitsLen: 0, - data: nil, + workchain: -1, + bitsLen: 256, + data: []byte{4, 5, 6}, + }, + wantErr: false, + }, + { + name: "var address with empty data", + address: "\"VAR:91ffffffff00000100\"", + want: Address{ + flags: flags{ + bounceable: true, + testnet: true, + }, + addrType: VarAddress, + workchain: -1, + bitsLen: 256, + data: []byte{}, }, wantErr: false, },