From 250b2f31e652e2032b94ce2eda69997ad843f5c2 Mon Sep 17 00:00:00 2001 From: schimih Date: Mon, 14 Nov 2022 16:36:31 +0200 Subject: [PATCH 001/189] changed EncodeBytesSlice with EncodeSlice --- factory/wsIndexerFactory.go | 2 +- go.mod | 4 ++-- go.sum | 11 ++++++++++ mock/pubkeyConverterMock.go | 15 ++++++++++++-- mock/pubkeyConverterStub.go | 20 ++++++++++++++----- .../logsevents/delegatorsProcessor.go | 15 +++++++++----- .../logsevents/esdtIssueProcessor.go | 4 ++-- .../logsevents/esdtPropertiesProcessor.go | 4 ++-- .../logsevents/logsAndEventsProcessor.go | 9 +++++++-- .../elasticproc/logsevents/nftsProcessor.go | 4 +++- .../logsevents/nftsPropertiesProcessor.go | 2 +- process/elasticproc/logsevents/scDeploys.go | 6 ++++-- .../smartContractResultsProcessor.go | 15 ++++++++------ .../smartContractResultsProcessor_test.go | 8 ++++++-- .../transactions/transactionDBBuilder.go | 19 ++++++++++++------ .../transactions/transactionDBBuilder_test.go | 9 +++++++-- .../transactionsProcessor_test.go | 2 +- .../validators/validatorsProcessor.go | 2 +- 18 files changed, 108 insertions(+), 43 deletions(-) diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 06bb8c13..31637ccd 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -46,7 +46,7 @@ func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsin if err != nil { return nil, err } - addressPubkeyConverter, err := pubkeyConverter.NewBech32PubkeyConverter(cfg.Config.AddressConverter.Length, log) + addressPubkeyConverter, err := pubkeyConverter.NewBech32PubkeyConverter(cfg.Config.AddressConverter.Length, "erd") if err != nil { return nil, err } diff --git a/go.mod b/go.mod index f706054e..17d69578 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.24 + github.com/ElrondNetwork/elrond-go-core v1.1.25-0.20221114131354-7a10b0122311 github.com/ElrondNetwork/elrond-go-logger v1.0.8 - github.com/ElrondNetwork/elrond-vm-common v1.3.25 + github.com/ElrondNetwork/elrond-vm-common v1.3.27-0.20221114140858-157beda2e3f9 github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gorilla/websocket v1.5.0 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index 37c080af..17d43b39 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,25 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= +github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= +github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221108131533-15ef756cddb2/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= +github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221114101758-c732a00d744c h1:CNXlDtgASVLZZrSXL5NJGOOgnUE21TPWMS+2lHdShEE= +github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221114101758-c732a00d744c/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= github.com/ElrondNetwork/elrond-go-core v1.1.24 h1:jamx50C1dP50uwh11EXT0+4pq0E0G3YYTK2xKEewDEk= github.com/ElrondNetwork/elrond-go-core v1.1.24/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= +github.com/ElrondNetwork/elrond-go-core v1.1.25-0.20221114131354-7a10b0122311 h1:It+2XCbsj4JdmU1/AbRvnIwHOHrG1DDNEpgcVRkcqjk= +github.com/ElrondNetwork/elrond-go-core v1.1.25-0.20221114131354-7a10b0122311/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= github.com/ElrondNetwork/elrond-go-logger v1.0.8 h1:e8SkdCNI+eRq0jpsziwn063RGUuaHjQhO4Ayg5hiTuM= github.com/ElrondNetwork/elrond-go-logger v1.0.8/go.mod h1:86lz/Y9D1sMAZiukCYxCsETLl1zuI23qj37ct3KUT6A= github.com/ElrondNetwork/elrond-vm-common v1.1.0/go.mod h1:w3i6f8uiuRkE68Ie/gebRcLgTuHqvruJSYrFyZWuLrE= +github.com/ElrondNetwork/elrond-vm-common v1.3.24/go.mod h1:seROQuR7RJCoCS7mgRXVAlvjztltY1c+UroAgWr/USE= +github.com/ElrondNetwork/elrond-vm-common v1.3.25-0.20221114084819-36d6f8508136 h1:Of5h1gmYNHJzH9vIbidpSPgJ49c1EVXvdeNUd+/Bbtk= +github.com/ElrondNetwork/elrond-vm-common v1.3.25-0.20221114084819-36d6f8508136/go.mod h1:VfPG0Zwy+jQtaYOplYXJUctr88AWm2LduuAalY01t4I= github.com/ElrondNetwork/elrond-vm-common v1.3.25 h1:Q8VpD8vfgWS2odnwiTULz1beeUd6pLsSL3s+uKKV/3M= github.com/ElrondNetwork/elrond-vm-common v1.3.25/go.mod h1:PaQtmmGpxi3flbCAyvY6LdTVPe+I1JyfpZZp+EKGtUc= +github.com/ElrondNetwork/elrond-vm-common v1.3.27-0.20221114140858-157beda2e3f9 h1:G2dSsn6zLoZ0rLms7zq4VCRPj7EgwksLeGFGhGnZGjg= +github.com/ElrondNetwork/elrond-vm-common v1.3.27-0.20221114140858-157beda2e3f9/go.mod h1:VfPG0Zwy+jQtaYOplYXJUctr88AWm2LduuAalY01t4I= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= diff --git a/mock/pubkeyConverterMock.go b/mock/pubkeyConverterMock.go index 2495757f..34eb26ef 100644 --- a/mock/pubkeyConverterMock.go +++ b/mock/pubkeyConverterMock.go @@ -22,8 +22,19 @@ func (pcm *PubkeyConverterMock) Decode(humanReadable string) ([]byte, error) { } // Encode - -func (pcm *PubkeyConverterMock) Encode(pkBytes []byte) string { - return hex.EncodeToString(pkBytes) +func (pcm *PubkeyConverterMock) Encode(pkBytes []byte) (string, error) { + return hex.EncodeToString(pkBytes), nil +} + +// EncodeSlice - +func (pcm *PubkeyConverterMock) EncodeSlice(pkBytesSlice [][]byte) ([]string, error) { + decodedSlice := make([]string, 0) + + for _, pkBytes := range pkBytesSlice { + decodedSlice = append(decodedSlice, hex.EncodeToString(pkBytes)) + } + + return decodedSlice, nil } // Len - diff --git a/mock/pubkeyConverterStub.go b/mock/pubkeyConverterStub.go index f8ef5507..75b78c60 100644 --- a/mock/pubkeyConverterStub.go +++ b/mock/pubkeyConverterStub.go @@ -2,9 +2,10 @@ package mock // PubkeyConverterStub - type PubkeyConverterStub struct { - LenCalled func() int - DecodeCalled func(humanReadable string) ([]byte, error) - EncodeCalled func(pkBytes []byte) string + LenCalled func() int + DecodeCalled func(humanReadable string) ([]byte, error) + EncodeCalled func(pkBytes []byte) (string, error) + EncodeSliceCalled func(pkBytesSlice [][]byte) ([]string, error) } // Len - @@ -26,12 +27,21 @@ func (pcs *PubkeyConverterStub) Decode(humanReadable string) ([]byte, error) { } // Encode - -func (pcs *PubkeyConverterStub) Encode(pkBytes []byte) string { +func (pcs *PubkeyConverterStub) Encode(pkBytes []byte) (string, error) { if pcs.EncodeCalled != nil { return pcs.EncodeCalled(pkBytes) } - return "" + return "", nil +} + +// Encode - +func (pcs *PubkeyConverterStub) EncodeSlice(pkBytesSlice [][]byte) ([]string, error) { + if pcs.EncodeSliceCalled != nil { + return pcs.EncodeSliceCalled(pkBytesSlice) + } + + return make([]string, 0), nil } // IsInterfaceNil - diff --git a/process/elasticproc/logsevents/delegatorsProcessor.go b/process/elasticproc/logsevents/delegatorsProcessor.go index 61d4d3f3..9928a904 100644 --- a/process/elasticproc/logsevents/delegatorsProcessor.go +++ b/process/elasticproc/logsevents/delegatorsProcessor.go @@ -75,13 +75,15 @@ func (dp *delegatorsProc) processEvent(args *argsProcessEvent) argOutputProcessE // topics[4] = true - if the delegator was deleted in case of withdrawal OR the contract address in case of delegate operations from staking v3.5 (makeNewContractFromValidatorData, mergeValidatorToDelegationSameOwner or mergeValidatorToDelegationWithWhitelist) activeStake := big.NewInt(0).SetBytes(topics[1]) - contractAddr := dp.pubkeyConverter.Encode(args.logAddress) + contractAddr, _ := dp.pubkeyConverter.Encode(args.logAddress) if len(topics) >= minNumTopicsDelegators+1 && eventIdentifierStr == delegateFunc { - contractAddr = dp.pubkeyConverter.Encode(topics[4]) + contractAddr, _ = dp.pubkeyConverter.Encode(topics[4]) } + encodedAddr, _ := dp.pubkeyConverter.Encode(args.event.GetAddress()) + delegator := &data.Delegator{ - Address: dp.pubkeyConverter.Encode(args.event.GetAddress()), + Address: encodedAddr, Contract: contractAddr, ActiveStake: activeStake.String(), ActiveStakeNum: dp.balanceConverter.ComputeBalanceAsFloat(activeStake), @@ -113,9 +115,12 @@ func (dp *delegatorsProc) getDelegatorFromClaimRewardsEvent(args *argsProcessEve return nil } + encodedAddr, _ := dp.pubkeyConverter.Encode(args.event.GetAddress()) + encodedContractAddr, _ := dp.pubkeyConverter.Encode(args.logAddress) + return &data.Delegator{ - Address: dp.pubkeyConverter.Encode(args.event.GetAddress()), - Contract: dp.pubkeyConverter.Encode(args.logAddress), + Address: encodedAddr, + Contract: encodedContractAddr, ShouldDelete: shouldDelete, } } diff --git a/process/elasticproc/logsevents/esdtIssueProcessor.go b/process/elasticproc/logsevents/esdtIssueProcessor.go index 3542c962..13e5fc79 100644 --- a/process/elasticproc/logsevents/esdtIssueProcessor.go +++ b/process/elasticproc/logsevents/esdtIssueProcessor.go @@ -75,7 +75,7 @@ func (eip *esdtIssueProcessor) processEvent(args *argsProcessEvent) argOutputPro numDecimals = big.NewInt(0).SetBytes(topics[4]).Uint64() } - encodedAddr := eip.pubkeyConverter.Encode(args.event.GetAddress()) + encodedAddr, _ := eip.pubkeyConverter.Encode(args.event.GetAddress()) tokenInfo := &data.TokenInfo{ Token: string(topics[0]), @@ -96,7 +96,7 @@ func (eip *esdtIssueProcessor) processEvent(args *argsProcessEvent) argOutputPro } if identifierStr == transferOwnershipFunc && len(topics) >= numIssueLogTopics+1 { - newOwner := eip.pubkeyConverter.Encode(topics[4]) + newOwner, _ := eip.pubkeyConverter.Encode(topics[4]) tokenInfo.TransferOwnership = true tokenInfo.CurrentOwner = newOwner tokenInfo.OwnersHistory[0].Address = newOwner diff --git a/process/elasticproc/logsevents/esdtPropertiesProcessor.go b/process/elasticproc/logsevents/esdtPropertiesProcessor.go index 67db9f11..5647d1c4 100644 --- a/process/elasticproc/logsevents/esdtPropertiesProcessor.go +++ b/process/elasticproc/logsevents/esdtPropertiesProcessor.go @@ -68,7 +68,7 @@ func (epp *esdtPropertiesProc) processEvent(args *argsProcessEvent) argOutputPro } shouldAddRole := identifier == core.BuiltInFunctionSetESDTRole - addrBech := epp.pubKeyConverter.Encode(args.event.GetAddress()) + addrBech, _ := epp.pubKeyConverter.Encode(args.event.GetAddress()) for _, roleBytes := range rolesBytes { args.tokenRolesAndProperties.AddRole(string(topics[tokenTopicsIndex]), addrBech, string(roleBytes), shouldAddRole) } @@ -81,7 +81,7 @@ func (epp *esdtPropertiesProc) processEvent(args *argsProcessEvent) argOutputPro func (epp *esdtPropertiesProc) extractDataNFTCreateRoleTransfer(args *argsProcessEvent) argOutputProcessEvent { topics := args.event.GetTopics() - addrBech := epp.pubKeyConverter.Encode(args.event.GetAddress()) + addrBech, _ := epp.pubKeyConverter.Encode(args.event.GetAddress()) shouldAddCreateRole := bytesToBool(topics[3]) args.tokenRolesAndProperties.AddRole(string(topics[tokenTopicsIndex]), addrBech, core.ESDTRoleNFTCreate, shouldAddCreateRole) diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor.go b/process/elasticproc/logsevents/logsAndEventsProcessor.go index feb0d46e..e76ce9eb 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor.go @@ -198,10 +198,13 @@ func (lep *logsAndEventsProcessor) prepareLogsForDB( } events := logHandler.GetLogEvents() + + encodedAddr, _ := lep.pubKeyConverter.Encode(logHandler.GetAddress()) + logsDB := &data.Logs{ ID: encodedID, OriginalTxHash: originalTxHash, - Address: lep.pubKeyConverter.Encode(logHandler.GetAddress()), + Address: encodedAddr, Timestamp: time.Duration(timestamp), Events: make([]*data.Event, 0, len(events)), } @@ -211,8 +214,10 @@ func (lep *logsAndEventsProcessor) prepareLogsForDB( continue } + encodedAddress, _ := lep.pubKeyConverter.Encode(event.GetAddress()) + logsDB.Events = append(logsDB.Events, &data.Event{ - Address: lep.pubKeyConverter.Encode(event.GetAddress()), + Address: encodedAddress, Identifier: string(event.GetIdentifier()), Topics: event.GetTopics(), Data: event.GetData(), diff --git a/process/elasticproc/logsevents/nftsProcessor.go b/process/elasticproc/logsevents/nftsProcessor.go index 13bffc99..7ddb2dc5 100644 --- a/process/elasticproc/logsevents/nftsProcessor.go +++ b/process/elasticproc/logsevents/nftsProcessor.go @@ -159,10 +159,12 @@ func (np *nftsProcessor) convertMetaData(metaData *esdt.MetaData) *outport.Token return nil } + encodedCreatorAddr, _ := np.pubKeyConverter.Encode(metaData.Creator) + return &outport.TokenMetaData{ Nonce: metaData.Nonce, Name: string(metaData.Name), - Creator: np.pubKeyConverter.Encode(metaData.Creator), + Creator: encodedCreatorAddr, Royalties: metaData.Royalties, Hash: metaData.Hash, URIs: metaData.URIs, diff --git a/process/elasticproc/logsevents/nftsPropertiesProcessor.go b/process/elasticproc/logsevents/nftsPropertiesProcessor.go index eb001d9a..193db52b 100644 --- a/process/elasticproc/logsevents/nftsPropertiesProcessor.go +++ b/process/elasticproc/logsevents/nftsPropertiesProcessor.go @@ -44,7 +44,7 @@ func (npp *nftsPropertiesProc) processEvent(args *argsProcessEvent) argOutputPro } } - callerAddress := npp.pubKeyConverter.Encode(args.event.GetAddress()) + callerAddress, _ := npp.pubKeyConverter.Encode(args.event.GetAddress()) if callerAddress == "" { return argOutputProcessEvent{ processed: true, diff --git a/process/elasticproc/logsevents/scDeploys.go b/process/elasticproc/logsevents/scDeploys.go index 0d8cbb30..b5b774b4 100644 --- a/process/elasticproc/logsevents/scDeploys.go +++ b/process/elasticproc/logsevents/scDeploys.go @@ -34,10 +34,12 @@ func (sdp *scDeploysProcessor) processEvent(args *argsProcessEvent) argOutputPro } } - scAddress := sdp.pubKeyConverter.Encode(topics[0]) + scAddress, _ := sdp.pubKeyConverter.Encode(topics[0]) + creatorAddress, _ := sdp.pubKeyConverter.Encode(topics[1]) + args.scDeploys[scAddress] = &data.ScDeployInfo{ TxHash: args.txHashHexEncoded, - Creator: sdp.pubKeyConverter.Encode(topics[1]), + Creator: creatorAddress, Timestamp: args.timestamp, } diff --git a/process/elasticproc/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go index abbb2a0b..2d2faa12 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -14,7 +14,6 @@ import ( "github.com/ElrondNetwork/elrond-go-core/data/smartContractResult" "github.com/ElrondNetwork/elrond-go-core/hashing" "github.com/ElrondNetwork/elrond-go-core/marshal" - datafield "github.com/ElrondNetwork/elrond-vm-common/parsers/dataField" ) type smartContractResultsProcessor struct { @@ -126,7 +125,7 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( relayerAddr := "" if len(scr.RelayerAddr) > 0 { - relayerAddr = proc.pubKeyConverter.Encode(scr.RelayerAddr) + relayerAddr, _ = proc.pubKeyConverter.Encode(scr.RelayerAddr) } relayedValue := "" @@ -135,11 +134,15 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( } originalSenderAddr := "" if scr.OriginalSender != nil { - originalSenderAddr = proc.pubKeyConverter.Encode(scr.OriginalSender) + originalSenderAddr, _ = proc.pubKeyConverter.Encode(scr.OriginalSender) } res := proc.dataFieldParser.Parse(scr.Data, scr.SndAddr, scr.RcvAddr, numOfShards) + senderAddr, _ := proc.pubKeyConverter.Encode(scr.SndAddr) + receiverAddr, _ := proc.pubKeyConverter.Encode(scr.RcvAddr) + receiversAddr, _ := proc.pubKeyConverter.EncodeSlice(res.Receivers) + return &indexerData.ScResult{ Hash: hex.EncodeToString(scrHash), MBHash: hexEncodedMBHash, @@ -147,8 +150,8 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( GasLimit: scr.GasLimit, GasPrice: scr.GasPrice, Value: scr.Value.String(), - Sender: proc.pubKeyConverter.Encode(scr.SndAddr), - Receiver: proc.pubKeyConverter.Encode(scr.RcvAddr), + Sender: senderAddr, + Receiver: receiverAddr, RelayerAddr: relayerAddr, RelayedValue: relayedValue, Code: string(scr.Code), @@ -166,7 +169,7 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( Function: res.Function, ESDTValues: res.ESDTValues, Tokens: res.Tokens, - Receivers: datafield.EncodeBytesSlice(proc.pubKeyConverter.Encode, res.Receivers), + Receivers: receiversAddr, ReceiversShardIDs: res.ReceiversShardID, IsRelayed: res.IsRelayed, OriginalSender: originalSenderAddr, diff --git a/process/elasticproc/transactions/smartContractResultsProcessor_test.go b/process/elasticproc/transactions/smartContractResultsProcessor_test.go index 8a395050..f48c5d02 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor_test.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor_test.go @@ -49,6 +49,10 @@ func TestPrepareSmartContractResult(t *testing.T) { mbHash := []byte("hash") scRes := scrsProc.prepareSmartContractResult([]byte(scHash), mbHash, smartContractRes, header, 0, 1, big.NewInt(0), 0, 3) + senderAddr, err := pubKeyConverter.Encode(sndAddr) + require.Nil(t, err) + receiverAddr, _ := pubKeyConverter.Encode(rcvAddr) + require.Nil(t, err) expectedTx := &data.ScResult{ Nonce: nonce, Hash: hex.EncodeToString([]byte(scHash)), @@ -56,8 +60,8 @@ func TestPrepareSmartContractResult(t *testing.T) { MBHash: hex.EncodeToString(mbHash), Code: string(code), Data: make([]byte, 0), - Sender: pubKeyConverter.Encode(sndAddr), - Receiver: pubKeyConverter.Encode(rcvAddr), + Sender: senderAddr, + Receiver: receiverAddr, Value: "", CallType: "1", Timestamp: time.Duration(100), diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index 7e9cbeb9..8d20b560 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -13,7 +13,6 @@ import ( "github.com/ElrondNetwork/elrond-go-core/data/receipt" "github.com/ElrondNetwork/elrond-go-core/data/rewardTx" "github.com/ElrondNetwork/elrond-go-core/data/transaction" - datafield "github.com/ElrondNetwork/elrond-vm-common/parsers/dataField" ) type dbTransactionBuilder struct { @@ -46,14 +45,18 @@ func (dtb *dbTransactionBuilder) prepareTransaction( isScCall := core.IsSmartContractAddress(tx.RcvAddr) res := dtb.dataFieldParser.Parse(tx.Data, tx.SndAddr, tx.RcvAddr, numOfShards) + receiverAddr, _ := dtb.addressPubkeyConverter.Encode(tx.RcvAddr) + senderAddr, _ := dtb.addressPubkeyConverter.Encode(tx.SndAddr) + receiversAddr, _ := dtb.addressPubkeyConverter.EncodeSlice(res.Receivers) + return &data.Transaction{ Hash: hex.EncodeToString(txHash), MBHash: hex.EncodeToString(mbHash), Nonce: tx.Nonce, Round: header.GetRound(), Value: tx.Value.String(), - Receiver: dtb.addressPubkeyConverter.Encode(tx.RcvAddr), - Sender: dtb.addressPubkeyConverter.Encode(tx.SndAddr), + Receiver: receiverAddr, + Sender: senderAddr, ReceiverShard: mb.ReceiverShardID, SenderShard: mb.SenderShardID, GasPrice: tx.GasPrice, @@ -72,7 +75,7 @@ func (dtb *dbTransactionBuilder) prepareTransaction( Function: res.Function, ESDTValues: res.ESDTValues, Tokens: res.Tokens, - Receivers: datafield.EncodeBytesSlice(dtb.addressPubkeyConverter.Encode, res.Receivers), + Receivers: receiversAddr, ReceiversShardIDs: res.ReceiversShardID, IsRelayed: res.IsRelayed, Version: tx.Version, @@ -87,13 +90,15 @@ func (dtb *dbTransactionBuilder) prepareRewardTransaction( header coreData.HeaderHandler, txStatus string, ) *data.Transaction { + receiverAddr, _ := dtb.addressPubkeyConverter.Encode(rTx.RcvAddr) + return &data.Transaction{ Hash: hex.EncodeToString(txHash), MBHash: hex.EncodeToString(mbHash), Nonce: 0, Round: rTx.Round, Value: rTx.Value.String(), - Receiver: dtb.addressPubkeyConverter.Encode(rTx.RcvAddr), + Receiver: receiverAddr, Sender: fmt.Sprintf("%d", core.MetachainShardId), ReceiverShard: mb.ReceiverShardID, SenderShard: mb.SenderShardID, @@ -112,10 +117,12 @@ func (dtb *dbTransactionBuilder) prepareReceipt( rec *receipt.Receipt, header coreData.HeaderHandler, ) *data.Receipt { + senderAddr, _ := dtb.addressPubkeyConverter.Encode(rec.SndAddr) + return &data.Receipt{ Hash: hex.EncodeToString([]byte(recHash)), Value: rec.Value.String(), - Sender: dtb.addressPubkeyConverter.Encode(rec.SndAddr), + Sender: senderAddr, Data: string(rec.Data), TxHash: hex.EncodeToString(rec.TxHash), Timestamp: time.Duration(header.GetTimeStamp()), diff --git a/process/elasticproc/transactions/transactionDBBuilder_test.go b/process/elasticproc/transactions/transactionDBBuilder_test.go index 78448fba..b5c737cb 100644 --- a/process/elasticproc/transactions/transactionDBBuilder_test.go +++ b/process/elasticproc/transactions/transactionDBBuilder_test.go @@ -50,14 +50,19 @@ func TestGetMoveBalanceTransaction(t *testing.T) { SndUserName: []byte("snd"), } + senderAddr, err := cp.addressPubkeyConverter.Encode(tx.RcvAddr) + require.Nil(t, err) + receiverAddr, err := cp.addressPubkeyConverter.Encode(tx.SndAddr) + require.Nil(t, err) + expectedTx := &data.Transaction{ Hash: hex.EncodeToString(txHash), MBHash: hex.EncodeToString(mbHash), Nonce: tx.Nonce, Round: header.Round, Value: tx.Value.String(), - Receiver: cp.addressPubkeyConverter.Encode(tx.RcvAddr), - Sender: cp.addressPubkeyConverter.Encode(tx.SndAddr), + Receiver: senderAddr, + Sender: receiverAddr, ReceiverShard: mb.ReceiverShardID, SenderShard: mb.SenderShardID, GasPrice: gasPrice, diff --git a/process/elasticproc/transactions/transactionsProcessor_test.go b/process/elasticproc/transactions/transactionsProcessor_test.go index e32f0888..4e64b263 100644 --- a/process/elasticproc/transactions/transactionsProcessor_test.go +++ b/process/elasticproc/transactions/transactionsProcessor_test.go @@ -521,7 +521,7 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { t.Parallel() args := createMockArgsTxsDBProc() - pubKeyConv, _ := pubkeyConverter.NewBech32PubkeyConverter(32, log) + pubKeyConv, _ := pubkeyConverter.NewBech32PubkeyConverter(32, "erd") args.AddressPubkeyConverter = pubKeyConv txDbProc, _ := NewTransactionsProcessor(args) diff --git a/process/elasticproc/validators/validatorsProcessor.go b/process/elasticproc/validators/validatorsProcessor.go index 99cac30c..8b047d80 100644 --- a/process/elasticproc/validators/validatorsProcessor.go +++ b/process/elasticproc/validators/validatorsProcessor.go @@ -31,7 +31,7 @@ func (vp *validatorsProcessor) PrepareValidatorsPublicKeys(shardValidatorsPubKey } for _, validatorPk := range shardValidatorsPubKeys { - strValidatorPk := vp.validatorPubkeyConverter.Encode(validatorPk) + strValidatorPk, _ := vp.validatorPubkeyConverter.Encode(validatorPk) validatorsPubKeys.PublicKeys = append(validatorsPubKeys.PublicKeys, strValidatorPk) } From 95f07b9714ba6ef9d67dc9fe4f1237f6d21d898d Mon Sep 17 00:00:00 2001 From: schimih Date: Mon, 14 Nov 2022 16:53:40 +0200 Subject: [PATCH 002/189] update logger version --- go.mod | 2 +- go.sum | 12 ++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 17d69578..b18ffdbc 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/ElrondNetwork/elrond-go-core v1.1.25-0.20221114131354-7a10b0122311 - github.com/ElrondNetwork/elrond-go-logger v1.0.8 + github.com/ElrondNetwork/elrond-go-logger v1.0.9 github.com/ElrondNetwork/elrond-vm-common v1.3.27-0.20221114140858-157beda2e3f9 github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gorilla/websocket v1.5.0 diff --git a/go.sum b/go.sum index 17d43b39..d86b2f87 100644 --- a/go.sum +++ b/go.sum @@ -1,23 +1,15 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= -github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221108131533-15ef756cddb2/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= -github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221114101758-c732a00d744c h1:CNXlDtgASVLZZrSXL5NJGOOgnUE21TPWMS+2lHdShEE= -github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221114101758-c732a00d744c/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= -github.com/ElrondNetwork/elrond-go-core v1.1.24 h1:jamx50C1dP50uwh11EXT0+4pq0E0G3YYTK2xKEewDEk= -github.com/ElrondNetwork/elrond-go-core v1.1.24/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-core v1.1.25-0.20221114131354-7a10b0122311 h1:It+2XCbsj4JdmU1/AbRvnIwHOHrG1DDNEpgcVRkcqjk= github.com/ElrondNetwork/elrond-go-core v1.1.25-0.20221114131354-7a10b0122311/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= github.com/ElrondNetwork/elrond-go-logger v1.0.8 h1:e8SkdCNI+eRq0jpsziwn063RGUuaHjQhO4Ayg5hiTuM= github.com/ElrondNetwork/elrond-go-logger v1.0.8/go.mod h1:86lz/Y9D1sMAZiukCYxCsETLl1zuI23qj37ct3KUT6A= +github.com/ElrondNetwork/elrond-go-logger v1.0.9 h1:RByDnVHgSN4zy3bwmvQarsnCd/gDmZ+zwVI+b6i4Plc= +github.com/ElrondNetwork/elrond-go-logger v1.0.9/go.mod h1:86lz/Y9D1sMAZiukCYxCsETLl1zuI23qj37ct3KUT6A= github.com/ElrondNetwork/elrond-vm-common v1.1.0/go.mod h1:w3i6f8uiuRkE68Ie/gebRcLgTuHqvruJSYrFyZWuLrE= -github.com/ElrondNetwork/elrond-vm-common v1.3.24/go.mod h1:seROQuR7RJCoCS7mgRXVAlvjztltY1c+UroAgWr/USE= -github.com/ElrondNetwork/elrond-vm-common v1.3.25-0.20221114084819-36d6f8508136 h1:Of5h1gmYNHJzH9vIbidpSPgJ49c1EVXvdeNUd+/Bbtk= -github.com/ElrondNetwork/elrond-vm-common v1.3.25-0.20221114084819-36d6f8508136/go.mod h1:VfPG0Zwy+jQtaYOplYXJUctr88AWm2LduuAalY01t4I= -github.com/ElrondNetwork/elrond-vm-common v1.3.25 h1:Q8VpD8vfgWS2odnwiTULz1beeUd6pLsSL3s+uKKV/3M= -github.com/ElrondNetwork/elrond-vm-common v1.3.25/go.mod h1:PaQtmmGpxi3flbCAyvY6LdTVPe+I1JyfpZZp+EKGtUc= github.com/ElrondNetwork/elrond-vm-common v1.3.27-0.20221114140858-157beda2e3f9 h1:G2dSsn6zLoZ0rLms7zq4VCRPj7EgwksLeGFGhGnZGjg= github.com/ElrondNetwork/elrond-vm-common v1.3.27-0.20221114140858-157beda2e3f9/go.mod h1:VfPG0Zwy+jQtaYOplYXJUctr88AWm2LduuAalY01t4I= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= From 189032fd4b58034e452a1519baf93b3352f1e1fd Mon Sep 17 00:00:00 2001 From: schimih Date: Mon, 14 Nov 2022 17:01:10 +0200 Subject: [PATCH 003/189] update urfave/cli --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b18ffdbc..8521e6d9 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/stretchr/testify v1.7.0 github.com/tidwall/gjson v1.14.0 - github.com/urfave/cli v1.22.9 + github.com/urfave/cli v1.22.10 ) require ( diff --git a/go.sum b/go.sum index d86b2f87..ea0effb7 100644 --- a/go.sum +++ b/go.sum @@ -88,6 +88,8 @@ github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/urfave/cli v1.22.9 h1:cv3/KhXGBGjEXLC4bH0sLuJ9BewaAbpk5oyMOveu4pw= github.com/urfave/cli v1.22.9/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.10 h1:p8Fspmz3iTctJstry1PYS3HVdllxnEzTEsgIgtxTrCk= +github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= From fceb77dd9ea3cb1ee5ce4b42b1ef717d6ce3b89d Mon Sep 17 00:00:00 2001 From: schimih Date: Thu, 17 Nov 2022 14:54:51 +0200 Subject: [PATCH 004/189] constant prefix --- factory/wsIndexerFactory.go | 3 ++- integrationtests/consts.go | 2 ++ integrationtests/utils.go | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 31637ccd..e6b42dd1 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -15,6 +15,7 @@ var log = logger.GetOrCreate("factory") const ( indexerCacheSize = 1 + addressPrefix = "erd" ) // CreateWsIndexer will create a new instance of wsindexer.WSClient @@ -46,7 +47,7 @@ func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsin if err != nil { return nil, err } - addressPubkeyConverter, err := pubkeyConverter.NewBech32PubkeyConverter(cfg.Config.AddressConverter.Length, "erd") + addressPubkeyConverter, err := pubkeyConverter.NewBech32PubkeyConverter(cfg.Config.AddressConverter.Length, addressPrefix) if err != nil { return nil, err } diff --git a/integrationtests/consts.go b/integrationtests/consts.go index 6995ff9d..8593fcf6 100644 --- a/integrationtests/consts.go +++ b/integrationtests/consts.go @@ -5,4 +5,6 @@ const ( testNumOfShards = 3 //nolint esURL = "http://localhost:9200" + //nolint + addressPrefix = "erd" ) diff --git a/integrationtests/utils.go b/integrationtests/utils.go index 8e2a9f94..0ea92d20 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -18,7 +18,7 @@ import ( var ( log = logger.GetOrCreate("integration-tests") - pubKeyConverter, _ = pubkeyConverter.NewBech32PubkeyConverter(32, "erd") + pubKeyConverter, _ = pubkeyConverter.NewBech32PubkeyConverter(32, addressPrefix) ) func setLogLevelDebug() { From 83be2cd7b6fc68984e11076aa3fdd015dc2513a9 Mon Sep 17 00:00:00 2001 From: schimih Date: Thu, 24 Nov 2022 22:21:44 +0200 Subject: [PATCH 005/189] refactoring using SilentEncode method --- factory/wsIndexerFactory.go | 3 --- go.mod | 2 +- go.sum | 4 ++-- mock/pubkeyConverterMock.go | 7 +++++++ mock/pubkeyConverterStub.go | 20 +++++++++++++++---- .../logsevents/delegatorsProcessor.go | 10 +++++----- .../logsevents/esdtIssueProcessor.go | 4 ++-- .../logsevents/esdtPropertiesProcessor.go | 4 ++-- .../logsevents/logsAndEventsProcessor.go | 4 ++-- .../elasticproc/logsevents/nftsProcessor.go | 2 +- .../logsevents/nftsPropertiesProcessor.go | 2 +- process/elasticproc/logsevents/scDeploys.go | 4 ++-- .../smartContractResultsProcessor.go | 8 ++++---- .../transactions/transactionDBBuilder.go | 8 ++++---- .../validators/validatorsProcessor.go | 1 + 15 files changed, 50 insertions(+), 33 deletions(-) diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index e6b42dd1..c039ea82 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -8,11 +8,8 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core/pubkeyConverter" factoryHasher "github.com/ElrondNetwork/elrond-go-core/hashing/factory" factoryMarshaller "github.com/ElrondNetwork/elrond-go-core/marshal/factory" - logger "github.com/ElrondNetwork/elrond-go-logger" ) -var log = logger.GetOrCreate("factory") - const ( indexerCacheSize = 1 addressPrefix = "erd" diff --git a/go.mod b/go.mod index 8521e6d9..8171f7b3 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.25-0.20221114131354-7a10b0122311 + github.com/ElrondNetwork/elrond-go-core v1.1.25-0.20221124191416-f1a624eb9164 github.com/ElrondNetwork/elrond-go-logger v1.0.9 github.com/ElrondNetwork/elrond-vm-common v1.3.27-0.20221114140858-157beda2e3f9 github.com/elastic/go-elasticsearch/v7 v7.12.0 diff --git a/go.sum b/go.sum index 397b4f62..29216793 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221108131533-15ef756cddb2/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= -github.com/ElrondNetwork/elrond-go-core v1.1.25-0.20221114131354-7a10b0122311 h1:It+2XCbsj4JdmU1/AbRvnIwHOHrG1DDNEpgcVRkcqjk= -github.com/ElrondNetwork/elrond-go-core v1.1.25-0.20221114131354-7a10b0122311/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= +github.com/ElrondNetwork/elrond-go-core v1.1.25-0.20221124191416-f1a624eb9164 h1:nAg0G4f0ZaiqrtvWKg8gJ372RZm35o6fLtlxz7ixtbQ= +github.com/ElrondNetwork/elrond-go-core v1.1.25-0.20221124191416-f1a624eb9164/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= github.com/ElrondNetwork/elrond-go-logger v1.0.9 h1:RByDnVHgSN4zy3bwmvQarsnCd/gDmZ+zwVI+b6i4Plc= diff --git a/mock/pubkeyConverterMock.go b/mock/pubkeyConverterMock.go index 34eb26ef..e4527046 100644 --- a/mock/pubkeyConverterMock.go +++ b/mock/pubkeyConverterMock.go @@ -2,6 +2,8 @@ package mock import ( "encoding/hex" + + "github.com/ElrondNetwork/elrond-go-core/core" ) // PubkeyConverterMock - @@ -37,6 +39,11 @@ func (pcm *PubkeyConverterMock) EncodeSlice(pkBytesSlice [][]byte) ([]string, er return decodedSlice, nil } +// SilentEncode - +func (pcm *PubkeyConverterMock) SilentEncode(pkBytes []byte, log core.Logger) string { + return hex.EncodeToString(pkBytes) +} + // Len - func (pcm *PubkeyConverterMock) Len() int { return pcm.len diff --git a/mock/pubkeyConverterStub.go b/mock/pubkeyConverterStub.go index 75b78c60..b5c39182 100644 --- a/mock/pubkeyConverterStub.go +++ b/mock/pubkeyConverterStub.go @@ -1,11 +1,14 @@ package mock +import "github.com/ElrondNetwork/elrond-go-core/core" + // PubkeyConverterStub - type PubkeyConverterStub struct { - LenCalled func() int - DecodeCalled func(humanReadable string) ([]byte, error) - EncodeCalled func(pkBytes []byte) (string, error) - EncodeSliceCalled func(pkBytesSlice [][]byte) ([]string, error) + LenCalled func() int + DecodeCalled func(humanReadable string) ([]byte, error) + EncodeCalled func(pkBytes []byte) (string, error) + EncodeSliceCalled func(pkBytesSlice [][]byte) ([]string, error) + SilentEncodeCalled func(pkBytesSlice []byte, log core.Logger) string } // Len - @@ -44,6 +47,15 @@ func (pcs *PubkeyConverterStub) EncodeSlice(pkBytesSlice [][]byte) ([]string, er return make([]string, 0), nil } +// SilentEncode - +func (pcs *PubkeyConverterStub) SilentEncode(pkBytes []byte, log core.Logger) string { + if pcs.EncodeCalled != nil { + return pcs.SilentEncodeCalled(pkBytes, log) + } + + return "" +} + // IsInterfaceNil - func (pcs *PubkeyConverterStub) IsInterfaceNil() bool { return pcs == nil diff --git a/process/elasticproc/logsevents/delegatorsProcessor.go b/process/elasticproc/logsevents/delegatorsProcessor.go index 9928a904..4870dd80 100644 --- a/process/elasticproc/logsevents/delegatorsProcessor.go +++ b/process/elasticproc/logsevents/delegatorsProcessor.go @@ -75,12 +75,12 @@ func (dp *delegatorsProc) processEvent(args *argsProcessEvent) argOutputProcessE // topics[4] = true - if the delegator was deleted in case of withdrawal OR the contract address in case of delegate operations from staking v3.5 (makeNewContractFromValidatorData, mergeValidatorToDelegationSameOwner or mergeValidatorToDelegationWithWhitelist) activeStake := big.NewInt(0).SetBytes(topics[1]) - contractAddr, _ := dp.pubkeyConverter.Encode(args.logAddress) + contractAddr := dp.pubkeyConverter.SilentEncode(args.logAddress, log) if len(topics) >= minNumTopicsDelegators+1 && eventIdentifierStr == delegateFunc { - contractAddr, _ = dp.pubkeyConverter.Encode(topics[4]) + contractAddr = dp.pubkeyConverter.SilentEncode(topics[4], log) } - encodedAddr, _ := dp.pubkeyConverter.Encode(args.event.GetAddress()) + encodedAddr := dp.pubkeyConverter.SilentEncode(args.event.GetAddress(), log) delegator := &data.Delegator{ Address: encodedAddr, @@ -115,8 +115,8 @@ func (dp *delegatorsProc) getDelegatorFromClaimRewardsEvent(args *argsProcessEve return nil } - encodedAddr, _ := dp.pubkeyConverter.Encode(args.event.GetAddress()) - encodedContractAddr, _ := dp.pubkeyConverter.Encode(args.logAddress) + encodedAddr := dp.pubkeyConverter.SilentEncode(args.event.GetAddress(), log) + encodedContractAddr := dp.pubkeyConverter.SilentEncode(args.logAddress, log) return &data.Delegator{ Address: encodedAddr, diff --git a/process/elasticproc/logsevents/esdtIssueProcessor.go b/process/elasticproc/logsevents/esdtIssueProcessor.go index 13e5fc79..6f5d017b 100644 --- a/process/elasticproc/logsevents/esdtIssueProcessor.go +++ b/process/elasticproc/logsevents/esdtIssueProcessor.go @@ -75,7 +75,7 @@ func (eip *esdtIssueProcessor) processEvent(args *argsProcessEvent) argOutputPro numDecimals = big.NewInt(0).SetBytes(topics[4]).Uint64() } - encodedAddr, _ := eip.pubkeyConverter.Encode(args.event.GetAddress()) + encodedAddr := eip.pubkeyConverter.SilentEncode(args.event.GetAddress(), log) tokenInfo := &data.TokenInfo{ Token: string(topics[0]), @@ -96,7 +96,7 @@ func (eip *esdtIssueProcessor) processEvent(args *argsProcessEvent) argOutputPro } if identifierStr == transferOwnershipFunc && len(topics) >= numIssueLogTopics+1 { - newOwner, _ := eip.pubkeyConverter.Encode(topics[4]) + newOwner := eip.pubkeyConverter.SilentEncode(topics[4], log) tokenInfo.TransferOwnership = true tokenInfo.CurrentOwner = newOwner tokenInfo.OwnersHistory[0].Address = newOwner diff --git a/process/elasticproc/logsevents/esdtPropertiesProcessor.go b/process/elasticproc/logsevents/esdtPropertiesProcessor.go index 5647d1c4..7e90cb8e 100644 --- a/process/elasticproc/logsevents/esdtPropertiesProcessor.go +++ b/process/elasticproc/logsevents/esdtPropertiesProcessor.go @@ -68,7 +68,7 @@ func (epp *esdtPropertiesProc) processEvent(args *argsProcessEvent) argOutputPro } shouldAddRole := identifier == core.BuiltInFunctionSetESDTRole - addrBech, _ := epp.pubKeyConverter.Encode(args.event.GetAddress()) + addrBech := epp.pubKeyConverter.SilentEncode(args.event.GetAddress(), log) for _, roleBytes := range rolesBytes { args.tokenRolesAndProperties.AddRole(string(topics[tokenTopicsIndex]), addrBech, string(roleBytes), shouldAddRole) } @@ -81,7 +81,7 @@ func (epp *esdtPropertiesProc) processEvent(args *argsProcessEvent) argOutputPro func (epp *esdtPropertiesProc) extractDataNFTCreateRoleTransfer(args *argsProcessEvent) argOutputProcessEvent { topics := args.event.GetTopics() - addrBech, _ := epp.pubKeyConverter.Encode(args.event.GetAddress()) + addrBech := epp.pubKeyConverter.SilentEncode(args.event.GetAddress(), log) shouldAddCreateRole := bytesToBool(topics[3]) args.tokenRolesAndProperties.AddRole(string(topics[tokenTopicsIndex]), addrBech, core.ESDTRoleNFTCreate, shouldAddCreateRole) diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor.go b/process/elasticproc/logsevents/logsAndEventsProcessor.go index e76ce9eb..89308776 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor.go @@ -199,7 +199,7 @@ func (lep *logsAndEventsProcessor) prepareLogsForDB( events := logHandler.GetLogEvents() - encodedAddr, _ := lep.pubKeyConverter.Encode(logHandler.GetAddress()) + encodedAddr := lep.pubKeyConverter.SilentEncode(logHandler.GetAddress(), log) logsDB := &data.Logs{ ID: encodedID, @@ -214,7 +214,7 @@ func (lep *logsAndEventsProcessor) prepareLogsForDB( continue } - encodedAddress, _ := lep.pubKeyConverter.Encode(event.GetAddress()) + encodedAddress := lep.pubKeyConverter.SilentEncode(event.GetAddress(), log) logsDB.Events = append(logsDB.Events, &data.Event{ Address: encodedAddress, diff --git a/process/elasticproc/logsevents/nftsProcessor.go b/process/elasticproc/logsevents/nftsProcessor.go index 7ddb2dc5..36626503 100644 --- a/process/elasticproc/logsevents/nftsProcessor.go +++ b/process/elasticproc/logsevents/nftsProcessor.go @@ -159,7 +159,7 @@ func (np *nftsProcessor) convertMetaData(metaData *esdt.MetaData) *outport.Token return nil } - encodedCreatorAddr, _ := np.pubKeyConverter.Encode(metaData.Creator) + encodedCreatorAddr := np.pubKeyConverter.SilentEncode(metaData.Creator, log) return &outport.TokenMetaData{ Nonce: metaData.Nonce, diff --git a/process/elasticproc/logsevents/nftsPropertiesProcessor.go b/process/elasticproc/logsevents/nftsPropertiesProcessor.go index 193db52b..e07ef3a6 100644 --- a/process/elasticproc/logsevents/nftsPropertiesProcessor.go +++ b/process/elasticproc/logsevents/nftsPropertiesProcessor.go @@ -44,7 +44,7 @@ func (npp *nftsPropertiesProc) processEvent(args *argsProcessEvent) argOutputPro } } - callerAddress, _ := npp.pubKeyConverter.Encode(args.event.GetAddress()) + callerAddress := npp.pubKeyConverter.SilentEncode(args.event.GetAddress(), log) if callerAddress == "" { return argOutputProcessEvent{ processed: true, diff --git a/process/elasticproc/logsevents/scDeploys.go b/process/elasticproc/logsevents/scDeploys.go index b5b774b4..5989d435 100644 --- a/process/elasticproc/logsevents/scDeploys.go +++ b/process/elasticproc/logsevents/scDeploys.go @@ -34,8 +34,8 @@ func (sdp *scDeploysProcessor) processEvent(args *argsProcessEvent) argOutputPro } } - scAddress, _ := sdp.pubKeyConverter.Encode(topics[0]) - creatorAddress, _ := sdp.pubKeyConverter.Encode(topics[1]) + scAddress := sdp.pubKeyConverter.SilentEncode(topics[0], log) + creatorAddress := sdp.pubKeyConverter.SilentEncode(topics[1], log) args.scDeploys[scAddress] = &data.ScDeployInfo{ TxHash: args.txHashHexEncoded, diff --git a/process/elasticproc/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go index 2d2faa12..bca3a585 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -125,7 +125,7 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( relayerAddr := "" if len(scr.RelayerAddr) > 0 { - relayerAddr, _ = proc.pubKeyConverter.Encode(scr.RelayerAddr) + relayerAddr = proc.pubKeyConverter.SilentEncode(scr.RelayerAddr, log) } relayedValue := "" @@ -134,13 +134,13 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( } originalSenderAddr := "" if scr.OriginalSender != nil { - originalSenderAddr, _ = proc.pubKeyConverter.Encode(scr.OriginalSender) + originalSenderAddr = proc.pubKeyConverter.SilentEncode(scr.OriginalSender, log) } res := proc.dataFieldParser.Parse(scr.Data, scr.SndAddr, scr.RcvAddr, numOfShards) - senderAddr, _ := proc.pubKeyConverter.Encode(scr.SndAddr) - receiverAddr, _ := proc.pubKeyConverter.Encode(scr.RcvAddr) + senderAddr := proc.pubKeyConverter.SilentEncode(scr.SndAddr, log) + receiverAddr := proc.pubKeyConverter.SilentEncode(scr.RcvAddr, log) receiversAddr, _ := proc.pubKeyConverter.EncodeSlice(res.Receivers) return &indexerData.ScResult{ diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index 8d20b560..bb374bdf 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -45,8 +45,8 @@ func (dtb *dbTransactionBuilder) prepareTransaction( isScCall := core.IsSmartContractAddress(tx.RcvAddr) res := dtb.dataFieldParser.Parse(tx.Data, tx.SndAddr, tx.RcvAddr, numOfShards) - receiverAddr, _ := dtb.addressPubkeyConverter.Encode(tx.RcvAddr) - senderAddr, _ := dtb.addressPubkeyConverter.Encode(tx.SndAddr) + receiverAddr := dtb.addressPubkeyConverter.SilentEncode(tx.RcvAddr, log) + senderAddr := dtb.addressPubkeyConverter.SilentEncode(tx.SndAddr, log) receiversAddr, _ := dtb.addressPubkeyConverter.EncodeSlice(res.Receivers) return &data.Transaction{ @@ -90,7 +90,7 @@ func (dtb *dbTransactionBuilder) prepareRewardTransaction( header coreData.HeaderHandler, txStatus string, ) *data.Transaction { - receiverAddr, _ := dtb.addressPubkeyConverter.Encode(rTx.RcvAddr) + receiverAddr := dtb.addressPubkeyConverter.SilentEncode(rTx.RcvAddr, log) return &data.Transaction{ Hash: hex.EncodeToString(txHash), @@ -117,7 +117,7 @@ func (dtb *dbTransactionBuilder) prepareReceipt( rec *receipt.Receipt, header coreData.HeaderHandler, ) *data.Receipt { - senderAddr, _ := dtb.addressPubkeyConverter.Encode(rec.SndAddr) + senderAddr := dtb.addressPubkeyConverter.SilentEncode(rec.SndAddr, log) return &data.Receipt{ Hash: hex.EncodeToString([]byte(recHash)), diff --git a/process/elasticproc/validators/validatorsProcessor.go b/process/elasticproc/validators/validatorsProcessor.go index 8b047d80..be3fd628 100644 --- a/process/elasticproc/validators/validatorsProcessor.go +++ b/process/elasticproc/validators/validatorsProcessor.go @@ -31,6 +31,7 @@ func (vp *validatorsProcessor) PrepareValidatorsPublicKeys(shardValidatorsPubKey } for _, validatorPk := range shardValidatorsPubKeys { + // it will never throw an error here strValidatorPk, _ := vp.validatorPubkeyConverter.Encode(validatorPk) validatorsPubKeys.PublicKeys = append(validatorsPubKeys.PublicKeys, strValidatorPk) From 47283b62466fb0dc9eea168855691fa1b70a111a Mon Sep 17 00:00:00 2001 From: schimih Date: Thu, 24 Nov 2022 22:39:43 +0200 Subject: [PATCH 006/189] small fix --- factory/wsIndexerFactory.go | 6 +++--- mock/pubkeyConverterMock.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index c039ea82..12a31aca 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -11,8 +11,8 @@ import ( ) const ( - indexerCacheSize = 1 - addressPrefix = "erd" + indexerCacheSize = 1 + defaultAddressPrefix = "erd" ) // CreateWsIndexer will create a new instance of wsindexer.WSClient @@ -44,7 +44,7 @@ func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsin if err != nil { return nil, err } - addressPubkeyConverter, err := pubkeyConverter.NewBech32PubkeyConverter(cfg.Config.AddressConverter.Length, addressPrefix) + addressPubkeyConverter, err := pubkeyConverter.NewBech32PubkeyConverter(cfg.Config.AddressConverter.Length, defaultAddressPrefix) if err != nil { return nil, err } diff --git a/mock/pubkeyConverterMock.go b/mock/pubkeyConverterMock.go index e4527046..454697ef 100644 --- a/mock/pubkeyConverterMock.go +++ b/mock/pubkeyConverterMock.go @@ -30,13 +30,13 @@ func (pcm *PubkeyConverterMock) Encode(pkBytes []byte) (string, error) { // EncodeSlice - func (pcm *PubkeyConverterMock) EncodeSlice(pkBytesSlice [][]byte) ([]string, error) { - decodedSlice := make([]string, 0) + encodedSlice := make([]string, 0) for _, pkBytes := range pkBytesSlice { - decodedSlice = append(decodedSlice, hex.EncodeToString(pkBytes)) + encodedSlice = append(encodedSlice, hex.EncodeToString(pkBytes)) } - return decodedSlice, nil + return encodedSlice, nil } // SilentEncode - From 162aeebbc2c967cc3f9f0f44d6fabb0f45ff5de3 Mon Sep 17 00:00:00 2001 From: schimih Date: Tue, 6 Dec 2022 10:54:28 +0200 Subject: [PATCH 007/189] moved defaultAddressPrefix to config.toml --- cmd/elasticindexer/config/config.toml | 1 + config/config.go | 1 + factory/wsIndexerFactory.go | 5 ++--- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/elasticindexer/config/config.toml b/cmd/elasticindexer/config/config.toml index d1a9f2c7..a96919c9 100644 --- a/cmd/elasticindexer/config/config.toml +++ b/cmd/elasticindexer/config/config.toml @@ -7,6 +7,7 @@ [config.address-converter] length = 32 type = "bech32" + prefix = "erd" [config.validator-keys-converter] length = 96 type = "hex" diff --git a/config/config.go b/config/config.go index b726953f..40dead91 100644 --- a/config/config.go +++ b/config/config.go @@ -7,6 +7,7 @@ type Config struct { AddressConverter struct { Length int `toml:"length"` Type string `toml:"type"` + Prefix string `toml:"prefix"` } `toml:"address-converter"` ValidatorKeysConverter struct { Length int `toml:"length"` diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 12a31aca..959414b4 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -11,8 +11,7 @@ import ( ) const ( - indexerCacheSize = 1 - defaultAddressPrefix = "erd" + indexerCacheSize = 1 ) // CreateWsIndexer will create a new instance of wsindexer.WSClient @@ -44,7 +43,7 @@ func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsin if err != nil { return nil, err } - addressPubkeyConverter, err := pubkeyConverter.NewBech32PubkeyConverter(cfg.Config.AddressConverter.Length, defaultAddressPrefix) + addressPubkeyConverter, err := pubkeyConverter.NewBech32PubkeyConverter(cfg.Config.AddressConverter.Length, cfg.Config.AddressConverter.Prefix) if err != nil { return nil, err } From 1a7484b32d3319a3adef6dcf63b276e66a28ed20 Mon Sep 17 00:00:00 2001 From: schimih Date: Wed, 7 Dec 2022 11:54:13 +0200 Subject: [PATCH 008/189] change elrond-go-core version --- go.mod | 8 +- go.sum | 77 +++++++++++++++++-- mock/pubkeyConverterStub.go | 2 +- .../smartContractResultsProcessor_test.go | 2 +- 4 files changed, 78 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 8171f7b3..f333de43 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ElrondNetwork/elastic-indexer-go go 1.17 require ( - github.com/ElrondNetwork/elrond-go-core v1.1.25-0.20221124191416-f1a624eb9164 + github.com/ElrondNetwork/elrond-go-core v1.1.27 github.com/ElrondNetwork/elrond-go-logger v1.0.9 github.com/ElrondNetwork/elrond-vm-common v1.3.27-0.20221114140858-157beda2e3f9 github.com/elastic/go-elasticsearch/v7 v7.12.0 @@ -14,7 +14,7 @@ require ( ) require ( - github.com/btcsuite/btcutil v1.0.2 // indirect + github.com/btcsuite/btcd/btcutil v1.1.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/denisbrodbeck/machineid v1.0.1 // indirect @@ -27,8 +27,8 @@ require ( github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect - golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect - golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect + golang.org/x/crypto v0.3.0 // indirect + golang.org/x/sys v0.2.0 // indirect google.golang.org/protobuf v1.26.0 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect ) diff --git a/go.sum b/go.sum index 29216793..676aa20e 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221108131533-15ef756cddb2/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= -github.com/ElrondNetwork/elrond-go-core v1.1.25-0.20221124191416-f1a624eb9164 h1:nAg0G4f0ZaiqrtvWKg8gJ372RZm35o6fLtlxz7ixtbQ= -github.com/ElrondNetwork/elrond-go-core v1.1.25-0.20221124191416-f1a624eb9164/go.mod h1:UcZAiUqKBv3M3U6pJkYqIAx4OKubLYIc0QBVN+bY29g= +github.com/ElrondNetwork/elrond-go-core v1.1.27 h1:WDxSGyUQA5N8WX543nsOWC1PPrWKB7B+EyXmtC0nPt8= +github.com/ElrondNetwork/elrond-go-core v1.1.27/go.mod h1:N/RI++YU2M6OlnD1GSZepc1wPhI84ykRDQ1IyD3B0wk= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5aiSrvkGzcnF7sK06RD8mFxQ= github.com/ElrondNetwork/elrond-go-logger v1.0.9 h1:RByDnVHgSN4zy3bwmvQarsnCd/gDmZ+zwVI+b6i4Plc= @@ -13,13 +13,25 @@ github.com/ElrondNetwork/elrond-vm-common v1.3.27-0.20221114140858-157beda2e3f9/ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M= +github.com/btcsuite/btcd v0.23.0 h1:V2/ZgjfDFIygAX3ZapeigkVBoVUtOJKSwrhZdlpSvaA= +github.com/btcsuite/btcd v0.23.0/go.mod h1:0QJIIN1wwIXF/3G/m87gIwGniDMDQqjVn4SZgnFpsYY= +github.com/btcsuite/btcd/btcec/v2 v2.1.0/go.mod h1:2VzYrv4Gm4apmbVVsSq5bqf1Ec8v56E48Vt0Y/umPgA= +github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= +github.com/btcsuite/btcd/btcutil v1.0.0/go.mod h1:Uoxwv0pqYWhD//tfTiipkxNfdhG9UrLwaeswfjfdF0A= +github.com/btcsuite/btcd/btcutil v1.1.0/go.mod h1:5OapHB7A2hBBWLm48mmw4MOHNJCcUBTwmWH/0Jn8VHE= +github.com/btcsuite/btcd/btcutil v1.1.3 h1:xfbtw8lwpp0G6NwSHb+UE67ryTFHJAiNuipusjXSohQ= +github.com/btcsuite/btcd/btcutil v1.1.3/go.mod h1:UR7dsSJzJUfMmFiiLlIrMq1lS9jh9EdCV7FStZSnpi0= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcutil v1.0.2 h1:9iZ1Terx9fMIOtq1VrwdqfsATL9MC2l8ZrUY6YZ2uts= github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= @@ -28,20 +40,34 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= +github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/elastic/go-elasticsearch/v7 v7.12.0 h1:j4tvcMrZJLp39L2NYvBb7f+lHKPqPHSL3nvB8+/DV+s= github.com/elastic/go-elasticsearch/v7 v7.12.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= @@ -51,6 +77,7 @@ github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -59,9 +86,15 @@ github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -78,6 +111,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -88,45 +122,76 @@ github.com/urfave/cli v1.22.10 h1:p8Fspmz3iTctJstry1PYS3HVdllxnEzTEsgIgtxTrCk= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= +golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= @@ -136,5 +201,7 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/mock/pubkeyConverterStub.go b/mock/pubkeyConverterStub.go index b5c39182..97d03edd 100644 --- a/mock/pubkeyConverterStub.go +++ b/mock/pubkeyConverterStub.go @@ -49,7 +49,7 @@ func (pcs *PubkeyConverterStub) EncodeSlice(pkBytesSlice [][]byte) ([]string, er // SilentEncode - func (pcs *PubkeyConverterStub) SilentEncode(pkBytes []byte, log core.Logger) string { - if pcs.EncodeCalled != nil { + if pcs.SilentEncodeCalled != nil { return pcs.SilentEncodeCalled(pkBytes, log) } diff --git a/process/elasticproc/transactions/smartContractResultsProcessor_test.go b/process/elasticproc/transactions/smartContractResultsProcessor_test.go index f48c5d02..942a7558 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor_test.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor_test.go @@ -51,7 +51,7 @@ func TestPrepareSmartContractResult(t *testing.T) { scRes := scrsProc.prepareSmartContractResult([]byte(scHash), mbHash, smartContractRes, header, 0, 1, big.NewInt(0), 0, 3) senderAddr, err := pubKeyConverter.Encode(sndAddr) require.Nil(t, err) - receiverAddr, _ := pubKeyConverter.Encode(rcvAddr) + receiverAddr, err := pubKeyConverter.Encode(rcvAddr) require.Nil(t, err) expectedTx := &data.ScResult{ Nonce: nonce, From 2dcb048eb9c04a7e9a0324cafc654229a21af262 Mon Sep 17 00:00:00 2001 From: schimih Date: Wed, 7 Dec 2022 15:01:53 +0200 Subject: [PATCH 009/189] updated go mod --- go.mod | 2 +- go.sum | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index f333de43..c70a1dd8 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/ElrondNetwork/elrond-go-core v1.1.27 github.com/ElrondNetwork/elrond-go-logger v1.0.9 - github.com/ElrondNetwork/elrond-vm-common v1.3.27-0.20221114140858-157beda2e3f9 + github.com/ElrondNetwork/elrond-vm-common v1.3.29 github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gorilla/websocket v1.5.0 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index 676aa20e..21c7ab4b 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,5 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= -github.com/ElrondNetwork/elrond-go-core v1.1.24-0.20221108131533-15ef756cddb2/go.mod h1:Yz8JK5sGBctw7+gU8j2mZHbzQ09Ek4XHJ4Uinq1N6nM= github.com/ElrondNetwork/elrond-go-core v1.1.27 h1:WDxSGyUQA5N8WX543nsOWC1PPrWKB7B+EyXmtC0nPt8= github.com/ElrondNetwork/elrond-go-core v1.1.27/go.mod h1:N/RI++YU2M6OlnD1GSZepc1wPhI84ykRDQ1IyD3B0wk= github.com/ElrondNetwork/elrond-go-logger v1.0.4/go.mod h1:e5D+c97lKUfFdAzFX7rrI2Igl/z4Y0RkKYKWyzprTGk= @@ -8,8 +7,8 @@ github.com/ElrondNetwork/elrond-go-logger v1.0.7/go.mod h1:cBfgx0ST/CJx8jrxJSC5a github.com/ElrondNetwork/elrond-go-logger v1.0.9 h1:RByDnVHgSN4zy3bwmvQarsnCd/gDmZ+zwVI+b6i4Plc= github.com/ElrondNetwork/elrond-go-logger v1.0.9/go.mod h1:86lz/Y9D1sMAZiukCYxCsETLl1zuI23qj37ct3KUT6A= github.com/ElrondNetwork/elrond-vm-common v1.1.0/go.mod h1:w3i6f8uiuRkE68Ie/gebRcLgTuHqvruJSYrFyZWuLrE= -github.com/ElrondNetwork/elrond-vm-common v1.3.27-0.20221114140858-157beda2e3f9 h1:G2dSsn6zLoZ0rLms7zq4VCRPj7EgwksLeGFGhGnZGjg= -github.com/ElrondNetwork/elrond-vm-common v1.3.27-0.20221114140858-157beda2e3f9/go.mod h1:VfPG0Zwy+jQtaYOplYXJUctr88AWm2LduuAalY01t4I= +github.com/ElrondNetwork/elrond-vm-common v1.3.29 h1:KbJQ9doHOufZlJXs9WfngnvDAY9lF7T6xec3lz8j/wM= +github.com/ElrondNetwork/elrond-vm-common v1.3.29/go.mod h1:NmcLl1rEVc40ONjlkPA78IPkvNEkizwBuBNJNe+F7Q0= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= From 1c9eeed41ebbb4b4c189885e42ec8b8d679b998c Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 16 Feb 2023 15:16:23 +0200 Subject: [PATCH 010/189] fix linter issues --- integrationtests/utils.go | 3 +++ process/elasticproc/elasticProcessor.go | 1 + process/elasticproc/logsevents/nftsPropertiesProcessor.go | 1 + 3 files changed, 5 insertions(+) diff --git a/integrationtests/utils.go b/integrationtests/utils.go index 9cfbfc28..a01f6020 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -25,6 +25,7 @@ var ( pubKeyConverter, _ = pubkeyConverter.NewBech32PubkeyConverter(32, log) ) +//nolint func setLogLevelDebug() { _ = logger.SetLogLevel("process:DEBUG") } @@ -64,6 +65,7 @@ func CreateElasticProcessor( return factory.CreateElasticProcessor(args) } +//nolint func readExpectedResult(path string) string { jsonFile, _ := os.Open(path) byteValue, _ := ioutil.ReadAll(jsonFile) @@ -71,6 +73,7 @@ func readExpectedResult(path string) string { return string(byteValue) } +//nolint func getElementFromSlice(path string, index int) string { fileBytes := readExpectedResult(path) slice := make([]map[string]interface{}, 0) diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index 291d6af7..95bd2946 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -108,6 +108,7 @@ func (ei *elasticProcessor) init(useKibana bool, indexTemplates, _ map[string]*b return err } + //nolint if useKibana { // TODO: Re-activate after we think of a solid way to handle forks+rotating indexes // err = ei.createIndexPolicies(indexPolicies) diff --git a/process/elasticproc/logsevents/nftsPropertiesProcessor.go b/process/elasticproc/logsevents/nftsPropertiesProcessor.go index c21d819c..8610542f 100644 --- a/process/elasticproc/logsevents/nftsPropertiesProcessor.go +++ b/process/elasticproc/logsevents/nftsPropertiesProcessor.go @@ -30,6 +30,7 @@ func newNFTsPropertiesProcessor(pubKeyConverter core.PubkeyConverter) *nftsPrope } func (npp *nftsPropertiesProc) processEvent(args *argsProcessEvent) argOutputProcessEvent { + //nolint eventIdentifier := string(args.event.GetIdentifier()) _, ok := npp.propertiesChangeOperations[eventIdentifier] if !ok { From 88d48267cb6caaa35f3394bcb547e61b5b73f704 Mon Sep 17 00:00:00 2001 From: schimih Date: Thu, 16 Feb 2023 16:11:25 +0200 Subject: [PATCH 011/189] pubkey updates --- factory/wsIndexerFactory.go | 1 - go.mod | 4 ++-- go.sum | 7 ++++--- mock/pubkeyConverterMock.go | 2 +- mock/pubkeyConverterStub.go | 2 +- process/elasticproc/logsevents/nftsPropertiesProcessor.go | 4 ++-- .../transactions/smartContractResultsProcessor.go | 1 - process/elasticproc/transactions/transactionDBBuilder.go | 2 -- .../elasticproc/transactions/transactionDBBuilder_test.go | 4 ++-- 9 files changed, 12 insertions(+), 15 deletions(-) diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index a664fcfd..31c1ad62 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -8,7 +8,6 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/factory" "github.com/multiversx/mx-chain-es-indexer-go/process/wsclient" "github.com/multiversx/mx-chain-es-indexer-go/process/wsindexer" - logger "github.com/multiversx/mx-chain-logger-go" ) const ( diff --git a/go.mod b/go.mod index cbed2179..bfa0718b 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,9 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gorilla/websocket v1.5.0 - github.com/multiversx/mx-chain-core-go v1.1.30 + github.com/multiversx/mx-chain-core-go v1.1.34-0.20230215164556-5ec8f51e96d3 github.com/multiversx/mx-chain-logger-go v1.0.11 - github.com/multiversx/mx-chain-vm-common-go v1.3.34 + github.com/multiversx/mx-chain-vm-common-go v1.3.37-0.20230216122352-6dd6ff58ca2d github.com/stretchr/testify v1.7.0 github.com/tidwall/gjson v1.14.0 github.com/urfave/cli v1.22.10 diff --git a/go.sum b/go.sum index f4948e7f..bc7200d3 100644 --- a/go.sum +++ b/go.sum @@ -69,12 +69,13 @@ github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiversx/mx-chain-core-go v1.1.30 h1:BtURR4I6HU1OnSbxcPMTQSQXNqtOuH3RW6bg5N7FSM0= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= +github.com/multiversx/mx-chain-core-go v1.1.34-0.20230215164556-5ec8f51e96d3 h1:68ooIEnoUO69QGWXDa0VdDarOA4CaF9IiEMg9q/kisc= +github.com/multiversx/mx-chain-core-go v1.1.34-0.20230215164556-5ec8f51e96d3/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= -github.com/multiversx/mx-chain-vm-common-go v1.3.34 h1:6tSshEeVJpgFZmP8JV2wrPyRtL477oTYw1tjFcHTtv4= -github.com/multiversx/mx-chain-vm-common-go v1.3.34/go.mod h1:sZ2COLCxvf2GxAAJHGmGqWybObLtFuk2tZUyGqnMXE8= +github.com/multiversx/mx-chain-vm-common-go v1.3.37-0.20230216122352-6dd6ff58ca2d h1:n2qx7CceoqbEqZXWOsjLccS34zLPh5KNriIXTlNSIc4= +github.com/multiversx/mx-chain-vm-common-go v1.3.37-0.20230216122352-6dd6ff58ca2d/go.mod h1:Y9ggiJtjGCPK/8WEzFO91JLlppMa/uUMobMmsogeiRw= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= diff --git a/mock/pubkeyConverterMock.go b/mock/pubkeyConverterMock.go index 454697ef..e81d21ff 100644 --- a/mock/pubkeyConverterMock.go +++ b/mock/pubkeyConverterMock.go @@ -3,7 +3,7 @@ package mock import ( "encoding/hex" - "github.com/ElrondNetwork/elrond-go-core/core" + "github.com/multiversx/mx-chain-core-go/core" ) // PubkeyConverterMock - diff --git a/mock/pubkeyConverterStub.go b/mock/pubkeyConverterStub.go index 97d03edd..dcd3de04 100644 --- a/mock/pubkeyConverterStub.go +++ b/mock/pubkeyConverterStub.go @@ -1,6 +1,6 @@ package mock -import "github.com/ElrondNetwork/elrond-go-core/core" +import "github.com/multiversx/mx-chain-core-go/core" // PubkeyConverterStub - type PubkeyConverterStub struct { diff --git a/process/elasticproc/logsevents/nftsPropertiesProcessor.go b/process/elasticproc/logsevents/nftsPropertiesProcessor.go index 2a259d2f..990e844e 100644 --- a/process/elasticproc/logsevents/nftsPropertiesProcessor.go +++ b/process/elasticproc/logsevents/nftsPropertiesProcessor.go @@ -37,7 +37,7 @@ func (npp *nftsPropertiesProc) processEvent(args *argsProcessEvent) argOutputPro return argOutputProcessEvent{} } - callerAddress := npp.pubKeyConverter.Encode(args.event.GetAddress()) + callerAddress := npp.pubKeyConverter.SilentEncode(args.event.GetAddress(), log) if callerAddress == "" { return argOutputProcessEvent{ processed: true, @@ -60,7 +60,7 @@ func (npp *nftsPropertiesProc) processEvent(args *argsProcessEvent) argOutputPro } } - callerAddress := npp.pubKeyConverter.SilentEncode(args.event.GetAddress(), log) + callerAddress = npp.pubKeyConverter.SilentEncode(args.event.GetAddress(), log) if callerAddress == "" { return argOutputProcessEvent{ processed: true, diff --git a/process/elasticproc/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go index e3210dc0..dfd759e6 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -15,7 +15,6 @@ import ( "github.com/multiversx/mx-chain-core-go/marshal" indexerData "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" - datafield "github.com/multiversx/mx-chain-vm-common-go/parsers/dataField" ) type smartContractResultsProcessor struct { diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index 8e74a1a5..a0ae2d6c 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -7,7 +7,6 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/core" - "github.com/multiversx/mx-chain-core-go/core/sharding" coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/receipt" @@ -15,7 +14,6 @@ import ( "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" - datafield "github.com/multiversx/mx-chain-vm-common-go/parsers/dataField" ) type dbTransactionBuilder struct { diff --git a/process/elasticproc/transactions/transactionDBBuilder_test.go b/process/elasticproc/transactions/transactionDBBuilder_test.go index 6bad7e4b..a83188ba 100644 --- a/process/elasticproc/transactions/transactionDBBuilder_test.go +++ b/process/elasticproc/transactions/transactionDBBuilder_test.go @@ -154,8 +154,8 @@ func TestGetMoveBalanceTransactionInvalid(t *testing.T) { Round: header.Round, Value: tx.Value.String(), ValueNum: 1e-15, - Receiver: cp.addressPubkeyConverter.Encode(tx.RcvAddr), - Sender: cp.addressPubkeyConverter.Encode(tx.SndAddr), + Receiver: cp.addressPubkeyConverter.SilentEncode(tx.RcvAddr, log), + Sender: cp.addressPubkeyConverter.SilentEncode(tx.SndAddr, log), ReceiverShard: uint32(2), SenderShard: mb.SenderShardID, GasPrice: gasPrice, From cee11ea0d0f5fdb2fcfa95943792207432d340da Mon Sep 17 00:00:00 2001 From: schimih Date: Thu, 16 Feb 2023 16:31:22 +0200 Subject: [PATCH 012/189] fix after self-review --- process/elasticproc/transactions/transactionDBBuilder.go | 9 ++++++++- .../transactions/transactionDBBuilder_test.go | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index a0ae2d6c..abf24061 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -7,6 +7,7 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/core/sharding" coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/receipt" @@ -53,6 +54,11 @@ func (dtb *dbTransactionBuilder) prepareTransaction( senderAddr := dtb.addressPubkeyConverter.SilentEncode(tx.SndAddr, log) receiversAddr, _ := dtb.addressPubkeyConverter.EncodeSlice(res.Receivers) + receiverShardID := mb.ReceiverShardID + if mb.Type == block.InvalidBlock { + receiverShardID = sharding.ComputeShardID(tx.RcvAddr, numOfShards) + } + return &data.Transaction{ Hash: hex.EncodeToString(txHash), MBHash: hex.EncodeToString(mbHash), @@ -61,7 +67,8 @@ func (dtb *dbTransactionBuilder) prepareTransaction( Value: tx.Value.String(), Receiver: receiverAddr, Sender: senderAddr, - ReceiverShard: mb.ReceiverShardID, + ValueNum: dtb.balanceConverter.ComputeESDTBalanceAsFloat(tx.Value), + ReceiverShard: receiverShardID, SenderShard: mb.SenderShardID, GasPrice: tx.GasPrice, GasLimit: tx.GasLimit, diff --git a/process/elasticproc/transactions/transactionDBBuilder_test.go b/process/elasticproc/transactions/transactionDBBuilder_test.go index a83188ba..5c2c6d94 100644 --- a/process/elasticproc/transactions/transactionDBBuilder_test.go +++ b/process/elasticproc/transactions/transactionDBBuilder_test.go @@ -64,6 +64,7 @@ func TestGetMoveBalanceTransaction(t *testing.T) { Nonce: tx.Nonce, Round: header.Round, Value: tx.Value.String(), + ValueNum: 1e-15, Receiver: senderAddr, Sender: receiverAddr, ReceiverShard: mb.ReceiverShardID, From 252bfbf1420488ec3ba9b13428a00285ef44d435 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 14 Mar 2023 16:22:08 +0200 Subject: [PATCH 013/189] integrate new structures in indexer WIP --- data/account.go | 6 +- data/data.go | 2 +- go.mod | 2 +- go.sum | 7 +- process/dataindexer/dataIndexer.go | 65 ++-- process/dataindexer/interface.go | 34 +- process/dataindexer/workItems/interface.go | 22 +- .../dataindexer/workItems/workItemAccounts.go | 27 +- .../dataindexer/workItems/workItemBlock.go | 121 +++---- .../dataindexer/workItems/workItemRating.go | 14 +- .../workItems/workItemRemoveBlock.go | 36 ++- .../dataindexer/workItems/workItemRounds.go | 14 +- .../workItems/workItemValidators.go | 21 +- .../elasticproc/accounts/accountsProcessor.go | 18 +- .../accounts/accountsProcessor_test.go | 60 ++-- process/elasticproc/block/blockProcessor.go | 124 ++++---- .../elasticproc/block/blockProcessor_test.go | 297 +++++++++++------- .../elasticproc/converters/tokenMetaData.go | 4 +- .../converters/tokenMetaData_test.go | 4 +- process/elasticproc/elasticProcessor.go | 70 ++--- process/elasticproc/interface.go | 30 +- .../logsevents/logsAndEventsProcessor.go | 27 +- .../logsevents/logsAndEventsProcessor_test.go | 56 ++-- .../elasticproc/logsevents/nftsProcessor.go | 6 +- process/elasticproc/statistics/serialize.go | 17 +- .../elasticproc/statistics/serialize_test.go | 10 +- process/elasticproc/transactions/checkers.go | 2 +- .../elasticproc/transactions/checkers_test.go | 12 +- .../smartContractResultsProcessor.go | 38 +-- .../smartContractResultsProcessor_test.go | 12 +- .../transactions/transactionDBBuilder.go | 20 +- .../transactions/transactionsGrouper.go | 52 +-- .../transactions/transactionsProcessor.go | 8 +- .../transactionsProcessor_test.go | 231 ++++++++------ process/elasticproc/validators/serialize.go | 46 +-- .../elasticproc/validators/serialize_test.go | 51 +-- .../validators/validatorsProcessor.go | 44 ++- .../validators/validatorsProcessor_test.go | 18 +- process/wsindexer/indexer.go | 35 ++- process/wsindexer/interface.go | 13 +- process/wsindexer/types.go | 43 --- process/wsindexer/unmarshal.go | 192 ----------- process/wsindexer/unmarshal_test.go | 103 ------ 43 files changed, 809 insertions(+), 1205 deletions(-) delete mode 100644 process/wsindexer/types.go delete mode 100644 process/wsindexer/unmarshal.go delete mode 100644 process/wsindexer/unmarshal_test.go diff --git a/data/account.go b/data/account.go index 5175d6cd..8e370b13 100644 --- a/data/account.go +++ b/data/account.go @@ -3,7 +3,7 @@ package data import ( "time" - "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-core-go/data/alteredAccount" ) // AccountInfo holds (serializable) data about an account @@ -62,13 +62,13 @@ type AccountBalanceHistory struct { // Account is a structure that is needed for regular accounts type Account struct { - UserAccount *outport.AlteredAccount + UserAccount *alteredAccount.AlteredAccount IsSender bool } // AccountESDT is a structure that is needed for ESDT accounts type AccountESDT struct { - Account *outport.AlteredAccount + Account *alteredAccount.AlteredAccount TokenIdentifier string NFTNonce uint64 IsSender bool diff --git a/data/data.go b/data/data.go index b0f661c0..e17042e6 100644 --- a/data/data.go +++ b/data/data.go @@ -24,7 +24,7 @@ type ValidatorRatingInfo struct { // RoundInfo is a structure containing block signers and shard id type RoundInfo struct { - Index uint64 `json:"round"` + Round uint64 `json:"round"` SignersIndexes []uint64 `json:"signersIndexes"` BlockWasProposed bool `json:"blockWasProposed"` ShardId uint32 `json:"shardId"` diff --git a/go.mod b/go.mod index 202003d1..d0a08eb2 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gorilla/websocket v1.5.0 - github.com/multiversx/mx-chain-core-go v1.1.30 + github.com/multiversx/mx-chain-core-go v1.1.35-0.20230314140225-cdd3af9ba37e github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.3.34 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index 5db33d5b..1d07e500 100644 --- a/go.sum +++ b/go.sum @@ -69,8 +69,13 @@ github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiversx/mx-chain-core-go v1.1.30 h1:BtURR4I6HU1OnSbxcPMTQSQXNqtOuH3RW6bg5N7FSM0= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= +github.com/multiversx/mx-chain-core-go v1.1.35-0.20230306111403-9272fdab3455 h1:C96Si2yl6pn98SPvJM6cD33bCapT0JZ7NeUlPicljws= +github.com/multiversx/mx-chain-core-go v1.1.35-0.20230306111403-9272fdab3455/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= +github.com/multiversx/mx-chain-core-go v1.1.35-0.20230309140731-222b64e21ca6 h1:+J4X/BmNQPz7+/bhQ/mSss3Gl70GpwabZObFDUK39SE= +github.com/multiversx/mx-chain-core-go v1.1.35-0.20230309140731-222b64e21ca6/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= +github.com/multiversx/mx-chain-core-go v1.1.35-0.20230314140225-cdd3af9ba37e h1:B7Da37AgvQ1u82tjPv2L6EAhJ+PVPuU/9wq7v2aa0Xs= +github.com/multiversx/mx-chain-core-go v1.1.35-0.20230314140225-cdd3af9ba37e/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= github.com/multiversx/mx-chain-vm-common-go v1.3.34 h1:6tSshEeVJpgFZmP8JV2wrPyRtL477oTYw1tjFcHTtv4= diff --git a/process/dataindexer/dataIndexer.go b/process/dataindexer/dataIndexer.go index 435ce566..3fd13dfa 100644 --- a/process/dataindexer/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -1,11 +1,11 @@ package dataindexer import ( + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" - coreData "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/core/unmarshal" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/marshal" - "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" ) @@ -20,7 +20,8 @@ type dataIndexer struct { isNilIndexer bool dispatcher DispatcherHandler elasticProcessor ElasticProcessor - marshalizer marshal.Marshalizer + marshaller marshal.Marshalizer + headerMarshaller marshal.Marshalizer } // NewDataIndexer will create a new data indexer @@ -34,7 +35,7 @@ func NewDataIndexer(arguments ArgDataIndexer) (*dataIndexer, error) { isNilIndexer: false, dispatcher: arguments.DataDispatcher, elasticProcessor: arguments.ElasticProcessor, - marshalizer: arguments.Marshalizer, + marshaller: arguments.Marshalizer, } return dataIndexerObj, nil @@ -55,11 +56,18 @@ func checkIndexerArgs(arguments ArgDataIndexer) error { } // SaveBlock saves the block info in the queue to be sent to elastic -func (di *dataIndexer) SaveBlock(args *outport.ArgsSaveBlockData) error { +func (di *dataIndexer) SaveBlock(outportBlock *outport.OutportBlock) error { + header, err := unmarshal.GetHeaderFromBytes(di.headerMarshaller, core.HeaderType(outportBlock.BlockData.HeaderType), outportBlock.BlockData.HeaderBytes) + if err != nil { + return err + } + wi := workItems.NewItemBlock( di.elasticProcessor, - di.marshalizer, - args, + &outport.OutportBlockWithHeader{ + OutportBlock: *outportBlock, + Header: header, + }, ) di.dispatcher.Add(wi) @@ -72,11 +80,12 @@ func (di *dataIndexer) Close() error { } // RevertIndexedBlock will remove from database block and miniblocks -func (di *dataIndexer) RevertIndexedBlock(header coreData.HeaderHandler, body coreData.BodyHandler) error { +func (di *dataIndexer) RevertIndexedBlock(blockData *outport.BlockData) error { wi := workItems.NewItemRemoveBlock( di.elasticProcessor, - body, - header, + // TODO possible to be json or proto marshaller + di.headerMarshaller, + blockData, ) di.dispatcher.Add(wi) @@ -84,39 +93,18 @@ func (di *dataIndexer) RevertIndexedBlock(header coreData.HeaderHandler, body co } // SaveRoundsInfo will save data about a slice of rounds in elasticsearch -func (di *dataIndexer) SaveRoundsInfo(rf []*outport.RoundInfo) error { - roundsInfo := make([]*data.RoundInfo, 0) - for _, info := range rf { - roundsInfo = append(roundsInfo, &data.RoundInfo{ - Index: info.Index, - SignersIndexes: info.SignersIndexes, - BlockWasProposed: info.BlockWasProposed, - ShardId: info.ShardId, - Epoch: info.Epoch, - Timestamp: info.Timestamp, - }) - } - - wi := workItems.NewItemRounds(di.elasticProcessor, roundsInfo) +func (di *dataIndexer) SaveRoundsInfo(rounds *outport.RoundsInfo) error { + wi := workItems.NewItemRounds(di.elasticProcessor, rounds) di.dispatcher.Add(wi) return nil } // SaveValidatorsRating will save all validators rating info to elasticsearch -func (di *dataIndexer) SaveValidatorsRating(indexID string, validatorsRatingInfo []*outport.ValidatorRatingInfo) error { - valRatingInfo := make([]*data.ValidatorRatingInfo, 0) - for _, info := range validatorsRatingInfo { - valRatingInfo = append(valRatingInfo, &data.ValidatorRatingInfo{ - PublicKey: info.PublicKey, - Rating: info.Rating, - }) - } - +func (di *dataIndexer) SaveValidatorsRating(ratingData *outport.ValidatorsRating) error { wi := workItems.NewItemRating( di.elasticProcessor, - indexID, - valRatingInfo, + ratingData, ) di.dispatcher.Add(wi) @@ -124,10 +112,9 @@ func (di *dataIndexer) SaveValidatorsRating(indexID string, validatorsRatingInfo } // SaveValidatorsPubKeys will save all validators public keys to elasticsearch -func (di *dataIndexer) SaveValidatorsPubKeys(validatorsPubKeys map[uint32][][]byte, epoch uint32) error { +func (di *dataIndexer) SaveValidatorsPubKeys(validatorsPubKeys *outport.ValidatorsPubKeys) error { wi := workItems.NewItemValidators( di.elasticProcessor, - epoch, validatorsPubKeys, ) di.dispatcher.Add(wi) @@ -136,8 +123,8 @@ func (di *dataIndexer) SaveValidatorsPubKeys(validatorsPubKeys map[uint32][][]by } // SaveAccounts will save the provided accounts -func (di *dataIndexer) SaveAccounts(timestamp uint64, accounts map[string]*outport.AlteredAccount, shardID uint32) error { - wi := workItems.NewItemAccounts(di.elasticProcessor, timestamp, accounts, shardID) +func (di *dataIndexer) SaveAccounts(accounts *outport.Accounts) error { + wi := workItems.NewItemAccounts(di.elasticProcessor, accounts) di.dispatcher.Add(wi) return nil diff --git a/process/dataindexer/interface.go b/process/dataindexer/interface.go index 242b5afd..3469384a 100644 --- a/process/dataindexer/interface.go +++ b/process/dataindexer/interface.go @@ -6,7 +6,6 @@ import ( coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" - "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" ) @@ -20,26 +19,17 @@ type DispatcherHandler interface { // ElasticProcessor defines the interface for the elastic search indexer type ElasticProcessor interface { - SaveHeader( - headerHash []byte, - header coreData.HeaderHandler, - signersIndexes []uint64, - body *block.Body, - notarizedHeadersHashes []string, - gasConsumptionData outport.HeaderGasConsumption, - txsSize int, - pool *outport.Pool, - ) error + SaveHeader(outportBlockWithHeader *outport.OutportBlockWithHeader) error RemoveHeader(header coreData.HeaderHandler) error RemoveMiniblocks(header coreData.HeaderHandler, body *block.Body) error RemoveTransactions(header coreData.HeaderHandler, body *block.Body) error RemoveAccountsESDT(headerTimestamp uint64, shardID uint32) error SaveMiniblocks(header coreData.HeaderHandler, body *block.Body) error - SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount, isImportDB bool, numOfShards uint32) error - SaveValidatorsRating(index string, validatorsRatingInfo []*data.ValidatorRatingInfo) error - SaveRoundsInfo(infos []*data.RoundInfo) error - SaveShardValidatorsPubKeys(shardID, epoch uint32, shardValidatorsPubKeys [][]byte) error - SaveAccounts(blockTimestamp uint64, accounts []*data.Account, shardID uint32) error + SaveTransactions(outportBlockWithHeader *outport.OutportBlockWithHeader) error + SaveValidatorsRating(ratingData *outport.ValidatorsRating) error + SaveRoundsInfo(rounds *outport.RoundsInfo) error + SaveShardValidatorsPubKeys(validatorsPubKeys *outport.ValidatorsPubKeys) error + SaveAccounts(accounts *outport.Accounts) error IsInterfaceNil() bool } @@ -64,12 +54,12 @@ type ShardCoordinator interface { // Indexer is an interface for saving node specific data to other storage. // This could be an elastic search index, a MySql database or any other external services. type Indexer interface { - SaveBlock(args *outport.ArgsSaveBlockData) error - RevertIndexedBlock(header coreData.HeaderHandler, body coreData.BodyHandler) error - SaveRoundsInfo(roundsInfos []*outport.RoundInfo) error - SaveValidatorsPubKeys(validatorsPubKeys map[uint32][][]byte, epoch uint32) error - SaveValidatorsRating(indexID string, infoRating []*outport.ValidatorRatingInfo) error - SaveAccounts(blockTimestamp uint64, acc map[string]*outport.AlteredAccount, shardID uint32) error + SaveBlock(outportBlock *outport.OutportBlock) error + RevertIndexedBlock(blockData *outport.BlockData) error + SaveRoundsInfo(roundsInfos *outport.RoundsInfo) error + SaveValidatorsPubKeys(validatorsPubKeys *outport.ValidatorsPubKeys) error + SaveValidatorsRating(ratingData *outport.ValidatorsRating) error + SaveAccounts(accountsData *outport.Accounts) error FinalizedBlock(headerHash []byte) error Close() error IsInterfaceNil() bool diff --git a/process/dataindexer/workItems/interface.go b/process/dataindexer/workItems/interface.go index b65e51ef..f6be410c 100644 --- a/process/dataindexer/workItems/interface.go +++ b/process/dataindexer/workItems/interface.go @@ -4,7 +4,6 @@ import ( coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" - "github.com/multiversx/mx-chain-es-indexer-go/data" ) // WorkItemHandler defines the interface for item that needs to be saved in elasticsearch database @@ -14,22 +13,13 @@ type WorkItemHandler interface { } type saveBlockIndexer interface { - SaveHeader( - headerHash []byte, - header coreData.HeaderHandler, - signersIndexes []uint64, - body *block.Body, - notarizedHeadersHashes []string, - gasConsumptionData outport.HeaderGasConsumption, - txsSize int, - pool *outport.Pool, - ) error + SaveHeader(outportBlockWithHeader *outport.OutportBlockWithHeader) error SaveMiniblocks(header coreData.HeaderHandler, body *block.Body) error - SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount, isImportDB bool, numOfShards uint32) error + SaveTransactions(outportBlockWithHeader *outport.OutportBlockWithHeader) error } type saveRatingIndexer interface { - SaveValidatorsRating(index string, validatorsRatingInfo []*data.ValidatorRatingInfo) error + SaveValidatorsRating(ratingData *outport.ValidatorsRating) error } type removeIndexer interface { @@ -40,13 +30,13 @@ type removeIndexer interface { } type saveRounds interface { - SaveRoundsInfo(infos []*data.RoundInfo) error + SaveRoundsInfo(rounds *outport.RoundsInfo) error } type saveValidatorsIndexer interface { - SaveShardValidatorsPubKeys(shardID, epoch uint32, shardValidatorsPubKeys [][]byte) error + SaveShardValidatorsPubKeys(validatorsPubKeys *outport.ValidatorsPubKeys) error } type saveAccountsIndexer interface { - SaveAccounts(blockTimestamp uint64, accounts []*data.Account, shardID uint32) error + SaveAccounts(accounts *outport.Accounts) error } diff --git a/process/dataindexer/workItems/workItemAccounts.go b/process/dataindexer/workItems/workItemAccounts.go index 38852fab..54888d20 100644 --- a/process/dataindexer/workItems/workItemAccounts.go +++ b/process/dataindexer/workItems/workItemAccounts.go @@ -2,42 +2,31 @@ package workItems import ( "github.com/multiversx/mx-chain-core-go/data/outport" - "github.com/multiversx/mx-chain-es-indexer-go/data" ) type itemAccounts struct { - indexer saveAccountsIndexer - blockTimestamp uint64 - accounts map[string]*outport.AlteredAccount - shardID uint32 + indexer saveAccountsIndexer + accounts *outport.Accounts } // NewItemAccounts will create a new instance of itemAccounts func NewItemAccounts( indexer saveAccountsIndexer, - blockTimestamp uint64, - accounts map[string]*outport.AlteredAccount, - shardID uint32, + accounts *outport.Accounts, ) WorkItemHandler { return &itemAccounts{ - indexer: indexer, - accounts: accounts, - blockTimestamp: blockTimestamp, - shardID: shardID, + indexer: indexer, + accounts: accounts, } } // Save will save information about an account func (wiv *itemAccounts) Save() error { - accounts := make([]*data.Account, 0, len(wiv.accounts)) - for _, account := range wiv.accounts { - accounts = append(accounts, &data.Account{ - UserAccount: account, - IsSender: false, - }) + if wiv.accounts == nil { + return nil } - err := wiv.indexer.SaveAccounts(wiv.blockTimestamp, accounts, wiv.shardID) + err := wiv.indexer.SaveAccounts(wiv.accounts) if err != nil { log.Warn("itemAccounts.Save", "could not index account", diff --git a/process/dataindexer/workItems/workItemBlock.go b/process/dataindexer/workItems/workItemBlock.go index 52864da0..0020fc04 100644 --- a/process/dataindexer/workItems/workItemBlock.go +++ b/process/dataindexer/workItems/workItemBlock.go @@ -1,47 +1,36 @@ package workItems import ( - "encoding/hex" - "errors" "fmt" "time" "github.com/multiversx/mx-chain-core-go/core/check" - "github.com/multiversx/mx-chain-core-go/data" - "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" - "github.com/multiversx/mx-chain-core-go/marshal" logger "github.com/multiversx/mx-chain-logger-go" ) -// ErrBodyTypeAssertion signals that body type assertion failed -var ErrBodyTypeAssertion = errors.New("elasticsearch - body type assertion failed") - var log = logger.GetOrCreate("indexer/workItems") type itemBlock struct { - indexer saveBlockIndexer - marshalizer marshal.Marshalizer - argsSaveBlock *outport.ArgsSaveBlockData + indexer saveBlockIndexer + outportBlockWithHeader *outport.OutportBlockWithHeader } // NewItemBlock will create a new instance of ItemBlock func NewItemBlock( indexer saveBlockIndexer, - marshalizer marshal.Marshalizer, - args *outport.ArgsSaveBlockData, + outportBlock *outport.OutportBlockWithHeader, ) WorkItemHandler { return &itemBlock{ - indexer: indexer, - marshalizer: marshalizer, - argsSaveBlock: args, + indexer: indexer, + outportBlockWithHeader: outportBlock, } } // Save will prepare and save a block item in elasticsearch database func (wib *itemBlock) Save() error { - if check.IfNil(wib.argsSaveBlock.Header) { - log.Warn("nil header provided when trying to index block, will skip") + if check.IfNilReflect(wib.outportBlockWithHeader) { + log.Warn("nil outportBlock block provided when trying to index block, will skip") return nil } @@ -50,49 +39,33 @@ func (wib *itemBlock) Save() error { }(time.Now()) log.Debug("indexer: starting indexing block", - "hash", wib.argsSaveBlock.HeaderHash, - "nonce", wib.argsSaveBlock.Header.GetNonce()) - - body, ok := wib.argsSaveBlock.Body.(*block.Body) - if !ok { - return fmt.Errorf("%w when trying body assertion, block hash %s, nonce %d", - ErrBodyTypeAssertion, wib.argsSaveBlock.HeaderHash, wib.argsSaveBlock.Header.GetNonce()) - } + "hash", wib.outportBlockWithHeader.BlockData.HeaderHash, + "nonce", wib.outportBlockWithHeader.Header.GetNonce()) - if wib.argsSaveBlock.TransactionsPool == nil { - wib.argsSaveBlock.TransactionsPool = &outport.Pool{} + if wib.outportBlockWithHeader.TransactionPool == nil { + wib.outportBlockWithHeader.TransactionPool = &outport.TransactionPool{} } - txsSizeInBytes := ComputeSizeOfTxs(wib.marshalizer, wib.argsSaveBlock.TransactionsPool) - err := wib.indexer.SaveHeader( - wib.argsSaveBlock.HeaderHash, - wib.argsSaveBlock.Header, - wib.argsSaveBlock.SignersIndexes, - body, - wib.argsSaveBlock.NotarizedHeadersHashes, - wib.argsSaveBlock.HeaderGasConsumption, - txsSizeInBytes, - wib.argsSaveBlock.TransactionsPool, - ) + err := wib.indexer.SaveHeader(wib.outportBlockWithHeader) if err != nil { return fmt.Errorf("%w when saving header block, hash %s, nonce %d", - err, hex.EncodeToString(wib.argsSaveBlock.HeaderHash), wib.argsSaveBlock.Header.GetNonce()) + err, wib.outportBlockWithHeader.BlockData.HeaderHash, wib.outportBlockWithHeader.Header.GetNonce()) } - if len(body.MiniBlocks) == 0 { + if len(wib.outportBlockWithHeader.BlockData.Body.MiniBlocks) == 0 { return nil } - err = wib.indexer.SaveMiniblocks(wib.argsSaveBlock.Header, body) + err = wib.indexer.SaveMiniblocks(wib.outportBlockWithHeader.Header, wib.outportBlockWithHeader.BlockData.Body) if err != nil { return fmt.Errorf("%w when saving miniblocks, block hash %s, nonce %d", - err, hex.EncodeToString(wib.argsSaveBlock.HeaderHash), wib.argsSaveBlock.Header.GetNonce()) + err, wib.outportBlockWithHeader.BlockData.HeaderHash, wib.outportBlockWithHeader.Header.GetNonce()) } - err = wib.indexer.SaveTransactions(body, wib.argsSaveBlock.Header, wib.argsSaveBlock.TransactionsPool, wib.argsSaveBlock.AlteredAccounts, wib.argsSaveBlock.IsImportDB, wib.argsSaveBlock.NumberOfShards) + err = wib.indexer.SaveTransactions(wib.outportBlockWithHeader) if err != nil { return fmt.Errorf("%w when saving transactions, block hash %s, nonce %d", - err, hex.EncodeToString(wib.argsSaveBlock.HeaderHash), wib.argsSaveBlock.Header.GetNonce()) + err, wib.outportBlockWithHeader.BlockData.HeaderHash, wib.outportBlockWithHeader.Header.GetNonce()) } return nil @@ -104,32 +77,32 @@ func (wib *itemBlock) IsInterfaceNil() bool { } // ComputeSizeOfTxs will compute size of transactions in bytes -func ComputeSizeOfTxs(marshalizer marshal.Marshalizer, pool *outport.Pool) int { - sizeTxs := 0 - sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Txs) - sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Scrs) - sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Invalid) - sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Rewards) - sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Receipts) - - return sizeTxs -} - -func computeSizeOfMapTxs(marshalizer marshal.Marshalizer, mapTxs map[string]data.TransactionHandlerWithGasUsedAndFee) int { - txsSize := 0 - for _, tx := range mapTxs { - txsSize += computeTxSize(marshalizer, tx.GetTxHandler()) - } - - return txsSize -} - -func computeTxSize(marshalizer marshal.Marshalizer, tx data.TransactionHandler) int { - txBytes, err := marshalizer.Marshal(tx) - if err != nil { - log.Debug("itemBlock.computeTxSize", "error", err) - return 0 - } - - return len(txBytes) -} +//func ComputeSizeOfTxs(marshalizer marshal.Marshalizer, pool *outport.TransactionPool) int { +// sizeTxs := 0 +// sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Transactions) +// sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Scrs) +// sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Invalid) +// sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Rewards) +// sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Receipts) +// +// return sizeTxs +//} +// +//func computeSizeOfMapTxs(marshalizer marshal.Marshalizer, mapTxs map[string]data.TransactionHandlerWithGasUsedAndFee) int { +// txsSize := 0 +// for _, tx := range mapTxs { +// txsSize += computeTxSize(marshalizer, tx.GetTxHandler()) +// } +// +// return txsSize +//} +// +//func computeTxSize(marshalizer marshal.Marshalizer, tx data.TransactionHandler) int { +// txBytes, err := marshalizer.Marshal(tx) +// if err != nil { +// log.Debug("itemBlock.computeTxSize", "error", err) +// return 0 +// } +// +// return len(txBytes) +//} diff --git a/process/dataindexer/workItems/workItemRating.go b/process/dataindexer/workItems/workItemRating.go index 8cac1ce8..9f2e2f52 100644 --- a/process/dataindexer/workItems/workItemRating.go +++ b/process/dataindexer/workItems/workItemRating.go @@ -1,19 +1,19 @@ package workItems -import "github.com/multiversx/mx-chain-es-indexer-go/data" +import ( + "github.com/multiversx/mx-chain-core-go/data/outport" +) type itemRating struct { indexer saveRatingIndexer - indexID string - infoRating []*data.ValidatorRatingInfo + ratingData *outport.ValidatorsRating } // NewItemRating will create a new instance of itemRating -func NewItemRating(indexer saveRatingIndexer, indexID string, infoRating []*data.ValidatorRatingInfo) WorkItemHandler { +func NewItemRating(indexer saveRatingIndexer, ratingData *outport.ValidatorsRating) WorkItemHandler { return &itemRating{ indexer: indexer, - indexID: indexID, - infoRating: infoRating, + ratingData: ratingData, } } @@ -24,7 +24,7 @@ func (wir *itemRating) IsInterfaceNil() bool { // Save will save validators rating in elasticsearch database func (wir *itemRating) Save() error { - err := wir.indexer.SaveValidatorsRating(wir.indexID, wir.infoRating) + err := wir.indexer.SaveValidatorsRating(wir.ratingData) if err != nil { log.Warn("itemRating.Save", "could not index validators rating", err.Error()) return err diff --git a/process/dataindexer/workItems/workItemRemoveBlock.go b/process/dataindexer/workItems/workItemRemoveBlock.go index 1e0497f8..bc3cd8b6 100644 --- a/process/dataindexer/workItems/workItemRemoveBlock.go +++ b/process/dataindexer/workItems/workItemRemoveBlock.go @@ -1,26 +1,28 @@ package workItems import ( - "github.com/multiversx/mx-chain-core-go/data" - "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/core/unmarshal" + "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-core-go/marshal" ) type itemRemoveBlock struct { - indexer removeIndexer - bodyHandler data.BodyHandler - headerHandler data.HeaderHandler + indexer removeIndexer + marshaller marshal.Marshalizer + blockData *outport.BlockData } // NewItemRemoveBlock will create a new instance of itemRemoveBlock func NewItemRemoveBlock( indexer removeIndexer, - bodyHandler data.BodyHandler, - headerHandler data.HeaderHandler, + marshaller marshal.Marshalizer, + blockData *outport.BlockData, ) WorkItemHandler { return &itemRemoveBlock{ - indexer: indexer, - bodyHandler: bodyHandler, - headerHandler: headerHandler, + indexer: indexer, + marshaller: marshaller, + blockData: blockData, } } @@ -31,25 +33,25 @@ func (wirb *itemRemoveBlock) IsInterfaceNil() bool { // Save will remove a block and miniblocks from elasticsearch database func (wirb *itemRemoveBlock) Save() error { - err := wirb.indexer.RemoveHeader(wirb.headerHandler) + header, err := unmarshal.GetHeaderFromBytes(wirb.marshaller, core.HeaderType(wirb.blockData.HeaderType), wirb.blockData.HeaderBytes) if err != nil { return err } - body, ok := wirb.bodyHandler.(*block.Body) - if !ok { - return ErrBodyTypeAssertion + err = wirb.indexer.RemoveHeader(header) + if err != nil { + return err } - err = wirb.indexer.RemoveMiniblocks(wirb.headerHandler, body) + err = wirb.indexer.RemoveMiniblocks(header, wirb.blockData.Body) if err != nil { return err } - err = wirb.indexer.RemoveTransactions(wirb.headerHandler, body) + err = wirb.indexer.RemoveTransactions(header, wirb.blockData.Body) if err != nil { return err } - return wirb.indexer.RemoveAccountsESDT(wirb.headerHandler.GetTimeStamp(), wirb.headerHandler.GetShardID()) + return wirb.indexer.RemoveAccountsESDT(header.GetTimeStamp(), header.GetShardID()) } diff --git a/process/dataindexer/workItems/workItemRounds.go b/process/dataindexer/workItems/workItemRounds.go index 11349b2a..5e078cd6 100644 --- a/process/dataindexer/workItems/workItemRounds.go +++ b/process/dataindexer/workItems/workItemRounds.go @@ -1,25 +1,25 @@ package workItems import ( - "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/multiversx/mx-chain-core-go/data/outport" ) type itemRounds struct { - indexer saveRounds - roundsInfo []*data.RoundInfo + indexer saveRounds + rounds *outport.RoundsInfo } // NewItemRounds will create a new instance of itemRounds -func NewItemRounds(indexer saveRounds, roundsInfo []*data.RoundInfo) WorkItemHandler { +func NewItemRounds(indexer saveRounds, rounds *outport.RoundsInfo) WorkItemHandler { return &itemRounds{ - indexer: indexer, - roundsInfo: roundsInfo, + indexer: indexer, + rounds: rounds, } } // Save will save in elasticsearch database information about rounds func (wir *itemRounds) Save() error { - err := wir.indexer.SaveRoundsInfo(wir.roundsInfo) + err := wir.indexer.SaveRoundsInfo(wir.rounds) if err != nil { log.Warn("itemRounds.Save", "could not index rounds info", err.Error()) return err diff --git a/process/dataindexer/workItems/workItemValidators.go b/process/dataindexer/workItems/workItemValidators.go index c6f424f4..a70d0239 100644 --- a/process/dataindexer/workItems/workItemValidators.go +++ b/process/dataindexer/workItems/workItemValidators.go @@ -1,37 +1,26 @@ package workItems +import "github.com/multiversx/mx-chain-core-go/data/outport" + type itemValidators struct { indexer saveValidatorsIndexer - epoch uint32 - validatorsPubKeys map[uint32][][]byte + validatorsPubKeys *outport.ValidatorsPubKeys } // NewItemValidators will create a new instance of itemValidators func NewItemValidators( indexer saveValidatorsIndexer, - epoch uint32, - validatorsPubKeys map[uint32][][]byte, + validatorsPubKeys *outport.ValidatorsPubKeys, ) WorkItemHandler { return &itemValidators{ indexer: indexer, - epoch: epoch, validatorsPubKeys: validatorsPubKeys, } } // Save will save information about validators func (wiv *itemValidators) Save() error { - for shardID, shardPubKeys := range wiv.validatorsPubKeys { - err := wiv.indexer.SaveShardValidatorsPubKeys(shardID, wiv.epoch, shardPubKeys) - if err != nil { - log.Warn("itemValidators.Save could not index validators public keys", - "for shard", shardID, - "error", err.Error()) - return err - } - } - - return nil + return wiv.indexer.SaveShardValidatorsPubKeys(wiv.validatorsPubKeys) } // IsInterfaceNil returns true if there is no value under the interface diff --git a/process/elasticproc/accounts/accountsProcessor.go b/process/elasticproc/accounts/accountsProcessor.go index 935a855a..bd0d5d16 100644 --- a/process/elasticproc/accounts/accountsProcessor.go +++ b/process/elasticproc/accounts/accountsProcessor.go @@ -8,7 +8,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" - "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-core-go/data/alteredAccount" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" @@ -42,7 +42,7 @@ func NewAccountsProcessor( } // GetAccounts will get accounts for regular operations and esdt operations -func (ap *accountsProcessor) GetAccounts(coreAlteredAccounts map[string]*outport.AlteredAccount) ([]*data.Account, []*data.AccountESDT) { +func (ap *accountsProcessor) GetAccounts(coreAlteredAccounts map[string]*alteredAccount.AlteredAccount) ([]*data.Account, []*data.AccountESDT) { regularAccountsToIndex := make([]*data.Account, 0) accountsToIndexESDT := make([]*data.AccountESDT, 0) @@ -57,7 +57,7 @@ func (ap *accountsProcessor) GetAccounts(coreAlteredAccounts map[string]*outport } func splitAlteredAccounts( - account *outport.AlteredAccount, + account *alteredAccount.AlteredAccount, ) ([]*data.Account, []*data.AccountESDT) { regularAccountsToIndex := make([]*data.Account, 0) accountsToIndexESDT := make([]*data.AccountESDT, 0) @@ -143,7 +143,7 @@ func (ap *accountsProcessor) PrepareRegularAccountsMap(timestamp uint64, account return accountsMap } -func (ap *accountsProcessor) addAdditionalDataInAccount(additionalData *outport.AdditionalAccountData, account *data.AccountInfo) { +func (ap *accountsProcessor) addAdditionalDataInAccount(additionalData *alteredAccount.AdditionalAccountData, account *data.AccountInfo) { if additionalData == nil { return } @@ -154,7 +154,7 @@ func (ap *accountsProcessor) addAdditionalDataInAccount(additionalData *outport. ap.addDeveloperRewardsInAccount(additionalData, account) } -func (ap *accountsProcessor) addDeveloperRewardsInAccount(additionalData *outport.AdditionalAccountData, account *data.AccountInfo) { +func (ap *accountsProcessor) addDeveloperRewardsInAccount(additionalData *alteredAccount.AdditionalAccountData, account *data.AccountInfo) { if additionalData.DeveloperRewards == "" { return } @@ -281,7 +281,7 @@ func (ap *accountsProcessor) getESDTInfo(accountESDT *data.AccountESDT) (*big.In return big.NewInt(0), "", nil, nil } - accountTokenData := &outport.AccountTokenData{} + accountTokenData := &alteredAccount.AccountTokenData{} for _, tokenData := range accountESDT.Account.Tokens { if tokenData.Identifier == accountESDT.TokenIdentifier && tokenData.Nonce == accountESDT.NFTNonce { accountTokenData = tokenData @@ -299,7 +299,7 @@ func (ap *accountsProcessor) getESDTInfo(accountESDT *data.AccountESDT) (*big.In } // PutTokenMedataDataInTokens will put the TokenMedata in provided tokens data -func (ap *accountsProcessor) PutTokenMedataDataInTokens(tokensData []*data.TokenInfo, coreAlteredAccounts map[string]*outport.AlteredAccount) { +func (ap *accountsProcessor) PutTokenMedataDataInTokens(tokensData []*data.TokenInfo, coreAlteredAccounts map[string]*alteredAccount.AlteredAccount) { for _, tokenData := range tokensData { if tokenData.Data != nil || tokenData.Nonce == 0 { continue @@ -320,8 +320,8 @@ func (ap *accountsProcessor) PutTokenMedataDataInTokens(tokensData []*data.Token func (ap *accountsProcessor) loadMetadataForToken( tokenData *data.TokenInfo, - coreAlteredAccounts map[string]*outport.AlteredAccount, -) (*outport.TokenMetaData, error) { + coreAlteredAccounts map[string]*alteredAccount.AlteredAccount, +) (*alteredAccount.TokenMetaData, error) { for _, account := range coreAlteredAccounts { for _, token := range account.Tokens { if tokenData.Token == token.Identifier && tokenData.Nonce == token.Nonce { diff --git a/process/elasticproc/accounts/accountsProcessor_test.go b/process/elasticproc/accounts/accountsProcessor_test.go index d5c731ef..422ee0af 100644 --- a/process/elasticproc/accounts/accountsProcessor_test.go +++ b/process/elasticproc/accounts/accountsProcessor_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/core" - "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-core-go/data/alteredAccount" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" @@ -86,11 +86,11 @@ func TestGetESDTInfo(t *testing.T) { tokenIdentifier := "token-001" wrapAccount := &data.AccountESDT{ - Account: &outport.AlteredAccount{ + Account: &alteredAccount.AlteredAccount{ Address: "", Balance: "1000", Nonce: 0, - Tokens: []*outport.AccountTokenData{ + Tokens: []*alteredAccount.AccountTokenData{ { Identifier: tokenIdentifier, Balance: "1000", @@ -114,11 +114,11 @@ func TestGetESDTInfoNFT(t *testing.T) { tokenIdentifier := "token-001" wrapAccount := &data.AccountESDT{ - Account: &outport.AlteredAccount{ + Account: &alteredAccount.AlteredAccount{ Address: "", Balance: "1", Nonce: 10, - Tokens: []*outport.AccountTokenData{ + Tokens: []*alteredAccount.AccountTokenData{ { Identifier: tokenIdentifier, Balance: "1", @@ -149,17 +149,17 @@ func TestGetESDTInfoNFTWithMetaData(t *testing.T) { tokenIdentifier := "token-001" wrapAccount := &data.AccountESDT{ - Account: &outport.AlteredAccount{ + Account: &alteredAccount.AlteredAccount{ Address: "", Balance: "1", Nonce: 1, - Tokens: []*outport.AccountTokenData{ + Tokens: []*alteredAccount.AccountTokenData{ { Identifier: tokenIdentifier, Balance: "1", Properties: "6f6b", Nonce: 10, - MetaData: &outport.TokenMetaData{ + MetaData: &alteredAccount.TokenMetaData{ Nonce: 10, Name: nftName, Creator: creator, @@ -187,8 +187,8 @@ func TestAccountsProcessor_GetAccountsEGLDAccounts(t *testing.T) { t.Parallel() addr := "aaaabbbb" - acc := &outport.AlteredAccount{} - alteredAccountsMap := map[string]*outport.AlteredAccount{ + acc := &alteredAccount.AlteredAccount{} + alteredAccountsMap := map[string]*alteredAccount.AlteredAccount{ addr: acc, } ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) @@ -207,16 +207,16 @@ func TestAccountsProcessor_GetAccountsESDTAccount(t *testing.T) { t.Parallel() addr := "aaaabbbb" - acc := &outport.AlteredAccount{ + acc := &alteredAccount.AlteredAccount{ Address: addr, Balance: "1", - Tokens: []*outport.AccountTokenData{ + Tokens: []*alteredAccount.AccountTokenData{ { Identifier: "token", }, }, } - alteredAccountsMap := map[string]*outport.AlteredAccount{ + alteredAccountsMap := map[string]*alteredAccount.AlteredAccount{ addr: acc, } ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) @@ -233,14 +233,14 @@ func TestAccountsProcessor_GetAccountsESDTAccountNewAccountShouldBeInRegularAcco t.Parallel() addr := "aaaabbbb" - acc := &outport.AlteredAccount{ - Tokens: []*outport.AccountTokenData{ + acc := &alteredAccount.AlteredAccount{ + Tokens: []*alteredAccount.AccountTokenData{ { Identifier: "token", }, }, } - alteredAccountsMap := map[string]*outport.AlteredAccount{ + alteredAccountsMap := map[string]*alteredAccount.AlteredAccount{ addr: acc, } ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) @@ -263,7 +263,7 @@ func TestAccountsProcessor_PrepareAccountsMapEGLD(t *testing.T) { addrBytes := bytes.Repeat([]byte{0}, 32) addr := hex.EncodeToString(addrBytes) - account := &outport.AlteredAccount{ + account := &alteredAccount.AlteredAccount{ Address: addr, Balance: "1000", Nonce: 1, @@ -298,15 +298,15 @@ func TestAccountsProcessor_PrepareAccountsMapESDT(t *testing.T) { addr := "aaaabbbb" - account := &outport.AlteredAccount{ + account := &alteredAccount.AlteredAccount{ Address: hex.EncodeToString([]byte(addr)), - Tokens: []*outport.AccountTokenData{ + Tokens: []*alteredAccount.AccountTokenData{ { Balance: "1000", Identifier: "token", Nonce: 15, Properties: "3032", - MetaData: &outport.TokenMetaData{ + MetaData: &alteredAccount.TokenMetaData{ Creator: "creator", }, }, @@ -315,7 +315,7 @@ func TestAccountsProcessor_PrepareAccountsMapESDT(t *testing.T) { Identifier: "token", Nonce: 16, Properties: "3032", - MetaData: &outport.TokenMetaData{ + MetaData: &alteredAccount.TokenMetaData{ Creator: "creator", }, }, @@ -403,7 +403,7 @@ func TestAccountsProcessor_PutTokenMedataDataInTokens(t *testing.T) { tokensInfo := []*data.TokenInfo{ {Data: nil}, {Nonce: 5, Data: &data.TokenMetaData{Creator: oldCreator}}, } - emptyAlteredAccounts := map[string]*outport.AlteredAccount{} + emptyAlteredAccounts := map[string]*alteredAccount.AlteredAccount{} ap.PutTokenMedataDataInTokens(tokensInfo, emptyAlteredAccounts) require.Empty(t, emptyAlteredAccounts) require.Empty(t, tokensInfo[0].Data) @@ -423,8 +423,8 @@ func TestAccountsProcessor_PutTokenMedataDataInTokens(t *testing.T) { }, } - alteredAccounts := map[string]*outport.AlteredAccount{ - "addr": {Tokens: []*outport.AccountTokenData{}}, + alteredAccounts := map[string]*alteredAccount.AlteredAccount{ + "addr": {Tokens: []*alteredAccount.AccountTokenData{}}, } ap.PutTokenMedataDataInTokens(tokensInfo, alteredAccounts) require.Empty(t, tokensInfo[0].Data) @@ -435,7 +435,7 @@ func TestAccountsProcessor_PutTokenMedataDataInTokens(t *testing.T) { ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) - metadata0, metadata1 := &outport.TokenMetaData{Creator: "creator 0"}, &outport.TokenMetaData{Creator: "creator 1"} + metadata0, metadata1 := &alteredAccount.TokenMetaData{Creator: "creator 0"}, &alteredAccount.TokenMetaData{Creator: "creator 1"} tokensInfo := []*data.TokenInfo{ { Nonce: 5, @@ -449,9 +449,9 @@ func TestAccountsProcessor_PutTokenMedataDataInTokens(t *testing.T) { }, } - alteredAccounts := map[string]*outport.AlteredAccount{ + alteredAccounts := map[string]*alteredAccount.AlteredAccount{ "addr0": { - Tokens: []*outport.AccountTokenData{ + Tokens: []*alteredAccount.AccountTokenData{ { Identifier: "token0-5t6y7u", Nonce: 5, @@ -478,14 +478,14 @@ func TestAddAdditionalDataIntoAccounts(t *testing.T) { ap, _ := NewAccountsProcessor(mock.NewPubkeyConverterMock(32), balanceConverter) account := &data.AccountInfo{} - ap.addAdditionalDataInAccount(&outport.AdditionalAccountData{ + ap.addAdditionalDataInAccount(&alteredAccount.AdditionalAccountData{ DeveloperRewards: "10000", }, account) require.Equal(t, "10000", account.DeveloperRewards) require.Equal(t, 0.000001, account.DeveloperRewardsNum) account = &data.AccountInfo{} - ap.addAdditionalDataInAccount(&outport.AdditionalAccountData{ + ap.addAdditionalDataInAccount(&alteredAccount.AdditionalAccountData{ DeveloperRewards: "", }, account) require.Equal(t, "", account.DeveloperRewards) @@ -494,7 +494,7 @@ func TestAddAdditionalDataIntoAccounts(t *testing.T) { account = &data.AccountInfo{ Address: "addr", } - ap.addAdditionalDataInAccount(&outport.AdditionalAccountData{ + ap.addAdditionalDataInAccount(&alteredAccount.AdditionalAccountData{ DeveloperRewards: "wrong", }, account) require.Equal(t, "", account.DeveloperRewards) diff --git a/process/elasticproc/block/blockProcessor.go b/process/elasticproc/block/blockProcessor.go index f61477c6..49ffcb2c 100644 --- a/process/elasticproc/block/blockProcessor.go +++ b/process/elasticproc/block/blockProcessor.go @@ -48,62 +48,56 @@ func NewBlockProcessor(hasher hashing.Hasher, marshalizer marshal.Marshalizer) ( } // PrepareBlockForDB will prepare a database block and serialize it for database -func (bp *blockProcessor) PrepareBlockForDB( - headerHash []byte, - header coreData.HeaderHandler, - signersIndexes []uint64, - body *block.Body, - notarizedHeadersHashes []string, - gasConsumptionData outport.HeaderGasConsumption, - sizeTxs int, - pool *outport.Pool, -) (*data.Block, error) { - if check.IfNil(header) { +func (bp *blockProcessor) PrepareBlockForDB(obh *outport.OutportBlockWithHeader) (*data.Block, error) { + if check.IfNil(obh.Header) { return nil, indexer.ErrNilHeaderHandler } - if body == nil { + if obh.BlockData.Body == nil { return nil, indexer.ErrNilBlockBody } - blockSizeInBytes, err := bp.computeBlockSize(header, body) + blockSizeInBytes, err := bp.computeBlockSize(obh.Header, obh.BlockData.Body) if err != nil { return nil, err } - miniblocksHashes := bp.getEncodedMBSHashes(body) - leaderIndex := bp.getLeaderIndex(signersIndexes) + // TODO compute size of transactions + sizeTxs := obh.OutportBlock.TransactionPool.Size() - numTxs, notarizedTxs := getTxsCount(header) + miniblocksHashes := bp.getEncodedMBSHashes(obh.BlockData.Body) + leaderIndex := bp.getLeaderIndex(obh.SignersIndexes) + + numTxs, notarizedTxs := getTxsCount(obh.Header) elasticBlock := &data.Block{ - Nonce: header.GetNonce(), - Round: header.GetRound(), - Epoch: header.GetEpoch(), - ShardID: header.GetShardID(), - Hash: hex.EncodeToString(headerHash), + Nonce: obh.Header.GetNonce(), + Round: obh.Header.GetRound(), + Epoch: obh.Header.GetEpoch(), + ShardID: obh.Header.GetShardID(), + Hash: hex.EncodeToString(obh.BlockData.HeaderHash), MiniBlocksHashes: miniblocksHashes, - NotarizedBlocksHashes: notarizedHeadersHashes, + NotarizedBlocksHashes: obh.NotarizedHeadersHashes, Proposer: leaderIndex, - Validators: signersIndexes, - PubKeyBitmap: hex.EncodeToString(header.GetPubKeysBitmap()), + Validators: obh.SignersIndexes, + PubKeyBitmap: hex.EncodeToString(obh.Header.GetPubKeysBitmap()), Size: int64(blockSizeInBytes), SizeTxs: int64(sizeTxs), - Timestamp: time.Duration(header.GetTimeStamp()), + Timestamp: time.Duration(obh.Header.GetTimeStamp()), TxCount: numTxs, NotarizedTxsCount: notarizedTxs, - StateRootHash: hex.EncodeToString(header.GetRootHash()), - PrevHash: hex.EncodeToString(header.GetPrevHash()), - SearchOrder: computeBlockSearchOrder(header), - EpochStartBlock: header.IsStartOfEpochBlock(), - GasProvided: gasConsumptionData.GasProvided, - GasRefunded: gasConsumptionData.GasRefunded, - GasPenalized: gasConsumptionData.GasPenalized, - MaxGasLimit: gasConsumptionData.MaxGasPerBlock, - AccumulatedFees: converters.BigIntToString(header.GetAccumulatedFees()), - DeveloperFees: converters.BigIntToString(header.GetDeveloperFees()), + StateRootHash: hex.EncodeToString(obh.Header.GetRootHash()), + PrevHash: hex.EncodeToString(obh.Header.GetPrevHash()), + SearchOrder: computeBlockSearchOrder(obh.Header), + EpochStartBlock: obh.Header.IsStartOfEpochBlock(), + GasProvided: obh.HeaderGasConsumption.GasProvided, + GasRefunded: obh.HeaderGasConsumption.GasRefunded, + GasPenalized: obh.HeaderGasConsumption.GasPenalized, + MaxGasLimit: obh.HeaderGasConsumption.MaxGasPerBlock, + AccumulatedFees: converters.BigIntToString(obh.Header.GetAccumulatedFees()), + DeveloperFees: converters.BigIntToString(obh.Header.GetDeveloperFees()), } - additionalData := header.GetAdditionalData() - if header.GetAdditionalData() != nil { + additionalData := obh.Header.GetAdditionalData() + if obh.Header.GetAdditionalData() != nil { elasticBlock.ScheduledData = &data.ScheduledData{ ScheduledRootHash: hex.EncodeToString(additionalData.GetScheduledRootHash()), ScheduledAccumulatedFees: converters.BigIntToString(additionalData.GetScheduledAccumulatedFees()), @@ -114,8 +108,8 @@ func (bp *blockProcessor) PrepareBlockForDB( } } - bp.addEpochStartInfoForMeta(header, elasticBlock) - putMiniblocksDetailsInBlock(header, elasticBlock, pool, body) + bp.addEpochStartInfoForMeta(obh.Header, elasticBlock) + putMiniblocksDetailsInBlock(obh.Header, elasticBlock, obh.TransactionPool, obh.BlockData.Body) return elasticBlock, nil } @@ -233,7 +227,7 @@ func (bp *blockProcessor) getEncodedMBSHashes(body *block.Body) []string { return miniblocksHashes } -func putMiniblocksDetailsInBlock(header coreData.HeaderHandler, block *data.Block, pool *outport.Pool, body *block.Body) { +func putMiniblocksDetailsInBlock(header coreData.HeaderHandler, block *data.Block, pool *outport.TransactionPool, body *block.Body) { mbHeaders := header.GetMiniBlockHeaderHandlers() for idx, mbHeader := range mbHeaders { @@ -257,8 +251,8 @@ func putMiniblocksDetailsInBlock(header coreData.HeaderHandler, block *data.Bloc } } -func extractExecutionOrderIndicesFromPool(mbHeader coreData.MiniBlockHeaderHandler, txsHashes [][]byte, pool *outport.Pool) []int { - txsMap := getTxsMap(nodeBlock.Type(mbHeader.GetTypeInt32()), pool) +func extractExecutionOrderIndicesFromPool(mbHeader coreData.MiniBlockHeaderHandler, txsHashes [][]byte, pool *outport.TransactionPool) []int { + mbType := mbHeader.GetTypeInt32() executionOrderTxsIndices := make([]int, len(txsHashes)) indexOfFirstTxProcessed, indexOfLastTxProcessed := mbHeader.GetIndexOfFirstTxProcessed(), mbHeader.GetIndexOfLastTxProcessed() for idx, txHash := range txsHashes { @@ -268,19 +262,46 @@ func extractExecutionOrderIndicesFromPool(mbHeader coreData.MiniBlockHeaderHandl continue } - tx, found := txsMap[string(txHash)] + executionOrder, found := getExecutionOrderForTx(txHash, mbType, pool) if !found { log.Warn("blockProcessor.extractExecutionOrderIndicesFromPool cannot find tx in pool", "txHash", hex.EncodeToString(txHash)) executionOrderTxsIndices[idx] = notFound continue } - executionOrderTxsIndices[idx] = tx.GetExecutionOrder() + executionOrderTxsIndices[idx] = int(executionOrder) } return executionOrderTxsIndices } +type executionOrderHandler interface { + GetExecutionOrder() uint32 +} + +func getExecutionOrderForTx(txHash []byte, mbType int32, pool *outport.TransactionPool) (uint32, bool) { + var tx executionOrderHandler + var found bool + + switch mbType { + case nodeBlock.TxBlock: + tx, found = pool.Transactions[hex.EncodeToString(txHash)] + case nodeBlock.InvalidBlock: + tx, found = pool.InvalidTxs[hex.EncodeToString(txHash)] + case nodeBlock.RewardsBlock: + tx, found = pool.Rewards[hex.EncodeToString(txHash)] + case nodeBlock.SmartContractResultBlock: + tx, found = pool.SmartContractResults[hex.EncodeToString(txHash)] + default: + return 0, false + } + + if !found { + return 0, false + } + return tx.GetExecutionOrder(), true +} + func (bp *blockProcessor) computeBlockSize(header coreData.HeaderHandler, body *block.Body) (int, error) { headerBytes, err := bp.marshalizer.Marshal(header) if err != nil { @@ -332,21 +353,6 @@ func (bp *blockProcessor) ComputeHeaderHash(header coreData.HeaderHandler) ([]by return core.CalculateHash(bp.marshalizer, bp.hasher, header) } -func getTxsMap(mbType nodeBlock.Type, pool *outport.Pool) map[string]coreData.TransactionHandlerWithGasUsedAndFee { - switch mbType { - case nodeBlock.TxBlock: - return pool.Txs - case nodeBlock.InvalidBlock: - return pool.Invalid - case nodeBlock.RewardsBlock: - return pool.Rewards - case nodeBlock.SmartContractResultBlock: - return pool.Scrs - default: - return make(map[string]coreData.TransactionHandlerWithGasUsedAndFee) - } -} - func hexEncodeSlice(slice [][]byte) []string { res := make([]string, 0, len(slice)) for _, s := range slice { diff --git a/process/elasticproc/block/blockProcessor_test.go b/process/elasticproc/block/blockProcessor_test.go index 14b05780..cd03d290 100644 --- a/process/elasticproc/block/blockProcessor_test.go +++ b/process/elasticproc/block/blockProcessor_test.go @@ -7,7 +7,6 @@ import ( "testing" "github.com/multiversx/mx-chain-core-go/core" - coreData "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/hashing" @@ -60,20 +59,28 @@ func TestBlockProcessor_PrepareBlockForDBShouldWork(t *testing.T) { bp, _ := NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) - dbBlock, err := bp.PrepareBlockForDB( - []byte("hash"), - &dataBlock.Header{}, - []uint64{0, 1, 2}, - &dataBlock.Body{ - MiniBlocks: dataBlock.MiniBlockSlice{ - { - ReceiverShardID: 1, - }, - { - ReceiverShardID: 2, + outportBlockWithHeader := &outport.OutportBlockWithHeader{ + Header: &dataBlock.Header{}, + OutportBlock: outport.OutportBlock{ + BlockData: &outport.BlockData{ + HeaderHash: []byte("hash"), + Body: &dataBlock.Body{ + MiniBlocks: dataBlock.MiniBlockSlice{ + { + ReceiverShardID: 1, + }, + { + ReceiverShardID: 2, + }, + }, }, }, - }, nil, outport.HeaderGasConsumption{}, 0, &outport.Pool{}) + SignersIndexes: []uint64{0, 1, 2}, + TransactionPool: &outport.TransactionPool{}, + HeaderGasConsumption: &outport.HeaderGasConsumption{}, + }, + } + dbBlock, err := bp.PrepareBlockForDB(outportBlockWithHeader) require.Nil(t, err) expectedBlock := &data.Block{ @@ -95,7 +102,8 @@ func TestBlockProcessor_PrepareBlockForDBNilHeader(t *testing.T) { bp, _ := NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) - dbBlock, err := bp.PrepareBlockForDB([]byte("hash"), nil, nil, &dataBlock.Body{}, nil, outport.HeaderGasConsumption{}, 0, &outport.Pool{}) + outportBlockWithHeader := &outport.OutportBlockWithHeader{} + dbBlock, err := bp.PrepareBlockForDB(outportBlockWithHeader) require.Equal(t, indexer.ErrNilHeaderHandler, err) require.Nil(t, dbBlock) } @@ -105,7 +113,10 @@ func TestBlockProcessor_PrepareBlockForDBNilBody(t *testing.T) { bp, _ := NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) - dbBlock, err := bp.PrepareBlockForDB([]byte("hash"), &dataBlock.MetaBlock{}, nil, nil, nil, outport.HeaderGasConsumption{}, 0, &outport.Pool{}) + outportBlockWithHeader := &outport.OutportBlockWithHeader{ + Header: &dataBlock.MetaBlock{}, + } + dbBlock, err := bp.PrepareBlockForDB(outportBlockWithHeader) require.Equal(t, indexer.ErrNilBlockBody, err) require.Nil(t, dbBlock) } @@ -120,7 +131,19 @@ func TestBlockProcessor_PrepareBlockForDBMarshalFailHeader(t *testing.T) { }, }) - dbBlock, err := bp.PrepareBlockForDB([]byte("hash"), &dataBlock.MetaBlock{}, nil, &dataBlock.Body{}, nil, outport.HeaderGasConsumption{}, 0, &outport.Pool{}) + outportBlockWithHeader := &outport.OutportBlockWithHeader{ + Header: &dataBlock.Header{}, + OutportBlock: outport.OutportBlock{ + BlockData: &outport.BlockData{ + HeaderHash: []byte("hash"), + Body: &dataBlock.Body{}, + }, + SignersIndexes: []uint64{0, 1, 2}, + TransactionPool: &outport.TransactionPool{}, + HeaderGasConsumption: &outport.HeaderGasConsumption{}, + }, + } + dbBlock, err := bp.PrepareBlockForDB(outportBlockWithHeader) require.Equal(t, expectedErr, err) require.Nil(t, dbBlock) } @@ -142,7 +165,19 @@ func TestBlockProcessor_PrepareBlockForDBMarshalFailBlock(t *testing.T) { }, }) - dbBlock, err := bp.PrepareBlockForDB([]byte("hash"), &dataBlock.MetaBlock{}, nil, &dataBlock.Body{}, nil, outport.HeaderGasConsumption{}, 0, &outport.Pool{}) + outportBlockWithHeader := &outport.OutportBlockWithHeader{ + Header: &dataBlock.Header{}, + OutportBlock: outport.OutportBlock{ + BlockData: &outport.BlockData{ + HeaderHash: []byte("hash"), + Body: &dataBlock.Body{}, + }, + SignersIndexes: []uint64{0, 1, 2}, + TransactionPool: &outport.TransactionPool{}, + HeaderGasConsumption: &outport.HeaderGasConsumption{}, + }, + } + dbBlock, err := bp.PrepareBlockForDB(outportBlockWithHeader) require.Equal(t, expectedErr, err) require.Nil(t, dbBlock) } @@ -163,54 +198,66 @@ func TestBlockProcessor_PrepareBlockForDBEpochStartMeta(t *testing.T) { bp, _ := NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) - dbBlock, err := bp.PrepareBlockForDB([]byte("hash"), &dataBlock.MetaBlock{ - TxCount: 1000, - EpochStart: dataBlock.EpochStart{ - LastFinalizedHeaders: []dataBlock.EpochStartShardData{{ - ShardID: 1, - Nonce: 1234, - Round: 1500, - Epoch: 10, - HeaderHash: []byte("hh"), - RootHash: []byte("rh"), - ScheduledRootHash: []byte("sch"), - FirstPendingMetaBlock: []byte("fpmb"), - LastFinishedMetaBlock: []byte("lfmb"), - PendingMiniBlockHeaders: []dataBlock.MiniBlockHeader{ - { - Hash: []byte("mbh"), - SenderShardID: 0, - ReceiverShardID: 1, - Type: dataBlock.TxBlock, - Reserved: []byte("rrr"), + outportBlockWithHeader := &outport.OutportBlockWithHeader{ + Header: &dataBlock.MetaBlock{ + TxCount: 1000, + EpochStart: dataBlock.EpochStart{ + LastFinalizedHeaders: []dataBlock.EpochStartShardData{{ + ShardID: 1, + Nonce: 1234, + Round: 1500, + Epoch: 10, + HeaderHash: []byte("hh"), + RootHash: []byte("rh"), + ScheduledRootHash: []byte("sch"), + FirstPendingMetaBlock: []byte("fpmb"), + LastFinishedMetaBlock: []byte("lfmb"), + PendingMiniBlockHeaders: []dataBlock.MiniBlockHeader{ + { + Hash: []byte("mbh"), + SenderShardID: 0, + ReceiverShardID: 1, + Type: dataBlock.TxBlock, + Reserved: []byte("rrr"), + }, }, + }}, + Economics: dataBlock.Economics{ + TotalSupply: big.NewInt(100), + TotalToDistribute: big.NewInt(55), + TotalNewlyMinted: big.NewInt(20), + RewardsPerBlock: big.NewInt(15), + RewardsForProtocolSustainability: big.NewInt(2), + NodePrice: big.NewInt(10), + PrevEpochStartRound: 222, + PrevEpochStartHash: []byte("prevEpoch"), }, - }}, - Economics: dataBlock.Economics{ - TotalSupply: big.NewInt(100), - TotalToDistribute: big.NewInt(55), - TotalNewlyMinted: big.NewInt(20), - RewardsPerBlock: big.NewInt(15), - RewardsForProtocolSustainability: big.NewInt(2), - NodePrice: big.NewInt(10), - PrevEpochStartRound: 222, - PrevEpochStartHash: []byte("prevEpoch"), - }, - }, - MiniBlockHeaders: []dataBlock.MiniBlockHeader{ - { - TxCount: 50, }, - { - TxCount: 120, + MiniBlockHeaders: []dataBlock.MiniBlockHeader{ + { + TxCount: 50, + }, + { + TxCount: 120, + }, }, }, - }, nil, &dataBlock.Body{ - MiniBlocks: []*dataBlock.MiniBlock{ - {}, - {}, + OutportBlock: outport.OutportBlock{ + BlockData: &outport.BlockData{ + HeaderHash: []byte("hash"), + Body: &dataBlock.Body{ + MiniBlocks: []*dataBlock.MiniBlock{ + {}, + {}, + }, + }, + }, + TransactionPool: &outport.TransactionPool{}, + HeaderGasConsumption: &outport.HeaderGasConsumption{}, }, - }, nil, outport.HeaderGasConsumption{}, 0, &outport.Pool{}) + } + + dbBlock, err := bp.PrepareBlockForDB(outportBlockWithHeader) require.Equal(t, nil, err) require.Equal(t, &data.Block{ Nonce: 0, @@ -302,70 +349,84 @@ func TestBlockProcessor_PrepareBlockForDBMiniBlocksDetails(t *testing.T) { mbhrBytes, _ := gogoMarshaller.Marshal(mbhr) txHash, notExecutedTxHash, notFoundTxHash, invalidTxHash, rewardsTxHash, scrHash := "tx", "notExecuted", "notFound", "invalid", "reward", "scr" - dbBlock, err := bp.PrepareBlockForDB([]byte("hash"), &dataBlock.Header{ - TxCount: 5, - MiniBlockHeaders: []dataBlock.MiniBlockHeader{ - { - TxCount: 1, - Type: dataBlock.TxBlock, - Reserved: mbhrBytes, - }, - { - TxCount: 1, - Type: dataBlock.RewardsBlock, - }, - { - TxCount: 1, - Type: dataBlock.InvalidBlock, - }, - { - TxCount: 1, - Type: dataBlock.SmartContractResultBlock, - }, - }, - }, nil, &dataBlock.Body{ - MiniBlocks: []*dataBlock.MiniBlock{ - { - Type: dataBlock.TxBlock, - TxHashes: [][]byte{[]byte(txHash), []byte(notFoundTxHash), []byte(notExecutedTxHash)}, - }, - { - Type: dataBlock.RewardsBlock, - TxHashes: [][]byte{[]byte(rewardsTxHash)}, - }, - { - Type: dataBlock.InvalidBlock, - TxHashes: [][]byte{[]byte(invalidTxHash)}, - }, - { - Type: dataBlock.SmartContractResultBlock, - TxHashes: [][]byte{[]byte(scrHash)}, - }, - }, - }, nil, outport.HeaderGasConsumption{}, 0, &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - txHash: &outport.TransactionHandlerWithGasAndFee{ - ExecutionOrder: 2, - }, - notExecutedTxHash: &outport.TransactionHandlerWithGasAndFee{ - ExecutionOrder: 0, + + outportBlockWithHeader := &outport.OutportBlockWithHeader{ + Header: &dataBlock.Header{ + TxCount: 5, + MiniBlockHeaders: []dataBlock.MiniBlockHeader{ + { + TxCount: 1, + Type: dataBlock.TxBlock, + Reserved: mbhrBytes, + }, + { + TxCount: 1, + Type: dataBlock.RewardsBlock, + }, + { + TxCount: 1, + Type: dataBlock.InvalidBlock, + }, + { + TxCount: 1, + Type: dataBlock.SmartContractResultBlock, + }, }, }, - Rewards: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - rewardsTxHash: &outport.TransactionHandlerWithGasAndFee{ - ExecutionOrder: 3, + OutportBlock: outport.OutportBlock{ + BlockData: &outport.BlockData{ + HeaderHash: []byte("hash"), + Body: &dataBlock.Body{ + MiniBlocks: []*dataBlock.MiniBlock{ + { + Type: dataBlock.TxBlock, + TxHashes: [][]byte{[]byte(txHash), []byte(notFoundTxHash), []byte(notExecutedTxHash)}, + }, + { + Type: dataBlock.RewardsBlock, + TxHashes: [][]byte{[]byte(rewardsTxHash)}, + }, + { + Type: dataBlock.InvalidBlock, + TxHashes: [][]byte{[]byte(invalidTxHash)}, + }, + { + Type: dataBlock.SmartContractResultBlock, + TxHashes: [][]byte{[]byte(scrHash)}, + }, + }, + }, }, - }, - Invalid: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - invalidTxHash: &outport.TransactionHandlerWithGasAndFee{ - ExecutionOrder: 1, - }}, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - scrHash: &outport.TransactionHandlerWithGasAndFee{ - ExecutionOrder: 0, + SignersIndexes: []uint64{0, 1, 2}, + TransactionPool: &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + txHash: { + ExecutionOrder: 2, + }, + notExecutedTxHash: { + ExecutionOrder: 0, + }, + }, + Rewards: map[string]*outport.RewardInfo{ + rewardsTxHash: { + ExecutionOrder: 3, + }, + }, + InvalidTxs: map[string]*outport.TxInfo{ + invalidTxHash: { + ExecutionOrder: 1, + }}, + SmartContractResults: map[string]*outport.SCRInfo{ + scrHash: { + ExecutionOrder: 0, + }, + }, }, + HeaderGasConsumption: &outport.HeaderGasConsumption{}, }, - }) + } + + dbBlock, err := bp.PrepareBlockForDB(outportBlockWithHeader) require.Nil(t, err) require.Equal(t, &data.Block{ diff --git a/process/elasticproc/converters/tokenMetaData.go b/process/elasticproc/converters/tokenMetaData.go index 11ae9fbe..067e5414 100644 --- a/process/elasticproc/converters/tokenMetaData.go +++ b/process/elasticproc/converters/tokenMetaData.go @@ -6,7 +6,7 @@ import ( "fmt" "strings" - "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-core-go/data/alteredAccount" "github.com/multiversx/mx-chain-es-indexer-go/data" ) @@ -20,7 +20,7 @@ const ( ) // PrepareTokenMetaData will prepare the token metadata in a friendly format for database -func PrepareTokenMetaData(tokenMetadata *outport.TokenMetaData) *data.TokenMetaData { +func PrepareTokenMetaData(tokenMetadata *alteredAccount.TokenMetaData) *data.TokenMetaData { if tokenMetadata == nil { return nil } diff --git a/process/elasticproc/converters/tokenMetaData_test.go b/process/elasticproc/converters/tokenMetaData_test.go index 60bc95de..b68c6551 100644 --- a/process/elasticproc/converters/tokenMetaData_test.go +++ b/process/elasticproc/converters/tokenMetaData_test.go @@ -3,7 +3,7 @@ package converters import ( "testing" - "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-core-go/data/alteredAccount" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/stretchr/testify/require" ) @@ -26,7 +26,7 @@ func TestPrepareTokenMetaData(t *testing.T) { WhiteListedStorage: true, } - result := PrepareTokenMetaData(&outport.TokenMetaData{ + result := PrepareTokenMetaData(&alteredAccount.TokenMetaData{ Nonce: 2, Name: "token", Creator: "creator", diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index 95bd2946..a3559fbb 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -5,12 +5,13 @@ import ( "encoding/hex" "fmt" - "github.com/elastic/go-elasticsearch/v7/esapi" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" coreData "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/data/alteredAccount" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-es-indexer-go/data" elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" @@ -244,21 +245,12 @@ func getTemplateByName(templateName string, templateList map[string]*bytes.Buffe } // SaveHeader will prepare and save information about a header in elasticsearch server -func (ei *elasticProcessor) SaveHeader( - headerHash []byte, - header coreData.HeaderHandler, - signersIndexes []uint64, - body *block.Body, - notarizedHeadersHashes []string, - gasConsumptionData outport.HeaderGasConsumption, - txsSize int, - pool *outport.Pool, -) error { +func (ei *elasticProcessor) SaveHeader(outportBlockWithHeader *outport.OutportBlockWithHeader) error { if !ei.isIndexEnabled(elasticIndexer.BlockIndex) { return nil } - elasticBlock, err := ei.blockProc.PrepareBlockForDB(headerHash, header, signersIndexes, body, notarizedHeadersHashes, gasConsumptionData, txsSize, pool) + elasticBlock, err := ei.blockProc.PrepareBlockForDB(outportBlockWithHeader) if err != nil { return err } @@ -269,7 +261,7 @@ func (ei *elasticProcessor) SaveHeader( return err } - err = ei.indexEpochInfoData(header, buffSlice) + err = ei.indexEpochInfoData(outportBlockWithHeader.Header, buffSlice) if err != nil { return err } @@ -412,26 +404,19 @@ func (ei *elasticProcessor) miniblocksInDBMap(mbs []*data.Miniblock) (map[string } // SaveTransactions will prepare and save information about a transactions in elasticsearch server -func (ei *elasticProcessor) SaveTransactions( - body *block.Body, - header coreData.HeaderHandler, - pool *outport.Pool, - coreAlteredAccounts map[string]*outport.AlteredAccount, - isImportDB bool, - numOfShards uint32, -) error { - headerTimestamp := header.GetTimeStamp() +func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader) error { + headerTimestamp := obh.Header.GetTimeStamp() - preparedResults := ei.transactionsProc.PrepareTransactionsForDatabase(body, header, pool, isImportDB, numOfShards) - logsData := ei.logsAndEventsProc.ExtractDataFromLogs(pool.Logs, preparedResults, headerTimestamp, header.GetShardID(), numOfShards) + preparedResults := ei.transactionsProc.PrepareTransactionsForDatabase(obh.BlockData.Body, obh.Header, obh.TransactionPool, obh.IsImportDB, obh.NumberOfShards) + logsData := ei.logsAndEventsProc.ExtractDataFromLogs(obh.TransactionPool.Logs, preparedResults, headerTimestamp, obh.Header.GetShardID(), obh.NumberOfShards) buffers := data.NewBufferSlice(ei.bulkRequestMaxSize) - err := ei.indexTransactions(preparedResults.Transactions, preparedResults.TxHashStatus, header, buffers) + err := ei.indexTransactions(preparedResults.Transactions, preparedResults.TxHashStatus, obh.Header, buffers) if err != nil { return err } - err = ei.prepareAndIndexOperations(preparedResults.Transactions, preparedResults.TxHashStatus, header, preparedResults.ScResults, buffers, isImportDB) + err = ei.prepareAndIndexOperations(preparedResults.Transactions, preparedResults.TxHashStatus, obh.Header, preparedResults.ScResults, buffers, obh.IsImportDB) if err != nil { return err } @@ -441,12 +426,12 @@ func (ei *elasticProcessor) SaveTransactions( return err } - err = ei.indexNFTCreateInfo(logsData.Tokens, coreAlteredAccounts, buffers) + err = ei.indexNFTCreateInfo(logsData.Tokens, obh.AlteredAccounts, buffers) if err != nil { return err } - err = ei.prepareAndIndexLogs(pool.Logs, headerTimestamp, buffers) + err = ei.prepareAndIndexLogs(obh.TransactionPool.Logs, headerTimestamp, buffers) if err != nil { return err } @@ -462,7 +447,7 @@ func (ei *elasticProcessor) SaveTransactions( } tagsCount := tags.NewTagsCount() - err = ei.indexAlteredAccounts(headerTimestamp, logsData.NFTsDataUpdates, coreAlteredAccounts, buffers, tagsCount, header.GetShardID()) + err = ei.indexAlteredAccounts(headerTimestamp, logsData.NFTsDataUpdates, obh.AlteredAccounts, buffers, tagsCount, obh.Header.GetShardID()) if err != nil { return err } @@ -533,7 +518,7 @@ func (ei *elasticProcessor) indexTransactionsFeeData(txsHashFeeData map[string]* return ei.transactionsProc.SerializeTransactionsFeeData(txsHashFeeData, buffSlice, elasticIndexer.OperationsIndex) } -func (ei *elasticProcessor) prepareAndIndexLogs(logsAndEvents []*coreData.LogData, timestamp uint64, buffSlice *data.BufferSlice) error { +func (ei *elasticProcessor) prepareAndIndexLogs(logsAndEvents map[string]*transaction.Log, timestamp uint64, buffSlice *data.BufferSlice) error { if !ei.isIndexEnabled(elasticIndexer.LogsIndex) { return nil } @@ -582,12 +567,12 @@ func (ei *elasticProcessor) prepareAndIndexOperations( } // SaveValidatorsRating will save validators rating -func (ei *elasticProcessor) SaveValidatorsRating(index string, validatorsRatingInfo []*data.ValidatorRatingInfo) error { +func (ei *elasticProcessor) SaveValidatorsRating(ratingData *outport.ValidatorsRating) error { if !ei.isIndexEnabled(elasticIndexer.RatingIndex) { return nil } - buffSlice, err := ei.validatorsProc.SerializeValidatorsRating(index, validatorsRatingInfo) + buffSlice, err := ei.validatorsProc.SerializeValidatorsRating(ratingData) if err != nil { return err } @@ -596,33 +581,26 @@ func (ei *elasticProcessor) SaveValidatorsRating(index string, validatorsRatingI } // SaveShardValidatorsPubKeys will prepare and save information about a shard validators public keys in elasticsearch server -func (ei *elasticProcessor) SaveShardValidatorsPubKeys(shardID, epoch uint32, shardValidatorsPubKeys [][]byte) error { +func (ei *elasticProcessor) SaveShardValidatorsPubKeys(validatorsPubKeys *outport.ValidatorsPubKeys) error { if !ei.isIndexEnabled(elasticIndexer.ValidatorsIndex) { return nil } - validatorsPubKeys := ei.validatorsProc.PrepareValidatorsPublicKeys(shardValidatorsPubKeys) - buff, err := ei.validatorsProc.SerializeValidatorsPubKeys(validatorsPubKeys) + buffSlice, err := ei.validatorsProc.PrepareAnSerializeValidatorsPubKeys(validatorsPubKeys) if err != nil { return err } - req := &esapi.IndexRequest{ - Index: elasticIndexer.ValidatorsIndex, - DocumentID: fmt.Sprintf("%d_%d", shardID, epoch), - Body: bytes.NewReader(buff.Bytes()), - } - - return ei.elasticClient.DoRequest(req) + return ei.doBulkRequests(elasticIndexer.ValidatorsIndex, buffSlice) } // SaveRoundsInfo will prepare and save information about a slice of rounds in elasticsearch server -func (ei *elasticProcessor) SaveRoundsInfo(info []*data.RoundInfo) error { +func (ei *elasticProcessor) SaveRoundsInfo(rounds *outport.RoundsInfo) error { if !ei.isIndexEnabled(elasticIndexer.RoundsIndex) { return nil } - buff := ei.statisticsProc.SerializeRoundsInfo(info) + buff := ei.statisticsProc.SerializeRoundsInfo(rounds) return ei.elasticClient.DoBulkRequest(buff, elasticIndexer.RoundsIndex) } @@ -630,7 +608,7 @@ func (ei *elasticProcessor) SaveRoundsInfo(info []*data.RoundInfo) error { func (ei *elasticProcessor) indexAlteredAccounts( timestamp uint64, updatesNFTsData []*data.NFTDataUpdate, - coreAlteredAccounts map[string]*outport.AlteredAccount, + coreAlteredAccounts map[string]*alteredAccount.AlteredAccount, buffSlice *data.BufferSlice, tagsCount data.CountTags, shardID uint32, @@ -705,7 +683,7 @@ func (ei *elasticProcessor) indexAccountsESDT( return ei.accountsProc.SerializeAccountsESDT(accountsESDTMap, updatesNFTsData, buffSlice, elasticIndexer.AccountsESDTIndex) } -func (ei *elasticProcessor) indexNFTCreateInfo(tokensData data.TokensHandler, coreAlteredAccounts map[string]*outport.AlteredAccount, buffSlice *data.BufferSlice) error { +func (ei *elasticProcessor) indexNFTCreateInfo(tokensData data.TokensHandler, coreAlteredAccounts map[string]*alteredAccount.AlteredAccount, buffSlice *data.BufferSlice) error { shouldSkipIndex := !ei.isIndexEnabled(elasticIndexer.TokensIndex) || tokensData.Len() == 0 if shouldSkipIndex { return nil diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index 4cfcb78a..beb641d4 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -5,8 +5,10 @@ import ( "github.com/elastic/go-elasticsearch/v7/esapi" coreData "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/data/alteredAccount" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokeninfo" ) @@ -31,11 +33,11 @@ type DatabaseClientHandler interface { // DBAccountHandler defines the actions that an accounts' handler should do type DBAccountHandler interface { - GetAccounts(coreAlteredAccounts map[string]*outport.AlteredAccount) ([]*data.Account, []*data.AccountESDT) + GetAccounts(coreAlteredAccounts map[string]*alteredAccount.AlteredAccount) ([]*data.Account, []*data.AccountESDT) PrepareRegularAccountsMap(timestamp uint64, accounts []*data.Account, shardID uint32) map[string]*data.AccountInfo PrepareAccountsMapESDT(timestamp uint64, accounts []*data.AccountESDT, tagsCount data.CountTags, shardID uint32) (map[string]*data.AccountInfo, data.TokensHandler) PrepareAccountsHistory(timestamp uint64, accounts map[string]*data.AccountInfo, shardID uint32) map[string]*data.AccountBalanceHistory - PutTokenMedataDataInTokens(tokensData []*data.TokenInfo, coreAlteredAccounts map[string]*outport.AlteredAccount) + PutTokenMedataDataInTokens(tokensData []*data.TokenInfo, coreAlteredAccounts map[string]*alteredAccount.AlteredAccount) SerializeAccountsHistory(accounts map[string]*data.AccountBalanceHistory, buffSlice *data.BufferSlice, index string) error SerializeAccounts(accounts map[string]*data.AccountInfo, buffSlice *data.BufferSlice, index string) error @@ -46,16 +48,7 @@ type DBAccountHandler interface { // DBBlockHandler defines the actions that a block handler should do type DBBlockHandler interface { - PrepareBlockForDB( - headerHash []byte, - header coreData.HeaderHandler, - signersIndexes []uint64, - body *block.Body, - notarizedHeadersHashes []string, - gasConsumptionData outport.HeaderGasConsumption, - sizeTxs int, - pool *outport.Pool, - ) (*data.Block, error) + PrepareBlockForDB(obh *outport.OutportBlockWithHeader) (*data.Block, error) ComputeHeaderHash(header coreData.HeaderHandler) ([]byte, error) SerializeEpochInfoData(header coreData.HeaderHandler, buffSlice *data.BufferSlice, index string) error @@ -67,7 +60,7 @@ type DBTransactionsHandler interface { PrepareTransactionsForDatabase( body *block.Body, header coreData.HeaderHandler, - pool *outport.Pool, + pool *outport.TransactionPool, isImportDB bool, numOfShards uint32, ) *data.PreparedResults @@ -89,21 +82,20 @@ type DBMiniblocksHandler interface { // DBStatisticsHandler defines the actions that a database statistics handler should do type DBStatisticsHandler interface { - SerializeRoundsInfo(roundsInfo []*data.RoundInfo) *bytes.Buffer + SerializeRoundsInfo(rounds *outport.RoundsInfo) *bytes.Buffer } // DBValidatorsHandler defines the actions that a validators handler should do type DBValidatorsHandler interface { - PrepareValidatorsPublicKeys(shardValidatorsPubKeys [][]byte) *data.ValidatorsPublicKeys - SerializeValidatorsPubKeys(validatorsPubKeys *data.ValidatorsPublicKeys) (*bytes.Buffer, error) - SerializeValidatorsRating(index string, validatorsRatingInfo []*data.ValidatorRatingInfo) ([]*bytes.Buffer, error) + PrepareAnSerializeValidatorsPubKeys(validatorsPubKeys *outport.ValidatorsPubKeys) ([]*bytes.Buffer, error) + SerializeValidatorsRating(ratingData *outport.ValidatorsRating) ([]*bytes.Buffer, error) } // DBLogsAndEventsHandler defines the actions that a logs and events handler should do type DBLogsAndEventsHandler interface { - PrepareLogsForDB(logsAndEvents []*coreData.LogData, timestamp uint64) []*data.Logs + PrepareLogsForDB(logsAndEvents map[string]*transaction.Log, timestamp uint64) []*data.Logs ExtractDataFromLogs( - logsAndEvents []*coreData.LogData, + logsAndEvents map[string]*transaction.Log, preparedResults *data.PreparedResults, timestamp uint64, shardID uint32, diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor.go b/process/elasticproc/logsevents/logsAndEventsProcessor.go index b7a82c73..4997f3ae 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor.go @@ -7,6 +7,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" coreData "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-es-indexer-go/data" @@ -86,7 +87,7 @@ func createEventsProcessors(args ArgsLogsAndEventsProcessor) []eventsProcessor { // ExtractDataFromLogs will extract data from the provided logs and events and put in altered addresses func (lep *logsAndEventsProcessor) ExtractDataFromLogs( - logsAndEvents []*coreData.LogData, + logsAndEvents map[string]*transaction.Log, preparedResults *data.PreparedResults, timestamp uint64, shardID uint32, @@ -94,15 +95,14 @@ func (lep *logsAndEventsProcessor) ExtractDataFromLogs( ) *data.PreparedLogsResults { lep.logsData = newLogsData(timestamp, preparedResults.Transactions, preparedResults.ScResults) - for _, txLog := range logsAndEvents { - if txLog == nil || check.IfNil(txLog.LogHandler) { + for txHashHexEncoded, txLog := range logsAndEvents { + if txLog == nil { continue } - events := txLog.LogHandler.GetLogEvents() - lep.processEvents(txLog.TxHash, txLog.LogHandler.GetAddress(), events, shardID, numOfShards) + events := txLog.Events + lep.processEvents(txHashHexEncoded, txLog.Address, events, shardID, numOfShards) - txHashHexEncoded := hex.EncodeToString([]byte(txLog.TxHash)) tx, ok := lep.logsData.txsMap[txHashHexEncoded] if ok { tx.HasLogs = true @@ -126,18 +126,17 @@ func (lep *logsAndEventsProcessor) ExtractDataFromLogs( } } -func (lep *logsAndEventsProcessor) processEvents(logHash string, logAddress []byte, events []coreData.EventHandler, shardID uint32, numOfShards uint32) { +func (lep *logsAndEventsProcessor) processEvents(logHashHexEncoded string, logAddress []byte, events []*transaction.Event, shardID uint32, numOfShards uint32) { for _, event := range events { if check.IfNil(event) { continue } - lep.processEvent(logHash, logAddress, event, shardID, numOfShards) + lep.processEvent(logHashHexEncoded, logAddress, event, shardID, numOfShards) } } -func (lep *logsAndEventsProcessor) processEvent(logHash string, logAddress []byte, event coreData.EventHandler, shardID uint32, numOfShards uint32) { - logHashHexEncoded := hex.EncodeToString([]byte(logHash)) +func (lep *logsAndEventsProcessor) processEvent(logHashHexEncoded string, logAddress []byte, event coreData.EventHandler, shardID uint32, numOfShards uint32) { for _, proc := range lep.eventsProcessors { res := proc.processEvent(&argsProcessEvent{ event: event, @@ -181,17 +180,17 @@ func (lep *logsAndEventsProcessor) processEvent(logHash string, logAddress []byt // PrepareLogsForDB will prepare logs for database func (lep *logsAndEventsProcessor) PrepareLogsForDB( - logsAndEvents []*coreData.LogData, + logsAndEvents map[string]*transaction.Log, timestamp uint64, ) []*data.Logs { logs := make([]*data.Logs, 0, len(logsAndEvents)) - for _, txLog := range logsAndEvents { - if txLog == nil || check.IfNil(txLog.LogHandler) { + for txHashHex, txLog := range logsAndEvents { + if txLog == nil { continue } - logs = append(logs, lep.prepareLogsForDB(txLog.TxHash, txLog.LogHandler, timestamp)) + logs = append(logs, lep.prepareLogsForDB(txHashHex, txLog, timestamp)) } return logs diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go index c2ef33fa..51347c0d 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go @@ -6,7 +6,6 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/core" - coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/mock" @@ -57,9 +56,9 @@ func TestNewLogsAndEventsProcessor(t *testing.T) { func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) { t.Parallel() - logsAndEvents := map[string]coreData.LogHandler{ + logsAndEvents := map[string]*transaction.Log{ "wrong": nil, - "h3": &transaction.Log{ + "h3": { Events: []*transaction.Event{ { Address: []byte("addr"), @@ -69,7 +68,7 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) }, }, - "h1": &transaction.Log{ + "h1": { Address: []byte("address"), Events: []*transaction.Event{ { @@ -80,7 +79,7 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) }, }, - "h2": &transaction.Log{ + "h2": { Events: []*transaction.Event{ { Address: []byte("addr"), @@ -90,7 +89,7 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) nil, }, }, - "h4": &transaction.Log{ + "h4": { Events: []*transaction.Event{ { Address: []byte("addr"), @@ -100,7 +99,7 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) nil, }, }, - "h5": &transaction.Log{ + "h5": { Address: []byte("contract"), Events: []*transaction.Event{ { @@ -110,7 +109,7 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) }, }, }, - "h6": &transaction.Log{ + "h6": { Address: []byte("contract-second"), Events: []*transaction.Event{ { @@ -122,14 +121,6 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) }, } - logsAndEventsSlice := make([]*coreData.LogData, 0) - for hash, val := range logsAndEvents { - logsAndEventsSlice = append(logsAndEventsSlice, &coreData.LogData{ - TxHash: hash, - LogHandler: val, - }) - } - res := &data.PreparedResults{ Transactions: []*data.Transaction{ { @@ -148,7 +139,7 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) args.BalanceConverter = balanceConverter proc, _ := NewLogsAndEventsProcessor(args) - resLogs := proc.ExtractDataFromLogs(logsAndEventsSlice, res, 1000, core.MetachainShardId, 3) + resLogs := proc.ExtractDataFromLogs(logsAndEvents, res, 1000, core.MetachainShardId, 3) require.NotNil(t, resLogs.Tokens) require.True(t, res.Transactions[0].HasOperations) require.True(t, res.ScResults[0].HasOperations) @@ -197,7 +188,7 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) func TestLogsAndEventsProcessor_PrepareLogsForDB(t *testing.T) { t.Parallel() - logsAndEvents := map[string]coreData.LogHandler{ + logsAndEvents := map[string]*transaction.Log{ "wrong": nil, "txHash": &transaction.Log{ @@ -212,14 +203,6 @@ func TestLogsAndEventsProcessor_PrepareLogsForDB(t *testing.T) { }, } - logsAndEventsSlice := make([]*coreData.LogData, 0) - for hash, val := range logsAndEvents { - logsAndEventsSlice = append(logsAndEventsSlice, &coreData.LogData{ - TxHash: hash, - LogHandler: val, - }) - } - args := createMockArgs() proc, _ := NewLogsAndEventsProcessor(args) @@ -230,7 +213,7 @@ func TestLogsAndEventsProcessor_PrepareLogsForDB(t *testing.T) { }, }}, 1234, 0, 3) - logsDB := proc.PrepareLogsForDB(logsAndEventsSlice, 1234) + logsDB := proc.PrepareLogsForDB(logsAndEvents, 1234) require.Equal(t, &data.Logs{ ID: "747848617368", Address: "61646472657373", @@ -249,19 +232,16 @@ func TestLogsAndEventsProcessor_PrepareLogsForDB(t *testing.T) { func TestLogsAndEventsProcessor_ExtractDataFromLogsNFTBurn(t *testing.T) { t.Parallel() - logsAndEventsSlice := make([]*coreData.LogData, 1) - logsAndEventsSlice[0] = &coreData.LogData{ - LogHandler: &transaction.Log{ - Address: []byte("address"), - Events: []*transaction.Event{ - { - Address: []byte("addr"), - Identifier: []byte(core.BuiltInFunctionESDTNFTBurn), - Topics: [][]byte{[]byte("MY-NFT"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes()}, - }, + logsAndEventsSlice := make(map[string]*transaction.Log, 1) + logsAndEventsSlice["h1"] = &transaction.Log{ + Address: []byte("address"), + Events: []*transaction.Event{ + { + Address: []byte("addr"), + Identifier: []byte(core.BuiltInFunctionESDTNFTBurn), + Topics: [][]byte{[]byte("MY-NFT"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes()}, }, }, - TxHash: "h1", } res := &data.PreparedResults{ diff --git a/process/elasticproc/logsevents/nftsProcessor.go b/process/elasticproc/logsevents/nftsProcessor.go index c516e523..551b7f99 100644 --- a/process/elasticproc/logsevents/nftsProcessor.go +++ b/process/elasticproc/logsevents/nftsProcessor.go @@ -7,8 +7,8 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/sharding" coreData "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/data/alteredAccount" "github.com/multiversx/mx-chain-core-go/data/esdt" - "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" @@ -154,12 +154,12 @@ func (np *nftsProcessor) processNFTEventOnSender( }) } -func (np *nftsProcessor) convertMetaData(metaData *esdt.MetaData) *outport.TokenMetaData { +func (np *nftsProcessor) convertMetaData(metaData *esdt.MetaData) *alteredAccount.TokenMetaData { if metaData == nil { return nil } - return &outport.TokenMetaData{ + return &alteredAccount.TokenMetaData{ Nonce: metaData.Nonce, Name: string(metaData.Name), Creator: np.pubKeyConverter.Encode(metaData.Creator), diff --git a/process/elasticproc/statistics/serialize.go b/process/elasticproc/statistics/serialize.go index 0cac6d60..09879f25 100644 --- a/process/elasticproc/statistics/serialize.go +++ b/process/elasticproc/statistics/serialize.go @@ -4,7 +4,9 @@ import ( "bytes" "encoding/json" "fmt" + "time" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-es-indexer-go/data" logger "github.com/multiversx/mx-chain-logger-go" ) @@ -20,10 +22,17 @@ func NewStatisticsProcessor() *statisticsProcessor { } // SerializeRoundsInfo will serialize information about rounds -func (sp *statisticsProcessor) SerializeRoundsInfo(roundsInfo []*data.RoundInfo) *bytes.Buffer { +func (sp *statisticsProcessor) SerializeRoundsInfo(rounds *outport.RoundsInfo) *bytes.Buffer { buff := &bytes.Buffer{} - for _, info := range roundsInfo { - serializedRoundInfo, meta := serializeRoundInfo(info) + for _, info := range rounds.RoundsInfo { + serializedRoundInfo, meta := serializeRoundInfo(&data.RoundInfo{ + Round: info.Round, + SignersIndexes: info.SignersIndexes, + BlockWasProposed: info.BlockWasProposed, + ShardId: info.ShardId, + Epoch: info.Epoch, + Timestamp: time.Duration(info.Timestamp), + }) buff.Grow(len(meta) + len(serializedRoundInfo)) _, err := buff.Write(meta) @@ -41,7 +50,7 @@ func (sp *statisticsProcessor) SerializeRoundsInfo(roundsInfo []*data.RoundInfo) } func serializeRoundInfo(info *data.RoundInfo) ([]byte, []byte) { - meta := []byte(fmt.Sprintf(`{ "index" : { "_id" : "%d_%d" } }%s`, info.ShardId, info.Index, "\n")) + meta := []byte(fmt.Sprintf(`{ "index" : { "_id" : "%d_%d" } }%s`, info.ShardId, info.Round, "\n")) serializedInfo, err := json.Marshal(info) if err != nil { diff --git a/process/elasticproc/statistics/serialize_test.go b/process/elasticproc/statistics/serialize_test.go index 06d3b6cf..a0cd079c 100644 --- a/process/elasticproc/statistics/serialize_test.go +++ b/process/elasticproc/statistics/serialize_test.go @@ -3,7 +3,7 @@ package statistics import ( "testing" - "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/stretchr/testify/require" ) @@ -12,9 +12,11 @@ func TestStatisticsProcessor_SerializeRoundsInfo(t *testing.T) { sp := NewStatisticsProcessor() - buff := sp.SerializeRoundsInfo([]*data.RoundInfo{{ - Epoch: 1, - }}) + buff := sp.SerializeRoundsInfo(&outport.RoundsInfo{ + RoundsInfo: []*outport.RoundInfo{{ + Epoch: 1, + }}, + }) expectedBuff := `{ "index" : { "_id" : "0_0" } } {"round":0,"signersIndexes":null,"blockWasProposed":false,"shardId":0,"epoch":1,"timestamp":0} ` diff --git a/process/elasticproc/transactions/checkers.go b/process/elasticproc/transactions/checkers.go index 9620048a..041cdfb8 100644 --- a/process/elasticproc/transactions/checkers.go +++ b/process/elasticproc/transactions/checkers.go @@ -53,7 +53,7 @@ func areESDTValuesOK(values []string) bool { func checkPrepareTransactionForDatabaseArguments( body *block.Body, header coreData.HeaderHandler, - pool *outport.Pool, + pool *outport.TransactionPool, ) error { if body == nil { return elasticIndexer.ErrNilBlockBody diff --git a/process/elasticproc/transactions/checkers_test.go b/process/elasticproc/transactions/checkers_test.go index 8901b401..1377be2d 100644 --- a/process/elasticproc/transactions/checkers_test.go +++ b/process/elasticproc/transactions/checkers_test.go @@ -72,26 +72,26 @@ func TestCheckTxsProcessorArg(t *testing.T) { tests := []struct { name string - args func() (body *block.Body, header coreData.HeaderHandler, pool *outport.Pool) + args func() (body *block.Body, header coreData.HeaderHandler, pool *outport.TransactionPool) exErr error }{ { name: "NilBlockBody", - args: func() (body *block.Body, header coreData.HeaderHandler, pool *outport.Pool) { - return nil, &block.Header{}, &outport.Pool{} + args: func() (body *block.Body, header coreData.HeaderHandler, pool *outport.TransactionPool) { + return nil, &block.Header{}, &outport.TransactionPool{} }, exErr: elasticIndexer.ErrNilBlockBody, }, { name: "NilHeaderHandler", - args: func() (body *block.Body, header coreData.HeaderHandler, pool *outport.Pool) { - return &block.Body{}, nil, &outport.Pool{} + args: func() (body *block.Body, header coreData.HeaderHandler, pool *outport.TransactionPool) { + return &block.Body{}, nil, &outport.TransactionPool{} }, exErr: elasticIndexer.ErrNilHeaderHandler, }, { name: "NilPool", - args: func() (body *block.Body, header coreData.HeaderHandler, pool *outport.Pool) { + args: func() (body *block.Body, header coreData.HeaderHandler, pool *outport.TransactionPool) { return &block.Body{}, &block.Header{}, nil }, exErr: elasticIndexer.ErrNilPool, diff --git a/process/elasticproc/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go index ded80aa3..eb0b3ffb 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -2,15 +2,13 @@ package transactions import ( "encoding/hex" - "math/big" "strconv" "time" "github.com/multiversx/mx-chain-core-go/core" - "github.com/multiversx/mx-chain-core-go/data" coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" - "github.com/multiversx/mx-chain-core-go/data/smartContractResult" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" indexerData "github.com/multiversx/mx-chain-es-indexer-go/data" @@ -45,7 +43,7 @@ func newSmartContractResultsProcessor( func (proc *smartContractResultsProcessor) processSCRs( body *block.Body, header coreData.HeaderHandler, - txsHandler map[string]data.TransactionHandlerWithGasUsedAndFee, + txsHandler map[string]*outport.SCRInfo, numOfShards uint32, ) []*indexerData.ScResult { allSCRs := make([]*indexerData.ScResult, 0, len(txsHandler)) @@ -63,13 +61,8 @@ func (proc *smartContractResultsProcessor) processSCRs( } selfShardID := header.GetShardID() - for scrHash, noMBScr := range workingSCRSMap { - scr, ok := noMBScr.GetTxHandler().(*smartContractResult.SmartContractResult) - if !ok { - continue - } - - indexerScr := proc.prepareSmartContractResult([]byte(scrHash), nil, scr, header, selfShardID, selfShardID, noMBScr.GetFee(), noMBScr.GetGasUsed(), numOfShards) + for scrHash, noMBScrInfo := range workingSCRSMap { + indexerScr := proc.prepareSmartContractResult([]byte(scrHash), nil, noMBScrInfo, header, selfShardID, selfShardID, numOfShards) allSCRs = append(allSCRs, indexerScr) } @@ -80,7 +73,7 @@ func (proc *smartContractResultsProcessor) processSCRs( func (proc *smartContractResultsProcessor) processSCRsFromMiniblock( header coreData.HeaderHandler, mb *block.MiniBlock, - scrs map[string]data.TransactionHandlerWithGasUsedAndFee, + scrs map[string]*outport.SCRInfo, numOfShards uint32, ) []*indexerData.ScResult { mbHash, err := core.CalculateHash(proc.marshalizer, proc.hasher, mb) @@ -91,19 +84,15 @@ func (proc *smartContractResultsProcessor) processSCRsFromMiniblock( indexerSCRs := make([]*indexerData.ScResult, 0, len(mb.TxHashes)) for _, scrHash := range mb.TxHashes { - scrHandler, ok := scrs[string(scrHash)] + scrInfo, ok := scrs[hex.EncodeToString(scrHash)] if !ok { log.Warn("smartContractResultsProcessor.processSCRsFromMiniblock scr not found in map", "scr hash", hex.EncodeToString(scrHash), ) continue } - scr, ok := scrHandler.GetTxHandler().(*smartContractResult.SmartContractResult) - if !ok { - continue - } - indexerSCR := proc.prepareSmartContractResult(scrHash, mbHash, scr, header, mb.SenderShardID, mb.ReceiverShardID, scrHandler.GetFee(), scrHandler.GetGasUsed(), numOfShards) + indexerSCR := proc.prepareSmartContractResult(scrHash, mbHash, scrInfo, header, mb.SenderShardID, mb.ReceiverShardID, numOfShards) indexerSCRs = append(indexerSCRs, indexerSCR) delete(scrs, string(scrHash)) @@ -115,14 +104,13 @@ func (proc *smartContractResultsProcessor) processSCRsFromMiniblock( func (proc *smartContractResultsProcessor) prepareSmartContractResult( scrHash []byte, mbHash []byte, - scr *smartContractResult.SmartContractResult, + scrInfo *outport.SCRInfo, header coreData.HeaderHandler, senderShard uint32, receiverShard uint32, - initialTxFee *big.Int, - initialTxGasUsed uint64, numOfShards uint32, ) *indexerData.ScResult { + scr := scrInfo.SmartContractResult hexEncodedMBHash := "" if len(mbHash) > 0 { hexEncodedMBHash = hex.EncodeToString(mbHash) @@ -193,13 +181,13 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( ReceiversShardIDs: res.ReceiversShardID, IsRelayed: res.IsRelayed, OriginalSender: originalSenderAddr, - InitialTxFee: initialTxFee.String(), - InitialTxGasUsed: initialTxGasUsed, + InitialTxFee: scrInfo.FeeInfo.Fee.String(), + InitialTxGasUsed: scrInfo.FeeInfo.GasUsed, } } -func copySCRSMap(initial map[string]data.TransactionHandlerWithGasUsedAndFee) map[string]data.TransactionHandlerWithGasUsedAndFee { - newMap := make(map[string]data.TransactionHandlerWithGasUsedAndFee) +func copySCRSMap(initial map[string]*outport.SCRInfo) map[string]*outport.SCRInfo { + newMap := make(map[string]*outport.SCRInfo) for key, value := range initial { newMap[key] = value } diff --git a/process/elasticproc/transactions/smartContractResultsProcessor_test.go b/process/elasticproc/transactions/smartContractResultsProcessor_test.go index 0ec7429a..b2b5fd9d 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor_test.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/mock" @@ -38,6 +39,7 @@ func TestPrepareSmartContractResult(t *testing.T) { code := []byte("code") sndAddr, rcvAddr := []byte("snd"), []byte("rec") scHash := "scHash" + smartContractRes := &smartContractResult.SmartContractResult{ Nonce: nonce, PrevTxHash: txHash, @@ -47,10 +49,18 @@ func TestPrepareSmartContractResult(t *testing.T) { RcvAddr: rcvAddr, CallType: 1, } + + scrInfo := &outport.SCRInfo{ + SmartContractResult: smartContractRes, + FeeInfo: &outport.FeeInfo{ + Fee: big.NewInt(0), + }, + } + header := &block.Header{TimeStamp: 100} mbHash := []byte("hash") - scRes := scrsProc.prepareSmartContractResult([]byte(scHash), mbHash, smartContractRes, header, 0, 1, big.NewInt(0), 0, 3) + scRes := scrsProc.prepareSmartContractResult([]byte(scHash), mbHash, scrInfo, header, 0, 1, 3) expectedTx := &data.ScResult{ Nonce: nonce, Hash: hex.EncodeToString([]byte(scHash)), diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index b5720626..2b341311 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -3,16 +3,15 @@ package transactions import ( "encoding/hex" "fmt" - "math/big" "time" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/sharding" coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/receipt" "github.com/multiversx/mx-chain-core-go/data/rewardTx" - "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" datafield "github.com/multiversx/mx-chain-vm-common-go/parsers/dataField" @@ -37,17 +36,16 @@ func newTransactionDBBuilder( } func (dtb *dbTransactionBuilder) prepareTransaction( - tx *transaction.Transaction, + txInfo *outport.TxInfo, txHash []byte, mbHash []byte, mb *block.MiniBlock, header coreData.HeaderHandler, txStatus string, - fee *big.Int, - gasUsed uint64, - initialPaidFee *big.Int, numOfShards uint32, ) *data.Transaction { + tx := txInfo.Transaction + isScCall := core.IsSmartContractAddress(tx.RcvAddr) res := dtb.dataFieldParser.Parse(tx.Data, tx.SndAddr, tx.RcvAddr, numOfShards) @@ -61,9 +59,9 @@ func (dtb *dbTransactionBuilder) prepareTransaction( log.Warn("dbTransactionBuilder.prepareTransaction: cannot compute value as num", "value", tx.Value, "hash", txHash, "error", err) } - feeNum, err := dtb.balanceConverter.ComputeESDTBalanceAsFloat(fee) + feeNum, err := dtb.balanceConverter.ComputeESDTBalanceAsFloat(txInfo.FeeInfo.Fee) if err != nil { - log.Warn("dbTransactionBuilder.prepareTransaction: cannot compute transaction fee as num", "fee", fee, + log.Warn("dbTransactionBuilder.prepareTransaction: cannot compute transaction fee as num", "fee", txInfo.FeeInfo.Fee, "hash", txHash, "error", err) } esdtValuesNum, err := dtb.balanceConverter.ComputeSliceOfStringsAsFloat(res.ESDTValues) @@ -94,9 +92,9 @@ func (dtb *dbTransactionBuilder) prepareTransaction( Signature: hex.EncodeToString(tx.Signature), Timestamp: time.Duration(header.GetTimeStamp()), Status: txStatus, - GasUsed: gasUsed, - InitialPaidFee: initialPaidFee.String(), - Fee: fee.String(), + GasUsed: txInfo.FeeInfo.GasUsed, + InitialPaidFee: txInfo.FeeInfo.InitialPaidFee.String(), + Fee: txInfo.FeeInfo.Fee.String(), FeeNum: feeNum, ReceiverUserName: tx.RcvUserName, SenderUserName: tx.SndUserName, diff --git a/process/elasticproc/transactions/transactionsGrouper.go b/process/elasticproc/transactions/transactionsGrouper.go index 30ccc6d6..01734259 100644 --- a/process/elasticproc/transactions/transactionsGrouper.go +++ b/process/elasticproc/transactions/transactionsGrouper.go @@ -1,11 +1,13 @@ package transactions import ( + "encoding/hex" + "github.com/multiversx/mx-chain-core-go/core" coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/receipt" - "github.com/multiversx/mx-chain-core-go/data/rewardTx" "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" @@ -38,7 +40,7 @@ func (tg *txsGrouper) groupNormalTxs( mbIndex int, mb *block.MiniBlock, header coreData.HeaderHandler, - txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, + txs map[string]*outport.TxInfo, isImportDB bool, numOfShards uint32, ) (map[string]*data.Transaction, error) { @@ -92,21 +94,16 @@ func (tg *txsGrouper) prepareNormalTxForDB( mb *block.MiniBlock, mbStatus string, txHash []byte, - txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, + txs map[string]*outport.TxInfo, header coreData.HeaderHandler, numOfShards uint32, ) (*data.Transaction, bool) { - txHandler, okGet := txs[string(txHash)] + txInfo, okGet := txs[hex.EncodeToString(txHash)] if !okGet { return nil, false } - tx, okCast := txHandler.GetTxHandler().(*transaction.Transaction) - if !okCast { - return nil, false - } - - dbTx := tg.txBuilder.prepareTransaction(tx, txHash, mbHash, mb, header, mbStatus, txHandler.GetFee(), txHandler.GetGasUsed(), txHandler.GetInitialPaidFee(), numOfShards) + dbTx := tg.txBuilder.prepareTransaction(txInfo, txHash, mbHash, mb, header, mbStatus, numOfShards) return dbTx, true } @@ -115,7 +112,7 @@ func (tg *txsGrouper) groupRewardsTxs( mbIndex int, mb *block.MiniBlock, header coreData.HeaderHandler, - txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, + txs map[string]*outport.RewardInfo, isImportDB bool, ) (map[string]*data.Transaction, error) { rewardsTxs := make(map[string]*data.Transaction) @@ -146,20 +143,15 @@ func (tg *txsGrouper) prepareRewardTxForDB( mb *block.MiniBlock, mbStatus string, txHash []byte, - txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, + txs map[string]*outport.RewardInfo, header coreData.HeaderHandler, ) (*data.Transaction, bool) { - txHandler, okGet := txs[string(txHash)] + rtx, okGet := txs[hex.EncodeToString(txHash)] if !okGet { return nil, false } - rtx, okCast := txHandler.GetTxHandler().(*rewardTx.RewardTx) - if !okCast { - return nil, false - } - - dbTx := tg.txBuilder.prepareRewardTransaction(rtx, txHash, mbHash, mb, header, mbStatus) + dbTx := tg.txBuilder.prepareRewardTransaction(rtx.Reward, txHash, mbHash, mb, header, mbStatus) return dbTx, true } @@ -168,7 +160,7 @@ func (tg *txsGrouper) groupInvalidTxs( mbIndex int, mb *block.MiniBlock, header coreData.HeaderHandler, - txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, + txs map[string]*outport.TxInfo, numOfShards uint32, ) (map[string]*data.Transaction, error) { transactions := make(map[string]*data.Transaction) @@ -194,21 +186,16 @@ func (tg *txsGrouper) prepareInvalidTxForDB( mbHash []byte, mb *block.MiniBlock, txHash []byte, - txs map[string]coreData.TransactionHandlerWithGasUsedAndFee, + txs map[string]*outport.TxInfo, header coreData.HeaderHandler, numOfShards uint32, ) (*data.Transaction, bool) { - txHandler, okGet := txs[string(txHash)] + txInfo, okGet := txs[string(txHash)] if !okGet { return nil, false } - tx, okCast := txHandler.GetTxHandler().(*transaction.Transaction) - if !okCast { - return nil, false - } - - dbTx := tg.txBuilder.prepareTransaction(tx, txHash, mbHash, mb, header, transaction.TxStatusInvalid.String(), txHandler.GetFee(), txHandler.GetGasUsed(), txHandler.GetInitialPaidFee(), numOfShards) + dbTx := tg.txBuilder.prepareTransaction(txInfo, txHash, mbHash, mb, header, transaction.TxStatusInvalid.String(), numOfShards) return dbTx, true } @@ -221,14 +208,9 @@ func (tg *txsGrouper) shouldIndex(destinationShardID uint32, isImportDB bool, se return selfShardID == destinationShardID } -func (tg *txsGrouper) groupReceipts(header coreData.HeaderHandler, txsPool map[string]coreData.TransactionHandlerWithGasUsedAndFee) []*data.Receipt { +func (tg *txsGrouper) groupReceipts(header coreData.HeaderHandler, txsPool map[string]*receipt.Receipt) []*data.Receipt { dbReceipts := make([]*data.Receipt, 0) - for hash, tx := range txsPool { - rec, ok := tx.GetTxHandler().(*receipt.Receipt) - if !ok { - continue - } - + for hash, rec := range txsPool { dbReceipts = append(dbReceipts, tg.txBuilder.prepareReceipt(hash, rec, header)) } diff --git a/process/elasticproc/transactions/transactionsProcessor.go b/process/elasticproc/transactions/transactionsProcessor.go index 7022b8bc..7b6386ea 100644 --- a/process/elasticproc/transactions/transactionsProcessor.go +++ b/process/elasticproc/transactions/transactionsProcessor.go @@ -67,7 +67,7 @@ func NewTransactionsProcessor(args *ArgsTransactionProcessor) (*txsDatabaseProce func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( body *block.Body, header coreData.HeaderHandler, - pool *outport.Pool, + pool *outport.TransactionPool, isImportDB bool, numOfShards uint32, ) *data.PreparedResults { @@ -92,7 +92,7 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( continue } - txs, errGroup := tdp.txsGrouper.groupNormalTxs(mbIndex, mb, header, pool.Txs, isImportDB, numOfShards) + txs, errGroup := tdp.txsGrouper.groupNormalTxs(mbIndex, mb, header, pool.Transactions, isImportDB, numOfShards) if errGroup != nil { log.Warn("txsDatabaseProcessor.groupNormalTxs", "error", errGroup) continue @@ -106,7 +106,7 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( } mergeTxsMaps(rewardsTxs, txs) case block.InvalidBlock: - txs, errGroup := tdp.txsGrouper.groupInvalidTxs(mbIndex, mb, header, pool.Invalid, numOfShards) + txs, errGroup := tdp.txsGrouper.groupInvalidTxs(mbIndex, mb, header, pool.InvalidTxs, numOfShards) if errGroup != nil { log.Warn("txsDatabaseProcessor.groupInvalidTxs", "error", errGroup) continue @@ -119,7 +119,7 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( normalTxs = tdp.setTransactionSearchOrder(normalTxs) dbReceipts := tdp.txsGrouper.groupReceipts(header, pool.Receipts) - dbSCResults := tdp.scrsProc.processSCRs(body, header, pool.Scrs, numOfShards) + dbSCResults := tdp.scrsProc.processSCRs(body, header, pool.SmartContractResults, numOfShards) srcsNoTxInCurrentShard := tdp.scrsDataToTxs.attachSCRsToTransactionsAndReturnSCRsWithoutTx(normalTxs, dbSCResults) tdp.scrsDataToTxs.processTransactionsAfterSCRsWereAttached(normalTxs) diff --git a/process/elasticproc/transactions/transactionsProcessor_test.go b/process/elasticproc/transactions/transactionsProcessor_test.go index 0b20b10e..d4caac9c 100644 --- a/process/elasticproc/transactions/transactionsProcessor_test.go +++ b/process/elasticproc/transactions/transactionsProcessor_test.go @@ -6,7 +6,6 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/pubkeyConverter" - coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/receipt" @@ -57,55 +56,90 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { t.Parallel() txHash1 := []byte("txHash1") - tx1 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - GasLimit: 100, - GasPrice: 100, - }, 0, big.NewInt(0)) + tx1 := &outport.TxInfo{ + Transaction: &transaction.Transaction{ + GasLimit: 100, + GasPrice: 100, + }, + FeeInfo: &outport.FeeInfo{ + Fee: big.NewInt(0), + }, + } txHash2 := []byte("txHash2") - tx2 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - GasLimit: 100, - GasPrice: 100, - }, 0, big.NewInt(0)) + tx2 := &outport.TxInfo{ + Transaction: &transaction.Transaction{ + GasLimit: 100, + GasPrice: 100, + }, + FeeInfo: &outport.FeeInfo{ + Fee: big.NewInt(0), + }, + } txHash3 := []byte("txHash3") - tx3 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{}, 0, big.NewInt(0)) + tx3 := &outport.TxInfo{ + Transaction: &transaction.Transaction{}, + FeeInfo: &outport.FeeInfo{ + Fee: big.NewInt(0), + }, + } txHash4 := []byte("txHash4") - tx4 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{}, 0, big.NewInt(0)) + tx4 := &outport.TxInfo{ + Transaction: &transaction.Transaction{}, + FeeInfo: &outport.FeeInfo{ + Fee: big.NewInt(0), + }, + } txHash5 := []byte("txHash5") - tx5 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{}, 0, big.NewInt(0)) + tx5 := &outport.TxInfo{ + Transaction: &transaction.Transaction{}, + FeeInfo: &outport.FeeInfo{ + Fee: big.NewInt(0), + }, + } rTx1Hash := []byte("rTxHash1") - rTx1 := outport.NewTransactionHandlerWithGasAndFee(&rewardTx.RewardTx{}, 0, big.NewInt(0)) + rTx1 := &outport.RewardInfo{ + Reward: &rewardTx.RewardTx{}, + } rTx2Hash := []byte("rTxHash2") - rTx2 := outport.NewTransactionHandlerWithGasAndFee(&rewardTx.RewardTx{}, 0, big.NewInt(0)) + rTx2 := &outport.RewardInfo{ + Reward: &rewardTx.RewardTx{}, + } recHash1 := []byte("recHash1") - rec1 := outport.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ + rec1 := &receipt.Receipt{ Value: big.NewInt(100), TxHash: txHash1, - }, 0, big.NewInt(0)) + } recHash2 := []byte("recHash2") - rec2 := outport.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ + rec2 := &receipt.Receipt{ Value: big.NewInt(200), TxHash: txHash2, - }, 0, big.NewInt(0)) + } scHash1 := []byte("scHash1") - scResult1 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ - OriginalTxHash: txHash1, - PrevTxHash: txHash1, - GasLimit: 1, - }, 0, big.NewInt(0)) + scResult1 := &outport.SCRInfo{ + SmartContractResult: &smartContractResult.SmartContractResult{ + OriginalTxHash: txHash1, + PrevTxHash: txHash1, + GasLimit: 1, + }, + } scHash2 := []byte("scHash2") - scResult2 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ - OriginalTxHash: txHash1, - PrevTxHash: txHash1, - GasLimit: 1, - }, 0, big.NewInt(0)) + scResult2 := &outport.SCRInfo{ + SmartContractResult: &smartContractResult.SmartContractResult{ + OriginalTxHash: txHash1, + PrevTxHash: txHash1, + GasLimit: 1, + }, + } scHash3 := []byte("scHash3") - scResult3 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ - OriginalTxHash: txHash3, - Data: []byte("@" + "6F6B"), - }, 0, big.NewInt(0)) + scResult3 := &outport.SCRInfo{ + SmartContractResult: &smartContractResult.SmartContractResult{ + OriginalTxHash: txHash3, + Data: []byte("@" + "6F6B"), + }, + } body := &block.Body{ MiniBlocks: []*block.MiniBlock{ @@ -141,26 +175,26 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { } header := &block.Header{} - pool := &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ string(txHash1): tx1, string(txHash2): tx2, string(txHash3): tx3, string(txHash4): tx4, }, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + SmartContractResults: map[string]*outport.SCRInfo{ string(scHash1): scResult1, string(scHash2): scResult2, string(scHash3): scResult3, }, - Rewards: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + Rewards: map[string]*outport.RewardInfo{ string(rTx1Hash): rTx1, string(rTx2Hash): rTx2, }, - Invalid: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + InvalidTxs: map[string]*outport.TxInfo{ string(txHash5): tx5, }, - Receipts: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + Receipts: map[string]*receipt.Receipt{ string(recHash1): rec1, string(recHash2): rec2, }, @@ -177,24 +211,24 @@ func TestRelayedTransactions(t *testing.T) { t.Parallel() txHash1 := []byte("txHash1") - tx1 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + tx1 := &outport.TxInfo{Transaction: &transaction.Transaction{ GasLimit: 100, GasPrice: 100, Data: []byte("relayedTx@blablabllablalba"), - }, 0, big.NewInt(0)) + }} scHash1 := []byte("scHash1") - scResult1 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + scResult1 := &outport.SCRInfo{SmartContractResult: &smartContractResult.SmartContractResult{ OriginalTxHash: txHash1, PrevTxHash: txHash1, GasLimit: 1, - }, 0, big.NewInt(0)) + }} scHash2 := []byte("scHash2") - scResult2 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + scResult2 := &outport.SCRInfo{SmartContractResult: &smartContractResult.SmartContractResult{ OriginalTxHash: txHash1, PrevTxHash: txHash1, GasLimit: 1, - }, 0, big.NewInt(0)) + }} body := &block.Body{ MiniBlocks: []*block.MiniBlock{ @@ -211,11 +245,11 @@ func TestRelayedTransactions(t *testing.T) { header := &block.Header{} - pool := &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ string(txHash1): tx1, }, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + SmartContractResults: map[string]*outport.SCRInfo{ string(scHash1): scResult1, string(scHash2): scResult2, }, @@ -295,15 +329,20 @@ func TestCheckGasUsedInvalidTransaction(t *testing.T) { txDbProc, _ := NewTransactionsProcessor(createMockArgsTxsDBProc()) txHash1 := []byte("txHash1") - tx1 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - GasLimit: 100, - GasPrice: 100, - }, 100, big.NewInt(0)) + tx1 := &outport.TxInfo{ + Transaction: &transaction.Transaction{ + GasLimit: 100, + GasPrice: 100, + }, + FeeInfo: &outport.FeeInfo{ + GasUsed: 100, + }, + } recHash1 := []byte("recHash1") - rec1 := outport.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ + rec1 := &receipt.Receipt{ Value: big.NewInt(100), TxHash: txHash1, - }, 0, big.NewInt(0)) + } body := &block.Body{ MiniBlocks: []*block.MiniBlock{ @@ -320,18 +359,18 @@ func TestCheckGasUsedInvalidTransaction(t *testing.T) { header := &block.Header{} - pool := &outport.Pool{ - Invalid: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + pool := &outport.TransactionPool{ + InvalidTxs: map[string]*outport.TxInfo{ string(txHash1): tx1, }, - Receipts: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + Receipts: map[string]*receipt.Receipt{ string(recHash1): rec1, }, } results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false, 3) require.Len(t, results.Transactions, 1) - require.Equal(t, tx1.GetGasLimit(), results.Transactions[0].GasUsed) + require.Equal(t, tx1.Transaction.GetGasLimit(), results.Transactions[0].GasUsed) } func TestGetRewardsTxsHashesHexEncoded(t *testing.T) { @@ -421,15 +460,18 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseInvalidTxWithSCR(t * txDbProc, _ := NewTransactionsProcessor(createMockArgsTxsDBProc()) txHash1 := []byte("txHash1") - tx1 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - GasLimit: 100, - GasPrice: 123456, - Data: []byte("ESDTTransfer@54474e2d383862383366@0a"), - }, 100, big.NewInt(0)) + tx1 := &outport.TxInfo{ + Transaction: &transaction.Transaction{ + GasLimit: 100, + GasPrice: 123456, + Data: []byte("ESDTTransfer@54474e2d383862383366@0a"), + }, + FeeInfo: &outport.FeeInfo{GasUsed: 100}, + } scResHash1 := []byte("scResHash1") - scRes1 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + scRes1 := &outport.SCRInfo{SmartContractResult: &smartContractResult.SmartContractResult{ OriginalTxHash: txHash1, - }, 0, big.NewInt(0)) + }} body := &block.Body{ MiniBlocks: []*block.MiniBlock{ @@ -446,11 +488,11 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseInvalidTxWithSCR(t * header := &block.Header{} - pool := &outport.Pool{ - Invalid: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + pool := &outport.TransactionPool{ + InvalidTxs: map[string]*outport.TxInfo{ string(txHash1): tx1, }, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + SmartContractResults: map[string]*outport.SCRInfo{ string(scResHash1): scRes1, }, } @@ -472,18 +514,21 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseESDTNFTTransfer(t *t txDbProc, _ := NewTransactionsProcessor(createMockArgsTxsDBProc()) txHash1 := []byte("txHash1") - tx1 := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - GasLimit: 100, - GasPrice: 123456, - Data: []byte("ESDTNFTTransfer@595959453643392D303837363661@01@01@000000000000000005005C83E0C42EDCE394F40B24D29D298B0249C41F028974@66756E64@890479AFC610F4BEBC087D3ADA3F7C2775C736BBA91F41FD3D65092AA482D8B0@1c20"), - }, 100, big.NewInt(0)) + tx1 := &outport.TxInfo{ + Transaction: &transaction.Transaction{ + GasLimit: 100, + GasPrice: 123456, + Data: []byte("ESDTNFTTransfer@595959453643392D303837363661@01@01@000000000000000005005C83E0C42EDCE394F40B24D29D298B0249C41F028974@66756E64@890479AFC610F4BEBC087D3ADA3F7C2775C736BBA91F41FD3D65092AA482D8B0@1c20"), + }, + FeeInfo: &outport.FeeInfo{GasUsed: 100}, + } scResHash1 := []byte("scResHash1") - scRes1 := outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + scRes1 := &outport.SCRInfo{SmartContractResult: &smartContractResult.SmartContractResult{ Nonce: 1, Data: []byte("@" + okHexEncoded), OriginalTxHash: txHash1, PrevTxHash: txHash1, - }, 0, big.NewInt(0)) + }} body := &block.Body{ MiniBlocks: []*block.MiniBlock{ @@ -500,11 +545,11 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseESDTNFTTransfer(t *t header := &block.Header{} - pool := &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ string(txHash1): tx1, }, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ + SmartContractResults: map[string]*outport.SCRInfo{ string(scResHash1): scRes1, }, } @@ -551,27 +596,27 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { header := &block.Header{ ShardID: core.MetachainShardId, } - pool := &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - "t1": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + "t1": {Transaction: &transaction.Transaction{ SndAddr: decodeBech32("erd1dglncxk6sl9a3xumj78n6z2xux4ghp5c92cstv5zsn56tjgtdwpsk46qrs"), RcvAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), Data: []byte("issue@4141414141@41414141414141@0186a0@01@63616e467265657a65@74727565@63616e57697065@74727565@63616e5061757365@74727565@63616e4d696e74@74727565@63616e4275726e@74727565@63616e4368616e67654f776e6572@74727565@63616e55706772616465@74727565"), - }, 0, big.NewInt(0)), + }}, }, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - "scr1": outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + SmartContractResults: map[string]*outport.SCRInfo{ + "scr1": {SmartContractResult: &smartContractResult.SmartContractResult{ OriginalTxHash: []byte("t1"), Data: []byte("ESDTTransfer@414141414141412d323436626461@0186a0"), SndAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), RcvAddr: decodeBech32("erd1dglncxk6sl9a3xumj78n6z2xux4ghp5c92cstv5zsn56tjgtdwpsk46qrs"), - }, 0, big.NewInt(0)), - "scr2": outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + }}, + "scr2": {SmartContractResult: &smartContractResult.SmartContractResult{ OriginalTxHash: []byte("t1"), Data: []byte("@6f6b"), SndAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), RcvAddr: decodeBech32("erd1dglncxk6sl9a3xumj78n6z2xux4ghp5c92cstv5zsn56tjgtdwpsk46qrs"), - }, 0, big.NewInt(0)), + }}, }, } @@ -580,21 +625,21 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { require.Equal(t, 2, len(res.ScResults)) // transaction fail - pool = &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - "t1": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + pool = &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + "t1": {Transaction: &transaction.Transaction{ SndAddr: decodeBech32("erd1dglncxk6sl9a3xumj78n6z2xux4ghp5c92cstv5zsn56tjgtdwpsk46qrs"), RcvAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), Data: []byte("issue@4141414141@41414141414141@0186a0@01@63616e467265657a65@74727565@63616e57697065@74727565@63616e5061757365@74727565@63616e4d696e74@74727565@63616e4275726e@74727565@63616e4368616e67654f776e6572@74727565@63616e55706772616465@74727565"), - }, 0, big.NewInt(0)), + }}, }, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - "scr1": outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + SmartContractResults: map[string]*outport.SCRInfo{ + "scr1": {SmartContractResult: &smartContractResult.SmartContractResult{ OriginalTxHash: []byte("t1"), Data: []byte("75736572206572726f72"), SndAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), RcvAddr: decodeBech32("erd1dglncxk6sl9a3xumj78n6z2xux4ghp5c92cstv5zsn56tjgtdwpsk46qrs"), - }, 0, big.NewInt(0)), + }}, }, } diff --git a/process/elasticproc/validators/serialize.go b/process/elasticproc/validators/serialize.go index eb662a5d..f683b839 100644 --- a/process/elasticproc/validators/serialize.go +++ b/process/elasticproc/validators/serialize.go @@ -4,43 +4,24 @@ import ( "bytes" "encoding/json" "fmt" - "strings" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-es-indexer-go/data" ) -// SerializeValidatorsPubKeys will serialize validators public keys -func (vp *validatorsProcessor) SerializeValidatorsPubKeys(validatorsPubKeys *data.ValidatorsPublicKeys) (*bytes.Buffer, error) { - marshalizedValidatorPubKeys, err := json.Marshal(validatorsPubKeys) - if err != nil { - return nil, err - } - - buffer := &bytes.Buffer{} - buffer.Grow(len(marshalizedValidatorPubKeys)) - _, err = buffer.Write(marshalizedValidatorPubKeys) - if err != nil { - return nil, err - } - - return buffer, nil -} - // SerializeValidatorsRating will serialize validators rating -func (vp *validatorsProcessor) SerializeValidatorsRating( - index string, - validatorsRatingInfo []*data.ValidatorRatingInfo, -) ([]*bytes.Buffer, error) { +func (vp *validatorsProcessor) SerializeValidatorsRating(ratingData *outport.ValidatorsRating) ([]*bytes.Buffer, error) { buffSlice := data.NewBufferSlice(vp.bulkSizeMaxSize) - // inside mx-chain-go, the index is "shardID_epoch" so in order to keep backwards compatibility some adjustments have to be made. - // shardID from index name has to be removed because it is sufficient to have document id = blsKey_epoch - indexWithoutShardID := removeShardIDFromIndex(index) - for _, valRatingInfo := range validatorsRatingInfo { - id := fmt.Sprintf("%s_%s", valRatingInfo.PublicKey, indexWithoutShardID) + for _, ratingInfo := range ratingData.ValidatorsRatingInfo { + id := fmt.Sprintf("%s_%d", ratingInfo.PublicKey, ratingData.Epoch) meta := []byte(fmt.Sprintf(`{ "index" : { "_id" : "%s" } }%s`, id, "\n")) - serializedData, err := json.Marshal(valRatingInfo) + validatorRatingInfo := &data.ValidatorRatingInfo{ + PublicKey: ratingInfo.PublicKey, + Rating: ratingInfo.Rating, + } + serializedData, err := json.Marshal(validatorRatingInfo) if err != nil { continue } @@ -53,12 +34,3 @@ func (vp *validatorsProcessor) SerializeValidatorsRating( return buffSlice.Buffers(), nil } - -func removeShardIDFromIndex(index string) string { - splitIndex := strings.Split(index, "_") - if len(splitIndex) == 2 { - return splitIndex[1] - } - - return index -} diff --git a/process/elasticproc/validators/serialize_test.go b/process/elasticproc/validators/serialize_test.go index a05653b6..8ebea109 100644 --- a/process/elasticproc/validators/serialize_test.go +++ b/process/elasticproc/validators/serialize_test.go @@ -3,36 +3,28 @@ package validators import ( "testing" - "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/stretchr/testify/require" ) -func TestValidatorsProcessor_SerializeValidatorsPubKeys(t *testing.T) { - t.Parallel() - - validatorsPubKeys := &data.ValidatorsPublicKeys{ - PublicKeys: []string{"bls1", "bls2"}, - } - buff, err := (&validatorsProcessor{}).SerializeValidatorsPubKeys(validatorsPubKeys) - require.Nil(t, err) - - expected := `{"publicKeys":["bls1","bls2"]}` - require.Equal(t, expected, buff.String()) -} - func TestValidatorsProcessor_SerializeValidatorsRating(t *testing.T) { t.Parallel() - buff, err := (&validatorsProcessor{}).SerializeValidatorsRating("0", []*data.ValidatorRatingInfo{ - { - PublicKey: "bls1", - Rating: 50.1, - }, - { - PublicKey: "bls3", - Rating: 50.2, + ratingInfo := &outport.ValidatorsRating{ + ShardID: 0, + Epoch: 0, + ValidatorsRatingInfo: []*outport.ValidatorRatingInfo{ + { + PublicKey: "bls1", + Rating: 50.1, + }, + { + PublicKey: "bls3", + Rating: 50.2, + }, }, - }) + } + buff, err := (&validatorsProcessor{}).SerializeValidatorsRating(ratingInfo) require.Nil(t, err) expected := `{ "index" : { "_id" : "bls1_0" } } {"rating":50.1} @@ -41,16 +33,3 @@ func TestValidatorsProcessor_SerializeValidatorsRating(t *testing.T) { ` require.Equal(t, expected, buff[0].String()) } - -func TestRemoveShardIDFromIndex(t *testing.T) { - t.Parallel() - - index := "2_1230" - require.Equal(t, "1230", removeShardIDFromIndex(index)) - - index = "1230" - require.Equal(t, index, removeShardIDFromIndex(index)) - - index = "4294967295_0" - require.Equal(t, "0", removeShardIDFromIndex(index)) -} diff --git a/process/elasticproc/validators/validatorsProcessor.go b/process/elasticproc/validators/validatorsProcessor.go index ee4e2b4c..e70ef815 100644 --- a/process/elasticproc/validators/validatorsProcessor.go +++ b/process/elasticproc/validators/validatorsProcessor.go @@ -1,8 +1,13 @@ package validators import ( + "bytes" + "encoding/json" + "fmt" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" ) @@ -24,17 +29,42 @@ func NewValidatorsProcessor(validatorPubkeyConverter core.PubkeyConverter, bulkS }, nil } -// PrepareValidatorsPublicKeys will prepare validators public keys -func (vp *validatorsProcessor) PrepareValidatorsPublicKeys(shardValidatorsPubKeys [][]byte) *data.ValidatorsPublicKeys { - validatorsPubKeys := &data.ValidatorsPublicKeys{ +// PrepareAnSerializeValidatorsPubKeys will prepare validators public keys and serialize validators public keys +func (vp *validatorsProcessor) PrepareAnSerializeValidatorsPubKeys(validatorsPubKeys *outport.ValidatorsPubKeys) ([]*bytes.Buffer, error) { + buffSlice := data.NewBufferSlice(vp.bulkSizeMaxSize) + + for shardID, validatorPk := range validatorsPubKeys.ShardValidatorsPubKeys { + err := vp.prepareAndSerializeValidatorsKeysForShard(shardID, validatorsPubKeys.Epoch, validatorPk.Keys, buffSlice) + if err != nil { + return nil, err + } + } + + return buffSlice.Buffers(), nil +} + +func (vp *validatorsProcessor) prepareAndSerializeValidatorsKeysForShard(shardID uint32, epoch uint32, keys [][]byte, buffSlice *data.BufferSlice) error { + preparedValidatorsPubKeys := &data.ValidatorsPublicKeys{ PublicKeys: make([]string, 0), } - for _, validatorPk := range shardValidatorsPubKeys { - strValidatorPk := vp.validatorPubkeyConverter.Encode(validatorPk) + for _, key := range keys { + strValidatorPk := vp.validatorPubkeyConverter.Encode(key) + preparedValidatorsPubKeys.PublicKeys = append(preparedValidatorsPubKeys.PublicKeys, strValidatorPk) + } + + id := fmt.Sprintf("%d_%d", shardID, epoch) + meta := []byte(fmt.Sprintf(`{ "index" : { "_id" : "%s" } }%s`, id, "\n")) + + serializedData, err := json.Marshal(preparedValidatorsPubKeys) + if err != nil { + return err + } - validatorsPubKeys.PublicKeys = append(validatorsPubKeys.PublicKeys, strValidatorPk) + err = buffSlice.PutData(meta, serializedData) + if err != nil { + return err } - return validatorsPubKeys + return nil } diff --git a/process/elasticproc/validators/validatorsProcessor_test.go b/process/elasticproc/validators/validatorsProcessor_test.go index 078b2433..ef2de6b6 100644 --- a/process/elasticproc/validators/validatorsProcessor_test.go +++ b/process/elasticproc/validators/validatorsProcessor_test.go @@ -3,8 +3,6 @@ package validators import ( "testing" - "github.com/multiversx/mx-chain-es-indexer-go/data" - "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/stretchr/testify/require" ) @@ -17,18 +15,4 @@ func TestNewValidatorsProcessor(t *testing.T) { require.Equal(t, dataindexer.ErrNilPubkeyConverter, err) } -func TestValidatorsProcessor_PrepareValidatorsPublicKeys(t *testing.T) { - t.Parallel() - - vp, _ := NewValidatorsProcessor(&mock.PubkeyConverterMock{}, 0) - - blsKeys := [][]byte{ - []byte("key1"), []byte("key2"), - } - res := vp.PrepareValidatorsPublicKeys(blsKeys) - require.Equal(t, &data.ValidatorsPublicKeys{ - PublicKeys: []string{ - "6b657931", "6b657932", - }, - }, res) -} +// TODO add unit tests diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index c2e4382c..82c0b47f 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-core-go/websocketOutportDriver/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" @@ -49,61 +50,63 @@ func (i *indexer) GetOperationsMap() map[data.OperationType]func(d []byte) error } func (i *indexer) saveBlock(marshalledData []byte) error { - argsSaveBlockS, err := i.getArgsSaveBlock(marshalledData) + outportBlock := &outport.OutportBlock{} + err := i.marshaller.Unmarshal(outportBlock, marshalledData) if err != nil { return err } - return i.di.SaveBlock(argsSaveBlockS) + return i.di.SaveBlock(outportBlock) } func (i *indexer) revertIndexedBlock(marshalledData []byte) error { - header, body, err := i.getHeaderAndBody(marshalledData) + blockData := &outport.BlockData{} + err := i.marshaller.Unmarshal(blockData, marshalledData) if err != nil { return err } - return i.di.RevertIndexedBlock(header, body) + return i.di.RevertIndexedBlock(blockData) } func (i *indexer) saveRounds(marshalledData []byte) error { - argsRounds := &data.ArgsSaveRoundsInfo{} - err := i.marshaller.Unmarshal(argsRounds, marshalledData) + roundsInfo := &outport.RoundsInfo{} + err := i.marshaller.Unmarshal(roundsInfo, marshalledData) if err != nil { return err } - return i.di.SaveRoundsInfo(argsRounds.RoundsInfos) + return i.di.SaveRoundsInfo(roundsInfo) } func (i *indexer) saveValidatorsRating(marshalledData []byte) error { - argsValidatorsRating := &data.ArgsSaveValidatorsRating{} - err := i.marshaller.Unmarshal(argsValidatorsRating, marshalledData) + ratingData := &outport.ValidatorsRating{} + err := i.marshaller.Unmarshal(ratingData, marshalledData) if err != nil { return err } - return i.di.SaveValidatorsRating(argsValidatorsRating.IndexID, argsValidatorsRating.InfoRating) + return i.di.SaveValidatorsRating(ratingData) } func (i *indexer) saveValidatorsPubKeys(marshalledData []byte) error { - argsValidators := &data.ArgsSaveValidatorsPubKeys{} - err := i.marshaller.Unmarshal(argsValidators, marshalledData) + validatorsPubKeys := &outport.ValidatorsPubKeys{} + err := i.marshaller.Unmarshal(validatorsPubKeys, marshalledData) if err != nil { return err } - return i.di.SaveValidatorsPubKeys(argsValidators.ValidatorsPubKeys, argsValidators.Epoch) + return i.di.SaveValidatorsPubKeys(validatorsPubKeys) } func (i *indexer) saveAccounts(marshalledData []byte) error { - argsSaveAccounts := &data.ArgsSaveAccounts{} - err := i.marshaller.Unmarshal(argsSaveAccounts, marshalledData) + accounts := &outport.Accounts{} + err := i.marshaller.Unmarshal(accounts, marshalledData) if err != nil { return err } - return i.di.SaveAccounts(argsSaveAccounts.BlockTimestamp, argsSaveAccounts.Acc, argsSaveAccounts.ShardID) + return i.di.SaveAccounts(accounts) } func (i *indexer) finalizedBlock(_ []byte) error { diff --git a/process/wsindexer/interface.go b/process/wsindexer/interface.go index 59798a9e..9ae25885 100644 --- a/process/wsindexer/interface.go +++ b/process/wsindexer/interface.go @@ -1,7 +1,6 @@ package wsindexer import ( - "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/outport" ) @@ -13,12 +12,12 @@ type WSClient interface { // DataIndexer dines what a data indexer should do type DataIndexer interface { - SaveBlock(args *outport.ArgsSaveBlockData) error - RevertIndexedBlock(header data.HeaderHandler, body data.BodyHandler) error - SaveRoundsInfo(roundsInfos []*outport.RoundInfo) error - SaveValidatorsPubKeys(validatorsPubKeys map[uint32][][]byte, epoch uint32) error - SaveValidatorsRating(indexID string, infoRating []*outport.ValidatorRatingInfo) error - SaveAccounts(blockTimestamp uint64, acc map[string]*outport.AlteredAccount, shardID uint32) error + SaveBlock(outportBlock *outport.OutportBlock) error + RevertIndexedBlock(blockData *outport.BlockData) error + SaveRoundsInfo(roundsInfos *outport.RoundsInfo) error + SaveValidatorsPubKeys(validatorsPubKeys *outport.ValidatorsPubKeys) error + SaveValidatorsRating(ratingData *outport.ValidatorsRating) error + SaveAccounts(accountsData *outport.Accounts) error FinalizedBlock(headerHash []byte) error Close() error IsInterfaceNil() bool diff --git a/process/wsindexer/types.go b/process/wsindexer/types.go deleted file mode 100644 index 47ae52b1..00000000 --- a/process/wsindexer/types.go +++ /dev/null @@ -1,43 +0,0 @@ -package wsindexer - -import ( - "github.com/multiversx/mx-chain-core-go/data/outport" - "github.com/multiversx/mx-chain-core-go/data/receipt" - "github.com/multiversx/mx-chain-core-go/data/rewardTx" - "github.com/multiversx/mx-chain-core-go/data/smartContractResult" - "github.com/multiversx/mx-chain-core-go/data/transaction" -) - -type normalTxWrapped struct { - TransactionHandler *transaction.Transaction - outport.FeeInfo -} -type rewardsTxsWrapped struct { - TransactionHandler *rewardTx.RewardTx - outport.FeeInfo -} -type scrWrapped struct { - TransactionHandler *smartContractResult.SmartContractResult - outport.FeeInfo -} -type receiptWrapped struct { - TransactionHandler *receipt.Receipt - outport.FeeInfo -} -type logWrapped struct { - TxHash string - LogHandler *transaction.Log -} - -type poolStruct struct { - Txs map[string]*normalTxWrapped - Invalid map[string]*normalTxWrapped - Scrs map[string]*scrWrapped - Rewards map[string]*rewardsTxsWrapped - Receipts map[string]*receiptWrapped - Logs []*logWrapped -} - -type argsSaveBlock struct { - TransactionsPool *poolStruct -} diff --git a/process/wsindexer/unmarshal.go b/process/wsindexer/unmarshal.go deleted file mode 100644 index ed7806c5..00000000 --- a/process/wsindexer/unmarshal.go +++ /dev/null @@ -1,192 +0,0 @@ -package wsindexer - -import ( - "encoding/hex" - "errors" - "time" - - "github.com/multiversx/mx-chain-core-go/core" - "github.com/multiversx/mx-chain-core-go/data" - "github.com/multiversx/mx-chain-core-go/data/block" - "github.com/multiversx/mx-chain-core-go/data/outport" -) - -func (i *indexer) getArgsSaveBlock(marshaledData []byte) (*outport.ArgsSaveBlockData, error) { - defer func(start time.Time) { - log.Debug("indexer.getArgsSaveBlock", "duration", time.Since(start)) - }(time.Now()) - - header, err := i.getHeader(marshaledData) - if err != nil { - return nil, err - } - - body, err := i.getBody(marshaledData) - if err != nil { - return nil, err - } - - txsPool, err := i.getTxsPool(marshaledData) - if err != nil { - return nil, err - } - - argsBlockS := struct { - HeaderHash []byte - SignersIndexes []uint64 - NotarizedHeadersHashes []string - HeaderGasConsumption outport.HeaderGasConsumption - AlteredAccounts map[string]*outport.AlteredAccount - NumberOfShards uint32 - IsImportDB bool - }{} - err = i.marshaller.Unmarshal(&argsBlockS, marshaledData) - if err != nil { - return nil, err - } - - return &outport.ArgsSaveBlockData{ - HeaderHash: argsBlockS.HeaderHash, - Body: body, - Header: header, - SignersIndexes: argsBlockS.SignersIndexes, - NotarizedHeadersHashes: argsBlockS.NotarizedHeadersHashes, - HeaderGasConsumption: argsBlockS.HeaderGasConsumption, - TransactionsPool: txsPool, - AlteredAccounts: argsBlockS.AlteredAccounts, - NumberOfShards: argsBlockS.NumberOfShards, - IsImportDB: argsBlockS.IsImportDB, - }, nil -} - -func (i *indexer) getTxsPool(marshaledData []byte) (*outport.Pool, error) { - argSaveBlock := argsSaveBlock{} - - err := i.marshaller.Unmarshal(&argSaveBlock, marshaledData) - if err != nil { - return nil, err - } - - normalTxs := make(map[string]data.TransactionHandlerWithGasUsedAndFee, len(argSaveBlock.TransactionsPool.Txs)) - for txHash, tx := range argSaveBlock.TransactionsPool.Txs { - decoded := getDecodedHash(txHash) - - normalTxs[decoded] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) - normalTxs[decoded].SetInitialPaidFee(tx.InitialPaidFee) - } - - invalidTxs := make(map[string]data.TransactionHandlerWithGasUsedAndFee, len(argSaveBlock.TransactionsPool.Invalid)) - for txHash, tx := range argSaveBlock.TransactionsPool.Invalid { - decoded := getDecodedHash(txHash) - - invalidTxs[decoded] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) - invalidTxs[decoded].SetInitialPaidFee(tx.InitialPaidFee) - } - - scrs := make(map[string]data.TransactionHandlerWithGasUsedAndFee, len(argSaveBlock.TransactionsPool.Scrs)) - for txHash, tx := range argSaveBlock.TransactionsPool.Scrs { - decoded := getDecodedHash(txHash) - - scrs[decoded] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) - scrs[decoded].SetInitialPaidFee(tx.InitialPaidFee) - } - - receipts := make(map[string]data.TransactionHandlerWithGasUsedAndFee, len(argSaveBlock.TransactionsPool.Receipts)) - for txHash, tx := range argSaveBlock.TransactionsPool.Receipts { - decoded := getDecodedHash(txHash) - - receipts[decoded] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) - receipts[decoded].SetInitialPaidFee(tx.InitialPaidFee) - } - - rewards := make(map[string]data.TransactionHandlerWithGasUsedAndFee, len(argSaveBlock.TransactionsPool.Rewards)) - for txHash, tx := range argSaveBlock.TransactionsPool.Rewards { - decoded := getDecodedHash(txHash) - - rewards[decoded] = outport.NewTransactionHandlerWithGasAndFee(tx.TransactionHandler, tx.GasUsed, tx.Fee) - rewards[decoded].SetInitialPaidFee(tx.InitialPaidFee) - } - - logs := make([]*data.LogData, 0, len(argSaveBlock.TransactionsPool.Logs)) - for _, txLog := range argSaveBlock.TransactionsPool.Logs { - logs = append(logs, &data.LogData{ - LogHandler: txLog.LogHandler, - TxHash: getDecodedHash(txLog.TxHash), - }) - } - - return &outport.Pool{ - Txs: normalTxs, - Scrs: scrs, - Rewards: rewards, - Invalid: invalidTxs, - Receipts: receipts, - Logs: logs, - }, nil -} - -func getDecodedHash(hash string) string { - decoded, err := hex.DecodeString(hash) - if err != nil { - log.Warn("getDecodedHash.cannot decode hash", "error", err, "hash", hash) - return hash - } - return string(decoded) -} - -func (i *indexer) getHeaderAndBody(marshaledData []byte) (data.HeaderHandler, data.BodyHandler, error) { - body, err := i.getBody(marshaledData) - if err != nil { - return nil, nil, err - } - - header, err := i.getHeader(marshaledData) - if err != nil { - return nil, nil, err - } - - return header, body, nil -} - -func (i *indexer) getBody(marshaledData []byte) (data.BodyHandler, error) { - bodyStruct := struct { - Body *block.Body - }{} - - err := i.marshaller.Unmarshal(&bodyStruct, marshaledData) - return bodyStruct.Body, err -} - -func (i *indexer) getHeader(marshaledData []byte) (data.HeaderHandler, error) { - headerTypeStruct := struct { - HeaderType core.HeaderType - }{} - - err := i.marshaller.Unmarshal(&headerTypeStruct, marshaledData) - if err != nil { - return nil, err - } - - switch headerTypeStruct.HeaderType { - case core.MetaHeader: - hStruct := struct { - H1 *block.MetaBlock `json:"Header"` - }{} - err = i.marshaller.Unmarshal(&hStruct, marshaledData) - return hStruct.H1, err - case core.ShardHeaderV1: - hStruct := struct { - H1 *block.Header `json:"Header"` - }{} - err = i.marshaller.Unmarshal(&hStruct, marshaledData) - return hStruct.H1, err - case core.ShardHeaderV2: - hStruct := struct { - H1 *block.HeaderV2 `json:"Header"` - }{} - err = i.marshaller.Unmarshal(&hStruct, marshaledData) - return hStruct.H1, err - default: - return nil, errors.New("invalid header type") - } -} diff --git a/process/wsindexer/unmarshal_test.go b/process/wsindexer/unmarshal_test.go deleted file mode 100644 index 31c50f4f..00000000 --- a/process/wsindexer/unmarshal_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package wsindexer - -import ( - "math/big" - "testing" - - "github.com/multiversx/mx-chain-core-go/core" - coreData "github.com/multiversx/mx-chain-core-go/data" - "github.com/multiversx/mx-chain-core-go/data/block" - "github.com/multiversx/mx-chain-core-go/data/outport" - "github.com/multiversx/mx-chain-core-go/data/receipt" - "github.com/multiversx/mx-chain-core-go/data/rewardTx" - "github.com/multiversx/mx-chain-core-go/data/smartContractResult" - "github.com/multiversx/mx-chain-core-go/data/transaction" - "github.com/multiversx/mx-chain-core-go/marshal/factory" - outportData "github.com/multiversx/mx-chain-core-go/websocketOutportDriver/data" - "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" - "github.com/stretchr/testify/require" -) - -func TestGetHeaderAndBody(t *testing.T) { - t.Parallel() - - marshaller, _ := factory.NewMarshalizer("json") - nilDataIndexer := dataindexer.NewNilIndexer() - - di, _ := NewIndexer(marshaller, nilDataIndexer) - - arg := &outportData.ArgsRevertIndexedBlock{ - HeaderType: core.ShardHeaderV2, - Header: &block.HeaderV2{ScheduledRootHash: []byte("aaaaaa")}, - Body: &block.Body{MiniBlocks: []*block.MiniBlock{{}}}, - } - argBytes, _ := marshaller.Marshal(arg) - - body, header, err := di.getHeaderAndBody(argBytes) - require.Nil(t, err) - require.NotNil(t, body) - require.NotNil(t, header) -} - -func TestGetPool(t *testing.T) { - t.Parallel() - - marshaller, _ := factory.NewMarshalizer("json") - nilDataIndexer := dataindexer.NewNilIndexer() - - di, _ := NewIndexer(marshaller, nilDataIndexer) - - pool := &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - "txHash": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - Nonce: 1, - }, 1, big.NewInt(100)), - }, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - "scrHash": outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ - Nonce: 2, - }, 0, big.NewInt(0)), - }, - Rewards: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - "reward": outport.NewTransactionHandlerWithGasAndFee(&rewardTx.RewardTx{ - Value: big.NewInt(10), - }, 0, big.NewInt(0)), - }, - Invalid: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - "invalid": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - Nonce: 3, - }, 100, big.NewInt(1000)), - }, - Receipts: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - "rec": outport.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ - Value: big.NewInt(300), - }, 0, big.NewInt(0)), - }, - Logs: []*coreData.LogData{ - { - TxHash: "something", - LogHandler: &transaction.Log{ - Address: []byte("addr"), - }, - }, - }, - } - - argsSaveBlock := &outportData.ArgsSaveBlock{ - ArgsSaveBlockData: outport.ArgsSaveBlockData{ - TransactionsPool: pool, - }, - } - - argsSaveBlockBytes, _ := di.marshaller.Marshal(argsSaveBlock) - - resPool, err := di.getTxsPool(argsSaveBlockBytes) - require.Nil(t, err) - require.NotNil(t, resPool) - require.Equal(t, 1, len(resPool.Txs)) - require.Equal(t, 1, len(resPool.Scrs)) - require.Equal(t, 1, len(resPool.Rewards)) - require.Equal(t, 1, len(resPool.Receipts)) - require.Equal(t, 1, len(resPool.Invalid)) - require.Equal(t, 1, len(resPool.Logs)) -} From 756151608bcc334d3778421f30a1f79c2540c82e Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 14 Mar 2023 16:36:51 +0200 Subject: [PATCH 014/189] fix binary --- process/dataindexer/dataIndexer.go | 2 +- process/dataindexer/nilIndexer.go | 13 ++++++------- process/elasticproc/block/blockProcessor.go | 2 +- process/elasticproc/block/blockProcessor_test.go | 10 +++++----- process/elasticproc/elasticProcessor.go | 13 +++++++++++-- 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/process/dataindexer/dataIndexer.go b/process/dataindexer/dataIndexer.go index 3fd13dfa..56aae488 100644 --- a/process/dataindexer/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -65,7 +65,7 @@ func (di *dataIndexer) SaveBlock(outportBlock *outport.OutportBlock) error { wi := workItems.NewItemBlock( di.elasticProcessor, &outport.OutportBlockWithHeader{ - OutportBlock: *outportBlock, + OutportBlock: outportBlock, Header: header, }, ) diff --git a/process/dataindexer/nilIndexer.go b/process/dataindexer/nilIndexer.go index e4009687..b7205e96 100644 --- a/process/dataindexer/nilIndexer.go +++ b/process/dataindexer/nilIndexer.go @@ -1,7 +1,6 @@ package dataindexer import ( - "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/outport" ) @@ -15,32 +14,32 @@ func NewNilIndexer() *NilIndexer { } // SaveBlock returns nil -func (ni *NilIndexer) SaveBlock(_ *outport.ArgsSaveBlockData) error { +func (ni *NilIndexer) SaveBlock(_ *outport.OutportBlock) error { return nil } // RevertIndexedBlock returns nil -func (ni *NilIndexer) RevertIndexedBlock(_ data.HeaderHandler, _ data.BodyHandler) error { +func (ni *NilIndexer) RevertIndexedBlock(_ *outport.BlockData) error { return nil } // SaveRoundsInfo returns nil -func (ni *NilIndexer) SaveRoundsInfo(_ []*outport.RoundInfo) error { +func (ni *NilIndexer) SaveRoundsInfo(_ *outport.RoundsInfo) error { return nil } // SaveValidatorsRating returns nil -func (ni *NilIndexer) SaveValidatorsRating(_ string, _ []*outport.ValidatorRatingInfo) error { +func (ni *NilIndexer) SaveValidatorsRating(_ *outport.ValidatorsRating) error { return nil } // SaveValidatorsPubKeys returns nil -func (ni *NilIndexer) SaveValidatorsPubKeys(_ map[uint32][][]byte, _ uint32) error { +func (ni *NilIndexer) SaveValidatorsPubKeys(_ *outport.ValidatorsPubKeys) error { return nil } // SaveAccounts returns nil -func (ni *NilIndexer) SaveAccounts(_ uint64, _ map[string]*outport.AlteredAccount, _ uint32) error { +func (ni *NilIndexer) SaveAccounts(_ *outport.Accounts) error { return nil } diff --git a/process/elasticproc/block/blockProcessor.go b/process/elasticproc/block/blockProcessor.go index 49ffcb2c..6aab62cc 100644 --- a/process/elasticproc/block/blockProcessor.go +++ b/process/elasticproc/block/blockProcessor.go @@ -283,7 +283,7 @@ func getExecutionOrderForTx(txHash []byte, mbType int32, pool *outport.Transacti var tx executionOrderHandler var found bool - switch mbType { + switch nodeBlock.Type(mbType) { case nodeBlock.TxBlock: tx, found = pool.Transactions[hex.EncodeToString(txHash)] case nodeBlock.InvalidBlock: diff --git a/process/elasticproc/block/blockProcessor_test.go b/process/elasticproc/block/blockProcessor_test.go index cd03d290..3a90f280 100644 --- a/process/elasticproc/block/blockProcessor_test.go +++ b/process/elasticproc/block/blockProcessor_test.go @@ -61,7 +61,7 @@ func TestBlockProcessor_PrepareBlockForDBShouldWork(t *testing.T) { outportBlockWithHeader := &outport.OutportBlockWithHeader{ Header: &dataBlock.Header{}, - OutportBlock: outport.OutportBlock{ + OutportBlock: &outport.OutportBlock{ BlockData: &outport.BlockData{ HeaderHash: []byte("hash"), Body: &dataBlock.Body{ @@ -133,7 +133,7 @@ func TestBlockProcessor_PrepareBlockForDBMarshalFailHeader(t *testing.T) { outportBlockWithHeader := &outport.OutportBlockWithHeader{ Header: &dataBlock.Header{}, - OutportBlock: outport.OutportBlock{ + OutportBlock: &outport.OutportBlock{ BlockData: &outport.BlockData{ HeaderHash: []byte("hash"), Body: &dataBlock.Body{}, @@ -167,7 +167,7 @@ func TestBlockProcessor_PrepareBlockForDBMarshalFailBlock(t *testing.T) { outportBlockWithHeader := &outport.OutportBlockWithHeader{ Header: &dataBlock.Header{}, - OutportBlock: outport.OutportBlock{ + OutportBlock: &outport.OutportBlock{ BlockData: &outport.BlockData{ HeaderHash: []byte("hash"), Body: &dataBlock.Body{}, @@ -242,7 +242,7 @@ func TestBlockProcessor_PrepareBlockForDBEpochStartMeta(t *testing.T) { }, }, }, - OutportBlock: outport.OutportBlock{ + OutportBlock: &outport.OutportBlock{ BlockData: &outport.BlockData{ HeaderHash: []byte("hash"), Body: &dataBlock.Body{ @@ -373,7 +373,7 @@ func TestBlockProcessor_PrepareBlockForDBMiniBlocksDetails(t *testing.T) { }, }, }, - OutportBlock: outport.OutportBlock{ + OutportBlock: &outport.OutportBlock{ BlockData: &outport.BlockData{ HeaderHash: []byte("hash"), Body: &dataBlock.Body{ diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index a3559fbb..8e845b6a 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -721,9 +721,18 @@ func (ei *elasticProcessor) indexNFTBurnInfo(tokensData data.TokensHandler, buff } // SaveAccounts will prepare and save information about provided accounts in elasticsearch server -func (ei *elasticProcessor) SaveAccounts(timestamp uint64, accts []*data.Account, shardID uint32) error { +func (ei *elasticProcessor) SaveAccounts(accountsData *outport.Accounts) error { buffSlice := data.NewBufferSlice(ei.bulkRequestMaxSize) - return ei.saveAccounts(timestamp, accts, buffSlice, shardID) + + accounts := make([]*data.Account, 0, len(accountsData.AlteredAccounts)) + for _, account := range accountsData.AlteredAccounts { + accounts = append(accounts, &data.Account{ + UserAccount: account, + IsSender: false, + }) + } + + return ei.saveAccounts(accountsData.BlockTimestamp, accounts, buffSlice, accountsData.ShardID) } func (ei *elasticProcessor) saveAccounts(timestamp uint64, accts []*data.Account, buffSlice *data.BufferSlice, shardID uint32) error { From 4e4bc4a1706cf8484dff3d03b38f89a790e65a54 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 14 Mar 2023 16:44:56 +0200 Subject: [PATCH 015/189] fix mocks --- mock/dbAccountsHandlerStub.go | 6 ++-- mock/dbTransactionsHandlerStub.go | 4 +-- mock/elasticProcessorStub.go | 52 ++++++++++--------------------- 3 files changed, 22 insertions(+), 40 deletions(-) diff --git a/mock/dbAccountsHandlerStub.go b/mock/dbAccountsHandlerStub.go index 84899938..35747c27 100644 --- a/mock/dbAccountsHandlerStub.go +++ b/mock/dbAccountsHandlerStub.go @@ -1,7 +1,7 @@ package mock import ( - "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-core-go/data/alteredAccount" "github.com/multiversx/mx-chain-es-indexer-go/data" ) @@ -12,7 +12,7 @@ type DBAccountsHandlerStub struct { } // GetAccounts - -func (dba *DBAccountsHandlerStub) GetAccounts(_ map[string]*outport.AlteredAccount) ([]*data.Account, []*data.AccountESDT) { +func (dba *DBAccountsHandlerStub) GetAccounts(_ map[string]*alteredAccount.AlteredAccount) ([]*data.Account, []*data.AccountESDT) { return nil, nil } @@ -59,7 +59,7 @@ func (dba *DBAccountsHandlerStub) SerializeNFTCreateInfo(_ []*data.TokenInfo, _ } // PutTokenMedataDataInTokens - -func (dba *DBAccountsHandlerStub) PutTokenMedataDataInTokens(_ []*data.TokenInfo, _ map[string]*outport.AlteredAccount) { +func (dba *DBAccountsHandlerStub) PutTokenMedataDataInTokens(_ []*data.TokenInfo, _ map[string]*alteredAccount.AlteredAccount) { } // SerializeTypeForProvidedIDs - diff --git a/mock/dbTransactionsHandlerStub.go b/mock/dbTransactionsHandlerStub.go index 9ec841ba..15b50e11 100644 --- a/mock/dbTransactionsHandlerStub.go +++ b/mock/dbTransactionsHandlerStub.go @@ -9,7 +9,7 @@ import ( // DBTransactionProcessorStub - type DBTransactionProcessorStub struct { - PrepareTransactionsForDatabaseCalled func(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool) *data.PreparedResults + PrepareTransactionsForDatabaseCalled func(body *block.Body, header coreData.HeaderHandler, pool *outport.TransactionPool) *data.PreparedResults SerializeReceiptsCalled func(recs []*data.Receipt, buffSlice *data.BufferSlice, index string) error SerializeScResultsCalled func(scrs []*data.ScResult, buffSlice *data.BufferSlice, index string) error } @@ -20,7 +20,7 @@ func (tps *DBTransactionProcessorStub) SerializeTransactionsFeeData(_ map[string } // PrepareTransactionsForDatabase - -func (tps *DBTransactionProcessorStub) PrepareTransactionsForDatabase(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, _ bool, _ uint32) *data.PreparedResults { +func (tps *DBTransactionProcessorStub) PrepareTransactionsForDatabase(body *block.Body, header coreData.HeaderHandler, pool *outport.TransactionPool, _ bool, _ uint32) *data.PreparedResults { if tps.PrepareTransactionsForDatabaseCalled != nil { return tps.PrepareTransactionsForDatabaseCalled(body, header, pool) } diff --git a/mock/elasticProcessorStub.go b/mock/elasticProcessorStub.go index a49eedd3..d8aa1000 100644 --- a/mock/elasticProcessorStub.go +++ b/mock/elasticProcessorStub.go @@ -4,30 +4,20 @@ import ( coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" - "github.com/multiversx/mx-chain-es-indexer-go/data" ) // ElasticProcessorStub - type ElasticProcessorStub struct { - SaveHeaderCalled func( - headerHash []byte, - header coreData.HeaderHandler, - signersIndexes []uint64, - body *block.Body, - notarizedHeadersHashes []string, - gasConsumptionData outport.HeaderGasConsumption, - txsSize int, - pool *outport.Pool, - ) error + SaveHeaderCalled func(outportBlockWithHeader *outport.OutportBlockWithHeader) error RemoveHeaderCalled func(header coreData.HeaderHandler) error RemoveMiniblocksCalled func(header coreData.HeaderHandler, body *block.Body) error RemoveTransactionsCalled func(header coreData.HeaderHandler, body *block.Body) error SaveMiniblocksCalled func(header coreData.HeaderHandler, body *block.Body) error - SaveTransactionsCalled func(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount) error - SaveValidatorsRatingCalled func(index string, validatorsRatingInfo []*data.ValidatorRatingInfo) error - SaveRoundsInfoCalled func(infos []*data.RoundInfo) error - SaveShardValidatorsPubKeysCalled func(shardID, epoch uint32, shardValidatorsPubKeys [][]byte) error - SaveAccountsCalled func(timestamp uint64, acc []*data.Account) error + SaveTransactionsCalled func(outportBlockWithHeader *outport.OutportBlockWithHeader) error + SaveValidatorsRatingCalled func(validatorsRating *outport.ValidatorsRating) error + SaveRoundsInfoCalled func(infos *outport.RoundsInfo) error + SaveShardValidatorsPubKeysCalled func(validators *outport.ValidatorsPubKeys) error + SaveAccountsCalled func(accountsData *outport.Accounts) error RemoveAccountsESDTCalled func(headerTimestamp uint64) error } @@ -41,17 +31,9 @@ func (eim *ElasticProcessorStub) RemoveAccountsESDT(headerTimestamp uint64, _ ui } // SaveHeader - -func (eim *ElasticProcessorStub) SaveHeader( - headerHash []byte, - header coreData.HeaderHandler, - signersIndexes []uint64, - body *block.Body, - notarizedHeadersHashes []string, - gasConsumptionData outport.HeaderGasConsumption, - txsSize int, - pool *outport.Pool) error { +func (eim *ElasticProcessorStub) SaveHeader(obh *outport.OutportBlockWithHeader) error { if eim.SaveHeaderCalled != nil { - return eim.SaveHeaderCalled(headerHash, header, signersIndexes, body, notarizedHeadersHashes, gasConsumptionData, txsSize, pool) + return eim.SaveHeaderCalled(obh) } return nil } @@ -89,23 +71,23 @@ func (eim *ElasticProcessorStub) SaveMiniblocks(header coreData.HeaderHandler, b } // SaveTransactions - -func (eim *ElasticProcessorStub) SaveTransactions(body *block.Body, header coreData.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount, _ bool, _ uint32) error { +func (eim *ElasticProcessorStub) SaveTransactions(outportBlockWithHeader *outport.OutportBlockWithHeader) error { if eim.SaveTransactionsCalled != nil { - return eim.SaveTransactionsCalled(body, header, pool, coreAlteredAccounts) + return eim.SaveTransactionsCalled(outportBlockWithHeader) } return nil } // SaveValidatorsRating - -func (eim *ElasticProcessorStub) SaveValidatorsRating(index string, validatorsRatingInfo []*data.ValidatorRatingInfo) error { +func (eim *ElasticProcessorStub) SaveValidatorsRating(validatorsRating *outport.ValidatorsRating) error { if eim.SaveValidatorsRatingCalled != nil { - return eim.SaveValidatorsRatingCalled(index, validatorsRatingInfo) + return eim.SaveValidatorsRatingCalled(validatorsRating) } return nil } // SaveRoundsInfo - -func (eim *ElasticProcessorStub) SaveRoundsInfo(info []*data.RoundInfo) error { +func (eim *ElasticProcessorStub) SaveRoundsInfo(info *outport.RoundsInfo) error { if eim.SaveRoundsInfoCalled != nil { return eim.SaveRoundsInfoCalled(info) } @@ -113,17 +95,17 @@ func (eim *ElasticProcessorStub) SaveRoundsInfo(info []*data.RoundInfo) error { } // SaveShardValidatorsPubKeys - -func (eim *ElasticProcessorStub) SaveShardValidatorsPubKeys(shardID, epoch uint32, shardValidatorsPubKeys [][]byte) error { +func (eim *ElasticProcessorStub) SaveShardValidatorsPubKeys(validatorsPubKeys *outport.ValidatorsPubKeys) error { if eim.SaveShardValidatorsPubKeysCalled != nil { - return eim.SaveShardValidatorsPubKeysCalled(shardID, epoch, shardValidatorsPubKeys) + return eim.SaveShardValidatorsPubKeysCalled(validatorsPubKeys) } return nil } // SaveAccounts - -func (eim *ElasticProcessorStub) SaveAccounts(timestamp uint64, acc []*data.Account, _ uint32) error { +func (eim *ElasticProcessorStub) SaveAccounts(accounts *outport.Accounts) error { if eim.SaveAccountsCalled != nil { - return eim.SaveAccountsCalled(timestamp, acc) + return eim.SaveAccountsCalled(accounts) } return nil From a08f7f7021bdd58e649d06197ac7943f6ecaa88a Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 14 Mar 2023 16:51:55 +0200 Subject: [PATCH 016/189] fix interface --- process/dataindexer/dataIndexer.go | 2 +- process/dataindexer/interface.go | 2 +- process/wsindexer/interface.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/process/dataindexer/dataIndexer.go b/process/dataindexer/dataIndexer.go index 56aae488..efb1256e 100644 --- a/process/dataindexer/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -131,7 +131,7 @@ func (di *dataIndexer) SaveAccounts(accounts *outport.Accounts) error { } // FinalizedBlock returns nil -func (di *dataIndexer) FinalizedBlock(_ []byte) error { +func (di *dataIndexer) FinalizedBlock(_ *outport.FinalizedBlock) error { return nil } diff --git a/process/dataindexer/interface.go b/process/dataindexer/interface.go index 3469384a..f50d98a4 100644 --- a/process/dataindexer/interface.go +++ b/process/dataindexer/interface.go @@ -60,7 +60,7 @@ type Indexer interface { SaveValidatorsPubKeys(validatorsPubKeys *outport.ValidatorsPubKeys) error SaveValidatorsRating(ratingData *outport.ValidatorsRating) error SaveAccounts(accountsData *outport.Accounts) error - FinalizedBlock(headerHash []byte) error + FinalizedBlock(finalizedBlock *outport.FinalizedBlock) error Close() error IsInterfaceNil() bool IsNilIndexer() bool diff --git a/process/wsindexer/interface.go b/process/wsindexer/interface.go index 9ae25885..91caee7d 100644 --- a/process/wsindexer/interface.go +++ b/process/wsindexer/interface.go @@ -18,7 +18,7 @@ type DataIndexer interface { SaveValidatorsPubKeys(validatorsPubKeys *outport.ValidatorsPubKeys) error SaveValidatorsRating(ratingData *outport.ValidatorsRating) error SaveAccounts(accountsData *outport.Accounts) error - FinalizedBlock(headerHash []byte) error + FinalizedBlock(finalizedBlock *outport.FinalizedBlock) error Close() error IsInterfaceNil() bool } From 937ced0be86393b23ab1115714cfead26bfef944 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 15 Mar 2023 12:14:34 +0200 Subject: [PATCH 017/189] fix unit tests 1 --- factory/wsIndexerFactory.go | 8 +- process/dataindexer/dataDispatcher_test.go | 34 ++--- process/dataindexer/dataIndexer.go | 17 ++- process/dataindexer/dataIndexer_test.go | 38 ++--- process/dataindexer/nilIndexer.go | 5 - .../workItems/workItemAccounts_test.go | 11 +- .../dataindexer/workItems/workItemBlock.go | 35 +---- .../workItems/workItemBlock_test.go | 134 ++++++------------ .../workItems/workItemRating_test.go | 19 +-- .../workItems/workItemRemoveBlock.go | 35 ++--- .../workItems/workItemRemoveBlock_test.go | 6 +- .../workItems/workItemRounds_test.go | 14 +- .../workItems/workItemValidators_test.go | 17 +-- process/factory/indexerFactory.go | 6 +- 14 files changed, 136 insertions(+), 243 deletions(-) diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 5bedd89a..eb5d21fe 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -3,6 +3,7 @@ package factory import ( "github.com/multiversx/mx-chain-core-go/core/pubkeyConverter" factoryHasher "github.com/multiversx/mx-chain-core-go/hashing/factory" + "github.com/multiversx/mx-chain-core-go/marshal" factoryMarshaller "github.com/multiversx/mx-chain-core-go/marshal/factory" "github.com/multiversx/mx-chain-es-indexer-go/config" "github.com/multiversx/mx-chain-es-indexer-go/process/factory" @@ -19,12 +20,12 @@ const ( // CreateWsIndexer will create a new instance of wsindexer.WSClient func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsindexer.WSClient, error) { - dataIndexer, err := createDataIndexer(cfg, clusterCfg) + wsMarshaller, err := factoryMarshaller.NewMarshalizer(clusterCfg.Config.WebSocket.DataMarshallerType) if err != nil { return nil, err } - wsMarshaller, err := factoryMarshaller.NewMarshalizer(clusterCfg.Config.WebSocket.DataMarshallerType) + dataIndexer, err := createDataIndexer(cfg, clusterCfg, wsMarshaller) if err != nil { return nil, err } @@ -37,7 +38,7 @@ func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsinde return wsclient.New(clusterCfg.Config.WebSocket.ServerURL, indexer) } -func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsindexer.DataIndexer, error) { +func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig, wsMarshaller marshal.Marshalizer) (wsindexer.DataIndexer, error) { marshaller, err := factoryMarshaller.NewMarshalizer(cfg.Config.Marshaller.Type) if err != nil { return nil, err @@ -68,6 +69,7 @@ func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsin Hasher: hasher, AddressPubkeyConverter: addressPubkeyConverter, ValidatorPubkeyConverter: validatorPubkeyConverter, + HeaderMarshaller: wsMarshaller, }) } diff --git a/process/dataindexer/dataDispatcher_test.go b/process/dataindexer/dataDispatcher_test.go index cd7da157..fb7d3c2b 100644 --- a/process/dataindexer/dataDispatcher_test.go +++ b/process/dataindexer/dataDispatcher_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" "github.com/stretchr/testify/require" @@ -43,32 +43,32 @@ func TestDataDispatcher_StartIndexDataClose(t *testing.T) { wg := sync.WaitGroup{} wg.Add(1) elasticProc := &mock.ElasticProcessorStub{ - SaveRoundsInfoCalled: func(infos []*data.RoundInfo) error { + SaveRoundsInfoCalled: func(infos *outport.RoundsInfo) error { called = true wg.Done() return nil }, - SaveAccountsCalled: func(timestamp uint64, acc []*data.Account) error { + SaveAccountsCalled: func(accounts *outport.Accounts) error { time.Sleep(7 * time.Second) return nil }, - SaveValidatorsRatingCalled: func(index string, validatorsRatingInfo []*data.ValidatorRatingInfo) error { + SaveValidatorsRatingCalled: func(_ *outport.ValidatorsRating) error { time.Sleep(6 * time.Second) return nil }, } - dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems.NewItemRounds(elasticProc, &outport.RoundsInfo{})) wg.Wait() require.True(t, called) - dispatcher.Add(workItems.NewItemAccounts(elasticProc, 0, nil, 0)) + dispatcher.Add(workItems.NewItemAccounts(elasticProc, &outport.Accounts{})) wg.Add(1) - dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) - dispatcher.Add(workItems.NewItemRating(elasticProc, "", nil)) + dispatcher.Add(workItems.NewItemRounds(elasticProc, &outport.RoundsInfo{})) + dispatcher.Add(workItems.NewItemRating(elasticProc, &outport.ValidatorsRating{})) wg.Add(1) - dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems.NewItemRounds(elasticProc, &outport.RoundsInfo{})) err = dispatcher.Close() require.NoError(t, err) } @@ -84,7 +84,7 @@ func TestDataDispatcher_Add(t *testing.T) { wg := sync.WaitGroup{} wg.Add(1) elasticProc := &mock.ElasticProcessorStub{ - SaveRoundsInfoCalled: func(infos []*data.RoundInfo) error { + SaveRoundsInfoCalled: func(_ *outport.RoundsInfo) error { if calledCount < 2 { atomic.AddUint32(&calledCount, 1) return fmt.Errorf("%w: wrapped error", ErrBackOff) @@ -97,7 +97,7 @@ func TestDataDispatcher_Add(t *testing.T) { } start := time.Now() - dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems.NewItemRounds(elasticProc, &outport.RoundsInfo{})) wg.Wait() timePassed := time.Since(start) @@ -120,7 +120,7 @@ func TestDataDispatcher_AddWithErrorShouldRetryTheReprocessing(t *testing.T) { wg := sync.WaitGroup{} wg.Add(1) elasticProc := &mock.ElasticProcessorStub{ - SaveRoundsInfoCalled: func(infos []*data.RoundInfo) error { + SaveRoundsInfoCalled: func(_ *outport.RoundsInfo) error { if calledCount < 2 { atomic.AddUint32(&calledCount, 1) return errors.New("generic error") @@ -133,7 +133,7 @@ func TestDataDispatcher_AddWithErrorShouldRetryTheReprocessing(t *testing.T) { } start := time.Now() - dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems.NewItemRounds(elasticProc, &outport.RoundsInfo{})) wg.Wait() timePassed := time.Since(start) @@ -153,7 +153,7 @@ func TestDataDispatcher_Close(t *testing.T) { dispatcher.StartIndexData() elasticProc := &mock.ElasticProcessorStub{ - SaveRoundsInfoCalled: func(infos []*data.RoundInfo) error { + SaveRoundsInfoCalled: func(_ *outport.RoundsInfo) error { time.Sleep(1000*time.Millisecond + 200*time.Microsecond) return nil }, @@ -173,7 +173,7 @@ func TestDataDispatcher_Close(t *testing.T) { if count == 105 { w.Done() } - dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems.NewItemRounds(elasticProc, &outport.RoundsInfo{})) time.Sleep(50 * time.Millisecond) } } @@ -199,11 +199,11 @@ func TestDataDispatcher_RecoverPanic(t *testing.T) { require.NoError(t, err) elasticProc := &mock.ElasticProcessorStub{ - SaveRoundsInfoCalled: func(infos []*data.RoundInfo) error { + SaveRoundsInfoCalled: func(_ *outport.RoundsInfo) error { panic(1) }, } - dispatcher.Add(workItems.NewItemRounds(elasticProc, []*data.RoundInfo{})) + dispatcher.Add(workItems.NewItemRounds(elasticProc, &outport.RoundsInfo{})) dispatcher.doDataDispatch(context.Background()) } diff --git a/process/dataindexer/dataIndexer.go b/process/dataindexer/dataIndexer.go index efb1256e..e8930516 100644 --- a/process/dataindexer/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -11,7 +11,7 @@ import ( // ArgDataIndexer is a structure that is used to store all the components that are needed to create an indexer type ArgDataIndexer struct { - Marshalizer marshal.Marshalizer + HeaderMarshaller marshal.Marshalizer DataDispatcher DispatcherHandler ElasticProcessor ElasticProcessor } @@ -20,7 +20,6 @@ type dataIndexer struct { isNilIndexer bool dispatcher DispatcherHandler elasticProcessor ElasticProcessor - marshaller marshal.Marshalizer headerMarshaller marshal.Marshalizer } @@ -35,7 +34,7 @@ func NewDataIndexer(arguments ArgDataIndexer) (*dataIndexer, error) { isNilIndexer: false, dispatcher: arguments.DataDispatcher, elasticProcessor: arguments.ElasticProcessor, - marshaller: arguments.Marshalizer, + headerMarshaller: arguments.HeaderMarshaller, } return dataIndexerObj, nil @@ -48,7 +47,7 @@ func checkIndexerArgs(arguments ArgDataIndexer) error { if check.IfNil(arguments.ElasticProcessor) { return ErrNilElasticProcessor } - if check.IfNil(arguments.Marshalizer) { + if check.IfNil(arguments.HeaderMarshaller) { return ErrNilMarshalizer } @@ -81,11 +80,15 @@ func (di *dataIndexer) Close() error { // RevertIndexedBlock will remove from database block and miniblocks func (di *dataIndexer) RevertIndexedBlock(blockData *outport.BlockData) error { + header, err := unmarshal.GetHeaderFromBytes(di.headerMarshaller, core.HeaderType(blockData.HeaderType), blockData.HeaderBytes) + if err != nil { + return err + } + wi := workItems.NewItemRemoveBlock( di.elasticProcessor, - // TODO possible to be json or proto marshaller - di.headerMarshaller, - blockData, + header, + blockData.Body, ) di.dispatcher.Add(wi) diff --git a/process/dataindexer/dataIndexer_test.go b/process/dataindexer/dataIndexer_test.go index 5938110c..30b73f05 100644 --- a/process/dataindexer/dataIndexer_test.go +++ b/process/dataindexer/dataIndexer_test.go @@ -14,9 +14,9 @@ import ( func NewDataIndexerArguments() ArgDataIndexer { return ArgDataIndexer{ - Marshalizer: &mock.MarshalizerMock{}, DataDispatcher: &mock.DispatcherMock{}, ElasticProcessor: &mock.ElasticProcessorStub{}, + HeaderMarshaller: &mock.MarshalizerMock{}, } } @@ -40,7 +40,7 @@ func TestDataIndexer_NewIndexerWithNilElasticProcessorShouldErr(t *testing.T) { func TestDataIndexer_NewIndexerWithNilMarshalizerShouldErr(t *testing.T) { arguments := NewDataIndexerArguments() - arguments.Marshalizer = nil + arguments.HeaderMarshaller = nil ei, err := NewDataIndexer(arguments) require.Nil(t, ei) @@ -68,17 +68,16 @@ func TestDataIndexer_SaveBlock(t *testing.T) { } ei, _ := NewDataIndexer(arguments) - args := &outport.ArgsSaveBlockData{ - HeaderHash: []byte("hash"), - Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{}}, - Header: nil, - SignersIndexes: nil, - NotarizedHeadersHashes: nil, - TransactionsPool: nil, + args := &outport.OutportBlock{ + BlockData: &outport.BlockData{ + HeaderType: string(core.ShardHeaderV2), + Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{{}}}, + HeaderBytes: []byte("{}"), + }, } err := ei.SaveBlock(args) - require.True(t, called) require.Nil(t, err) + require.True(t, called) } func TestDataIndexer_SaveRoundInfo(t *testing.T) { @@ -91,11 +90,11 @@ func TestDataIndexer_SaveRoundInfo(t *testing.T) { }, } - arguments.Marshalizer = &mock.MarshalizerMock{Fail: true} + arguments.HeaderMarshaller = &mock.MarshalizerMock{Fail: true} ei, _ := NewDataIndexer(arguments) _ = ei.Close() - err := ei.SaveRoundsInfo([]*outport.RoundInfo{}) + err := ei.SaveRoundsInfo(&outport.RoundsInfo{}) require.True(t, called) require.Nil(t, err) } @@ -115,9 +114,8 @@ func TestDataIndexer_SaveValidatorsPubKeys(t *testing.T) { keys := [][]byte{[]byte("key")} valPubKey[0] = keys - epoch := uint32(0) - err := ei.SaveValidatorsPubKeys(valPubKey, epoch) + err := ei.SaveValidatorsPubKeys(&outport.ValidatorsPubKeys{}) require.True(t, called) require.Nil(t, err) } @@ -133,9 +131,7 @@ func TestDataIndexer_SaveValidatorsRating(t *testing.T) { } ei, _ := NewDataIndexer(arguments) - err := ei.SaveValidatorsRating("ID", []*outport.ValidatorRatingInfo{ - {Rating: 1}, {Rating: 2}, - }) + err := ei.SaveValidatorsRating(&outport.ValidatorsRating{}) require.True(t, called) require.Nil(t, err) } @@ -151,7 +147,11 @@ func TestDataIndexer_RevertIndexedBlock(t *testing.T) { } ei, _ := NewDataIndexer(arguments) - err := ei.RevertIndexedBlock(&dataBlock.Header{}, &dataBlock.Body{}) - require.True(t, called) + err := ei.RevertIndexedBlock(&outport.BlockData{ + HeaderType: string(core.ShardHeaderV2), + Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{{}}}, + HeaderBytes: []byte("{}"), + }) require.Nil(t, err) + require.True(t, called) } diff --git a/process/dataindexer/nilIndexer.go b/process/dataindexer/nilIndexer.go index b7205e96..318eeb39 100644 --- a/process/dataindexer/nilIndexer.go +++ b/process/dataindexer/nilIndexer.go @@ -8,11 +8,6 @@ import ( type NilIndexer struct { } -// NewNilIndexer will return a Nil indexer -func NewNilIndexer() *NilIndexer { - return new(NilIndexer) -} - // SaveBlock returns nil func (ni *NilIndexer) SaveBlock(_ *outport.OutportBlock) error { return nil diff --git a/process/dataindexer/workItems/workItemAccounts_test.go b/process/dataindexer/workItems/workItemAccounts_test.go index c4d85077..4f33d90a 100644 --- a/process/dataindexer/workItems/workItemAccounts_test.go +++ b/process/dataindexer/workItems/workItemAccounts_test.go @@ -5,7 +5,6 @@ import ( "testing" "github.com/multiversx/mx-chain-core-go/data/outport" - "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" "github.com/stretchr/testify/require" @@ -15,13 +14,12 @@ func TestItemAccounts_Save(t *testing.T) { called := false itemAccounts := workItems.NewItemAccounts( &mock.ElasticProcessorStub{ - SaveAccountsCalled: func(_ uint64, _ []*data.Account) error { + SaveAccountsCalled: func(_ *outport.Accounts) error { called = true return nil }, }, - 0, - make(map[string]*outport.AlteredAccount), 0, + &outport.Accounts{}, ) require.False(t, itemAccounts.IsInterfaceNil()) @@ -34,12 +32,11 @@ func TestItemAccounts_SaveAccountsShouldErr(t *testing.T) { localErr := errors.New("local err") itemAccounts := workItems.NewItemAccounts( &mock.ElasticProcessorStub{ - SaveAccountsCalled: func(_ uint64, _ []*data.Account) error { + SaveAccountsCalled: func(_ *outport.Accounts) error { return localErr }, }, - 0, - make(map[string]*outport.AlteredAccount), 0, + &outport.Accounts{}, ) require.False(t, itemAccounts.IsInterfaceNil()) diff --git a/process/dataindexer/workItems/workItemBlock.go b/process/dataindexer/workItems/workItemBlock.go index 0020fc04..eb34fb05 100644 --- a/process/dataindexer/workItems/workItemBlock.go +++ b/process/dataindexer/workItems/workItemBlock.go @@ -33,6 +33,10 @@ func (wib *itemBlock) Save() error { log.Warn("nil outportBlock block provided when trying to index block, will skip") return nil } + if check.IfNil(wib.outportBlockWithHeader.Header) { + log.Warn("nil header provided when trying to index block, will skip") + return nil + } defer func(startTime time.Time) { log.Debug("wib.SaveBlockData duration", "time", time.Since(startTime)) @@ -75,34 +79,3 @@ func (wib *itemBlock) Save() error { func (wib *itemBlock) IsInterfaceNil() bool { return wib == nil } - -// ComputeSizeOfTxs will compute size of transactions in bytes -//func ComputeSizeOfTxs(marshalizer marshal.Marshalizer, pool *outport.TransactionPool) int { -// sizeTxs := 0 -// sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Transactions) -// sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Scrs) -// sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Invalid) -// sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Rewards) -// sizeTxs += computeSizeOfMapTxs(marshalizer, pool.Receipts) -// -// return sizeTxs -//} -// -//func computeSizeOfMapTxs(marshalizer marshal.Marshalizer, mapTxs map[string]data.TransactionHandlerWithGasUsedAndFee) int { -// txsSize := 0 -// for _, tx := range mapTxs { -// txsSize += computeTxSize(marshalizer, tx.GetTxHandler()) -// } -// -// return txsSize -//} -// -//func computeTxSize(marshalizer marshal.Marshalizer, tx data.TransactionHandler) int { -// txBytes, err := marshalizer.Marshal(tx) -// if err != nil { -// log.Debug("itemBlock.computeTxSize", "error", err) -// return 0 -// } -// -// return len(txBytes) -//} diff --git a/process/dataindexer/workItems/workItemBlock_test.go b/process/dataindexer/workItems/workItemBlock_test.go index d17dd917..edc6c794 100644 --- a/process/dataindexer/workItems/workItemBlock_test.go +++ b/process/dataindexer/workItems/workItemBlock_test.go @@ -2,47 +2,21 @@ package workItems_test import ( "errors" - "fmt" - "math/big" - "reflect" "testing" - "time" "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" - "github.com/multiversx/mx-chain-core-go/data/transaction" - "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func generateTxs(numTxs int) map[string]data.TransactionHandlerWithGasUsedAndFee { - txs := make(map[string]data.TransactionHandlerWithGasUsedAndFee, numTxs) - for i := 0; i < numTxs; i++ { - tx := &transaction.Transaction{ - Nonce: uint64(i), - Value: big.NewInt(int64(i)), - RcvAddr: []byte("443e79a8d99ba093262c1db48c58ab3d59bcfeb313ca5cddf2a9d1d06f9894ec"), - SndAddr: []byte("443e79a8d99ba093262c1db48c58ab3d59bcfeb313ca5cddf2a9d1d06f9894ec"), - GasPrice: 10000000, - GasLimit: 1000, - Data: []byte("dasjdksakjdksajdjksajkdjkasjdksajkdasjdksakjdksajdjksajkdjkasjdksajkdasjdksakjdksajdjksajkdjkasjdksajk"), - Signature: []byte("randomSignatureasdasldkasdsahjgdlhjaskldsjkaldjklasjkdjskladjkl;sajkl"), - } - txs[fmt.Sprintf("%d", i)] = outport.NewTransactionHandlerWithGasAndFee(tx, 0, big.NewInt(0)) - } - - return txs -} - func TestItemBlock_SaveNilHeaderShouldRetNil(t *testing.T) { itemBlock := workItems.NewItemBlock( &mock.ElasticProcessorStub{}, - &mock.MarshalizerMock{}, - &outport.ArgsSaveBlockData{}, + &outport.OutportBlockWithHeader{}, ) require.False(t, itemBlock.IsInterfaceNil()) @@ -54,15 +28,15 @@ func TestItemBlock_SaveHeaderShouldErr(t *testing.T) { localErr := errors.New("local err") itemBlock := workItems.NewItemBlock( &mock.ElasticProcessorStub{ - SaveHeaderCalled: func(headerHash []byte, header data.HeaderHandler, signersIndexes []uint64, body *dataBlock.Body, notarizedHeadersHashes []string, gasConsumptionData outport.HeaderGasConsumption, txsSize int, _ *outport.Pool) error { + SaveHeaderCalled: func(_ *outport.OutportBlockWithHeader) error { return localErr }, }, - &mock.MarshalizerMock{}, - &outport.ArgsSaveBlockData{ - Header: &dataBlock.Header{}, - Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{{}}}, - TransactionsPool: &outport.Pool{}, + &outport.OutportBlockWithHeader{ + OutportBlock: &outport.OutportBlock{ + BlockData: &outport.BlockData{}, + }, + Header: &dataBlock.Header{}, }, ) require.False(t, itemBlock.IsInterfaceNil()) @@ -75,7 +49,7 @@ func TestItemBlock_SaveNoMiniblocksShoulCallSaveHeader(t *testing.T) { countCalled := 0 itemBlock := workItems.NewItemBlock( &mock.ElasticProcessorStub{ - SaveHeaderCalled: func(headerHash []byte, header data.HeaderHandler, signersIndexes []uint64, body *dataBlock.Body, notarizedHeadersHashes []string, gasConsumptionData outport.HeaderGasConsumption, txsSize int, _ *outport.Pool) error { + SaveHeaderCalled: func(_ *outport.OutportBlockWithHeader) error { countCalled++ return nil }, @@ -83,16 +57,18 @@ func TestItemBlock_SaveNoMiniblocksShoulCallSaveHeader(t *testing.T) { countCalled++ return nil }, - SaveTransactionsCalled: func(body *dataBlock.Body, header data.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount) error { + SaveTransactionsCalled: func(_ *outport.OutportBlockWithHeader) error { countCalled++ return nil }, }, - &mock.MarshalizerMock{}, - &outport.ArgsSaveBlockData{ - Body: &dataBlock.Body{}, - Header: &dataBlock.Header{}, - TransactionsPool: &outport.Pool{}, + &outport.OutportBlockWithHeader{ + OutportBlock: &outport.OutportBlock{ + BlockData: &outport.BlockData{ + Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{}}, + }, + }, + Header: &dataBlock.Header{}, }, ) require.False(t, itemBlock.IsInterfaceNil()) @@ -110,11 +86,13 @@ func TestItemBlock_SaveMiniblocksShouldErr(t *testing.T) { return localErr }, }, - &mock.MarshalizerMock{}, - &outport.ArgsSaveBlockData{ - Header: &dataBlock.Header{}, - Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{{}}}, - TransactionsPool: &outport.Pool{}, + &outport.OutportBlockWithHeader{ + OutportBlock: &outport.OutportBlock{ + BlockData: &outport.BlockData{ + Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{{}}}, + }, + }, + Header: &dataBlock.Header{}, }, ) require.False(t, itemBlock.IsInterfaceNil()) @@ -127,15 +105,17 @@ func TestItemBlock_SaveTransactionsShouldErr(t *testing.T) { localErr := errors.New("local err") itemBlock := workItems.NewItemBlock( &mock.ElasticProcessorStub{ - SaveTransactionsCalled: func(body *dataBlock.Body, header data.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount) error { + SaveTransactionsCalled: func(_ *outport.OutportBlockWithHeader) error { return localErr }, }, - &mock.MarshalizerMock{}, - &outport.ArgsSaveBlockData{ - Header: &dataBlock.Header{}, - Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{{}}}, - TransactionsPool: &outport.Pool{}, + &outport.OutportBlockWithHeader{ + OutportBlock: &outport.OutportBlock{ + BlockData: &outport.BlockData{ + Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{{}}}, + }, + }, + Header: &dataBlock.Header{}, }, ) require.False(t, itemBlock.IsInterfaceNil()) @@ -148,7 +128,7 @@ func TestItemBlock_SaveShouldWork(t *testing.T) { countCalled := 0 itemBlock := workItems.NewItemBlock( &mock.ElasticProcessorStub{ - SaveHeaderCalled: func(headerHash []byte, header data.HeaderHandler, signersIndexes []uint64, body *dataBlock.Body, notarizedHeadersHashes []string, gasConsumptionData outport.HeaderGasConsumption, txsSize int, _ *outport.Pool) error { + SaveHeaderCalled: func(_ *outport.OutportBlockWithHeader) error { countCalled++ return nil }, @@ -156,16 +136,18 @@ func TestItemBlock_SaveShouldWork(t *testing.T) { countCalled++ return nil }, - SaveTransactionsCalled: func(body *dataBlock.Body, header data.HeaderHandler, pool *outport.Pool, coreAlteredAccounts map[string]*outport.AlteredAccount) error { + SaveTransactionsCalled: func(_ *outport.OutportBlockWithHeader) error { countCalled++ return nil }, }, - &mock.MarshalizerMock{}, - &outport.ArgsSaveBlockData{ - Header: &dataBlock.Header{}, - Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{{}}}, - TransactionsPool: &outport.Pool{}, + &outport.OutportBlockWithHeader{ + OutportBlock: &outport.OutportBlock{ + BlockData: &outport.BlockData{ + Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{{}}}, + }, + }, + Header: &dataBlock.Header{}, }, ) require.False(t, itemBlock.IsInterfaceNil()) @@ -174,39 +156,3 @@ func TestItemBlock_SaveShouldWork(t *testing.T) { require.NoError(t, err) require.Equal(t, 3, countCalled) } - -func TestComputeSizeOfTxsDuration(t *testing.T) { - res := testing.Benchmark(benchmarkComputeSizeOfTxsDuration) - - fmt.Println("Time to calculate size of txs :", time.Duration(res.NsPerOp())) -} - -func benchmarkComputeSizeOfTxsDuration(b *testing.B) { - numTxs := 20000 - txs := generateTxs(numTxs) - gogoMarsh := &marshal.GogoProtoMarshalizer{} - - for i := 0; i < b.N; i++ { - workItems.ComputeSizeOfTxs(gogoMarsh, &outport.Pool{Txs: txs}) - } -} - -func TestComputeSizeOfTxs(t *testing.T) { - const kb = 1024 - numTxs := 20000 - - txs := generateTxs(numTxs) - gogoMarsh := &marshal.GogoProtoMarshalizer{} - lenTxs := workItems.ComputeSizeOfTxs(gogoMarsh, &outport.Pool{Txs: txs}) - - keys := reflect.ValueOf(txs).MapKeys() - oneTxBytes, _ := gogoMarsh.Marshal(txs[keys[0].String()].GetTxHandler()) - oneTxSize := len(oneTxBytes) - expectedSize := numTxs * oneTxSize - expectedSizeDeltaPlus := expectedSize + int(0.01*float64(expectedSize)) - expectedSizeDeltaMinus := expectedSize - int(0.01*float64(expectedSize)) - - require.Greater(t, lenTxs, expectedSizeDeltaMinus) - require.Less(t, lenTxs, expectedSizeDeltaPlus) - fmt.Printf("Size of %d transactions : %d Kbs \n", numTxs, lenTxs/kb) -} diff --git a/process/dataindexer/workItems/workItemRating_test.go b/process/dataindexer/workItems/workItemRating_test.go index 502e84ec..75f53111 100644 --- a/process/dataindexer/workItems/workItemRating_test.go +++ b/process/dataindexer/workItems/workItemRating_test.go @@ -4,27 +4,22 @@ import ( "errors" "testing" - "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" "github.com/stretchr/testify/require" ) func TestItemRating_Save(t *testing.T) { - id := "0_1" called := false itemRating := workItems.NewItemRating( &mock.ElasticProcessorStub{ - SaveValidatorsRatingCalled: func(index string, validatorsRatingInfo []*data.ValidatorRatingInfo) error { - require.Equal(t, id, index) + SaveValidatorsRatingCalled: func(_ *outport.ValidatorsRating) error { called = true return nil }, }, - id, - []*data.ValidatorRatingInfo{ - {PublicKey: "pub-key", Rating: 100}, - }, + &outport.ValidatorsRating{}, ) require.False(t, itemRating.IsInterfaceNil()) @@ -34,18 +29,14 @@ func TestItemRating_Save(t *testing.T) { } func TestItemRating_SaveShouldErr(t *testing.T) { - id := "0_1" localErr := errors.New("local err") itemRating := workItems.NewItemRating( &mock.ElasticProcessorStub{ - SaveValidatorsRatingCalled: func(index string, validatorsRatingInfo []*data.ValidatorRatingInfo) error { + SaveValidatorsRatingCalled: func(_ *outport.ValidatorsRating) error { return localErr }, }, - id, - []*data.ValidatorRatingInfo{ - {PublicKey: "pub-key", Rating: 100}, - }, + &outport.ValidatorsRating{}, ) require.False(t, itemRating.IsInterfaceNil()) diff --git a/process/dataindexer/workItems/workItemRemoveBlock.go b/process/dataindexer/workItems/workItemRemoveBlock.go index bc3cd8b6..0b1f6328 100644 --- a/process/dataindexer/workItems/workItemRemoveBlock.go +++ b/process/dataindexer/workItems/workItemRemoveBlock.go @@ -1,28 +1,26 @@ package workItems import ( - "github.com/multiversx/mx-chain-core-go/core" - "github.com/multiversx/mx-chain-core-go/core/unmarshal" - "github.com/multiversx/mx-chain-core-go/data/outport" - "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/data/block" ) type itemRemoveBlock struct { - indexer removeIndexer - marshaller marshal.Marshalizer - blockData *outport.BlockData + indexer removeIndexer + header data.HeaderHandler + body *block.Body } // NewItemRemoveBlock will create a new instance of itemRemoveBlock func NewItemRemoveBlock( indexer removeIndexer, - marshaller marshal.Marshalizer, - blockData *outport.BlockData, + header data.HeaderHandler, + body *block.Body, ) WorkItemHandler { return &itemRemoveBlock{ - indexer: indexer, - marshaller: marshaller, - blockData: blockData, + indexer: indexer, + header: header, + body: body, } } @@ -33,25 +31,20 @@ func (wirb *itemRemoveBlock) IsInterfaceNil() bool { // Save will remove a block and miniblocks from elasticsearch database func (wirb *itemRemoveBlock) Save() error { - header, err := unmarshal.GetHeaderFromBytes(wirb.marshaller, core.HeaderType(wirb.blockData.HeaderType), wirb.blockData.HeaderBytes) + err := wirb.indexer.RemoveHeader(wirb.header) if err != nil { return err } - err = wirb.indexer.RemoveHeader(header) + err = wirb.indexer.RemoveMiniblocks(wirb.header, wirb.body) if err != nil { return err } - err = wirb.indexer.RemoveMiniblocks(header, wirb.blockData.Body) + err = wirb.indexer.RemoveTransactions(wirb.header, wirb.body) if err != nil { return err } - err = wirb.indexer.RemoveTransactions(header, wirb.blockData.Body) - if err != nil { - return err - } - - return wirb.indexer.RemoveAccountsESDT(header.GetTimeStamp(), header.GetShardID()) + return wirb.indexer.RemoveAccountsESDT(wirb.header.GetTimeStamp(), wirb.header.GetShardID()) } diff --git a/process/dataindexer/workItems/workItemRemoveBlock_test.go b/process/dataindexer/workItems/workItemRemoveBlock_test.go index 2648fca0..8b58f9db 100644 --- a/process/dataindexer/workItems/workItemRemoveBlock_test.go +++ b/process/dataindexer/workItems/workItemRemoveBlock_test.go @@ -28,8 +28,8 @@ func TestItemRemoveBlock_Save(t *testing.T) { return nil }, }, - &dataBlock.Body{}, &dataBlock.Header{}, + &dataBlock.Body{}, ) require.False(t, itemRemove.IsInterfaceNil()) @@ -46,8 +46,8 @@ func TestItemRemoveBlock_SaveRemoveHeaderShouldErr(t *testing.T) { return localErr }, }, - &dataBlock.Body{}, &dataBlock.Header{}, + &dataBlock.Body{}, ) require.False(t, itemRemove.IsInterfaceNil()) @@ -63,8 +63,8 @@ func TestItemRemoveBlock_SaveRemoveMiniblocksShouldErr(t *testing.T) { return localErr }, }, - &dataBlock.Body{}, &dataBlock.Header{}, + &dataBlock.Body{}, ) require.False(t, itemRemove.IsInterfaceNil()) diff --git a/process/dataindexer/workItems/workItemRounds_test.go b/process/dataindexer/workItems/workItemRounds_test.go index 6b377690..f8c2ac7f 100644 --- a/process/dataindexer/workItems/workItemRounds_test.go +++ b/process/dataindexer/workItems/workItemRounds_test.go @@ -4,7 +4,7 @@ import ( "errors" "testing" - "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" "github.com/stretchr/testify/require" @@ -14,14 +14,12 @@ func TestItemRounds_Save(t *testing.T) { called := false itemRounds := workItems.NewItemRounds( &mock.ElasticProcessorStub{ - SaveRoundsInfoCalled: func(infos []*data.RoundInfo) error { + SaveRoundsInfoCalled: func(_ *outport.RoundsInfo) error { called = true return nil }, }, - []*data.RoundInfo{ - {}, - }, + &outport.RoundsInfo{}, ) require.False(t, itemRounds.IsInterfaceNil()) @@ -34,13 +32,11 @@ func TestItemRounds_SaveRoundsShouldErr(t *testing.T) { localErr := errors.New("local err") itemRounds := workItems.NewItemRounds( &mock.ElasticProcessorStub{ - SaveRoundsInfoCalled: func(infos []*data.RoundInfo) error { + SaveRoundsInfoCalled: func(_ *outport.RoundsInfo) error { return localErr }, }, - []*data.RoundInfo{ - {}, - }, + &outport.RoundsInfo{}, ) require.False(t, itemRounds.IsInterfaceNil()) diff --git a/process/dataindexer/workItems/workItemValidators_test.go b/process/dataindexer/workItems/workItemValidators_test.go index 10ed89a0..b188928a 100644 --- a/process/dataindexer/workItems/workItemValidators_test.go +++ b/process/dataindexer/workItems/workItemValidators_test.go @@ -4,6 +4,7 @@ import ( "errors" "testing" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" "github.com/stretchr/testify/require" @@ -11,18 +12,14 @@ import ( func TestItemValidators_Save(t *testing.T) { called := false - validators := map[uint32][][]byte{ - 0: {[]byte("val1"), []byte("val2")}, - } itemValidators := workItems.NewItemValidators( &mock.ElasticProcessorStub{ - SaveShardValidatorsPubKeysCalled: func(shardID, epoch uint32, shardValidatorsPubKeys [][]byte) error { + SaveShardValidatorsPubKeysCalled: func(_ *outport.ValidatorsPubKeys) error { called = true return nil }, }, - 1, - validators, + &outport.ValidatorsPubKeys{}, ) require.False(t, itemValidators.IsInterfaceNil()) @@ -33,17 +30,13 @@ func TestItemValidators_Save(t *testing.T) { func TestItemValidators_SaveValidatorsShouldErr(t *testing.T) { localErr := errors.New("local err") - validators := map[uint32][][]byte{ - 0: {[]byte("val1"), []byte("val2")}, - } itemValidators := workItems.NewItemValidators( &mock.ElasticProcessorStub{ - SaveShardValidatorsPubKeysCalled: func(shardID, epoch uint32, shardValidatorsPubKeys [][]byte) error { + SaveShardValidatorsPubKeysCalled: func(_ *outport.ValidatorsPubKeys) error { return localErr }, }, - 1, - validators, + &outport.ValidatorsPubKeys{}, ) require.False(t, itemValidators.IsInterfaceNil()) diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index 8b1449c1..91ff15b2 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -33,6 +33,7 @@ type ArgsIndexerFactory struct { Password string TemplatesPath string EnabledIndexes []string + HeaderMarshaller marshal.Marshalizer Marshalizer marshal.Marshalizer Hasher hashing.Hasher AddressPubkeyConverter core.PubkeyConverter @@ -59,7 +60,7 @@ func NewIndexer(args ArgsIndexerFactory) (dataindexer.Indexer, error) { dispatcher.StartIndexData() arguments := dataindexer.ArgDataIndexer{ - Marshalizer: args.Marshalizer, + HeaderMarshaller: args.HeaderMarshaller, ElasticProcessor: elasticProcessor, DataDispatcher: dispatcher, } @@ -121,6 +122,9 @@ func checkDataIndexerParams(arguments ArgsIndexerFactory) error { if check.IfNil(arguments.Hasher) { return dataindexer.ErrNilHasher } + if check.IfNil(arguments.HeaderMarshaller) { + return fmt.Errorf("%w header marshaller", dataindexer.ErrNilMarshalizer) + } return nil } From a3706492a38dc84a3d5f7822cbc78c1815f071e8 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 15 Mar 2023 13:34:34 +0200 Subject: [PATCH 018/189] fix unit tests 2 --- process/elasticproc/elasticProcessor_test.go | 105 +++++++++++---- .../smartContractResultsProcessor.go | 2 +- .../transactions/transactionDBBuilder.go | 4 +- .../transactions/transactionDBBuilder_test.go | 25 +++- .../transactions/transactionsGrouper.go | 6 +- .../transactions/transactionsGrouper_test.go | 71 +++++----- .../transactionsProcessor_test.go | 126 ++++++++++-------- 7 files changed, 216 insertions(+), 123 deletions(-) diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index 427922c2..33e78c66 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -360,7 +360,16 @@ func TestElasticseachDatabaseSaveHeader_RequestError(t *testing.T) { arguments.BlockProc, _ = block.NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) - err := elasticDatabase.SaveHeader([]byte("hh"), header, signerIndexes, &dataBlock.Body{}, nil, outport.HeaderGasConsumption{}, 1, &outport.Pool{}) + err := elasticDatabase.SaveHeader(&outport.OutportBlockWithHeader{ + Header: header, + OutportBlock: &outport.OutportBlock{ + BlockData: &outport.BlockData{ + Body: &dataBlock.Body{}, + }, + SignersIndexes: signerIndexes, + HeaderGasConsumption: &outport.HeaderGasConsumption{}, + }, + }) require.Equal(t, localErr, err) } @@ -401,7 +410,16 @@ func TestElasticseachDatabaseSaveHeader_CheckRequestBody(t *testing.T) { arguments.BlockProc, _ = block.NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) - err := elasticDatabase.SaveHeader([]byte("hh"), header, signerIndexes, blockBody, nil, outport.HeaderGasConsumption{}, 1, &outport.Pool{}) + err := elasticDatabase.SaveHeader(&outport.OutportBlockWithHeader{ + Header: header, + OutportBlock: &outport.OutportBlock{ + BlockData: &outport.BlockData{ + Body: blockBody, + }, + SignersIndexes: signerIndexes, + HeaderGasConsumption: &outport.HeaderGasConsumption{}, + }, + }) require.Nil(t, err) } @@ -416,7 +434,6 @@ func TestElasticseachSaveTransactions(t *testing.T) { body := newTestBlockBody() header := &dataBlock.Header{Nonce: 1, TxCount: 2} - txPool := newTestTxPool() bc, _ := converters.NewBalanceConverter(18) args := &transactions.ArgsTransactionProcessor{ @@ -429,17 +446,26 @@ func TestElasticseachSaveTransactions(t *testing.T) { arguments.TransactionsProc = txDbProc elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) - pool := &outport.Pool{Txs: txPool} - err := elasticDatabase.SaveTransactions(body, header, pool, nil, false, 3) + err := elasticDatabase.SaveTransactions(&outport.OutportBlockWithHeader{ + Header: header, + OutportBlock: &outport.OutportBlock{ + BlockData: &outport.BlockData{ + Body: body, + }, + HeaderGasConsumption: &outport.HeaderGasConsumption{}, + TransactionPool: &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + hex.EncodeToString([]byte("tx1")): {Transaction: &transaction.Transaction{}, FeeInfo: &outport.FeeInfo{}}, + }, + }, + }, + }) require.Equal(t, localErr, err) } func TestElasticProcessor_SaveValidatorsRating(t *testing.T) { - docID := "0_1" localErr := errors.New("localErr") - blsKey := "bls" - arguments := createMockElasticProcessorArgs() arguments.DBClient = &mock.DatabaseWriterStub{ DoBulkRequestCalled: func(buff *bytes.Buffer, index string) error { @@ -450,15 +476,11 @@ func TestElasticProcessor_SaveValidatorsRating(t *testing.T) { arguments.ValidatorsProc, _ = validators.NewValidatorsProcessor(mock.NewPubkeyConverterMock(32), 0) elasticProc, _ := NewElasticProcessor(arguments) - err := elasticProc.SaveValidatorsRating( - docID, - []*data.ValidatorRatingInfo{ - { - PublicKey: blsKey, - Rating: 100, - }, - }, - ) + err := elasticProc.SaveValidatorsRating(&outport.ValidatorsRating{ + ShardID: 0, + Epoch: 1, + ValidatorsRatingInfo: []*outport.ValidatorRatingInfo{{}}, + }) require.Equal(t, localErr, err) } @@ -493,14 +515,19 @@ func TestElasticsearch_saveShardValidatorsPubKeys_RequestError(t *testing.T) { localErr := errors.New("localErr") arguments := createMockElasticProcessorArgs() dbWriter := &mock.DatabaseWriterStub{ - DoRequestCalled: func(req *esapi.IndexRequest) error { + DoBulkRequestCalled: func(buff *bytes.Buffer, index string) error { return localErr }, } arguments.ValidatorsProc, _ = validators.NewValidatorsProcessor(mock.NewPubkeyConverterMock(32), 0) elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) - err := elasticDatabase.SaveShardValidatorsPubKeys(shardID, epoch, valPubKeys) + err := elasticDatabase.SaveShardValidatorsPubKeys(&outport.ValidatorsPubKeys{ + Epoch: epoch, + ShardValidatorsPubKeys: map[uint32]*outport.PubKeys{ + shardID: {Keys: valPubKeys}, + }, + }) require.Equal(t, localErr, err) } @@ -517,29 +544,34 @@ func TestElasticsearch_saveShardValidatorsPubKeys(t *testing.T) { } elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) - err := elasticDatabase.SaveShardValidatorsPubKeys(shardID, epoch, valPubKeys) + err := elasticDatabase.SaveShardValidatorsPubKeys(&outport.ValidatorsPubKeys{ + Epoch: epoch, + ShardValidatorsPubKeys: map[uint32]*outport.PubKeys{ + shardID: {Keys: valPubKeys}, + }, + }) require.Nil(t, err) } func TestElasticsearch_saveRoundInfo(t *testing.T) { - roundInfo := &data.RoundInfo{ - Index: 1, ShardId: 0, BlockWasProposed: true, + roundInfo := &outport.RoundInfo{ + Round: 1, ShardId: 0, BlockWasProposed: true, } arguments := createMockElasticProcessorArgs() dbWriter := &mock.DatabaseWriterStub{ DoRequestCalled: func(req *esapi.IndexRequest) error { - require.Equal(t, strconv.FormatUint(uint64(roundInfo.ShardId), 10)+"_"+strconv.FormatUint(roundInfo.Index, 10), req.DocumentID) + require.Equal(t, strconv.FormatUint(uint64(roundInfo.ShardId), 10)+"_"+strconv.FormatUint(roundInfo.Round, 10), req.DocumentID) return nil }, } elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) - err := elasticDatabase.SaveRoundsInfo([]*data.RoundInfo{roundInfo}) + err := elasticDatabase.SaveRoundsInfo(&outport.RoundsInfo{RoundsInfo: []*outport.RoundInfo{roundInfo}}) require.Nil(t, err) } func TestElasticsearch_saveRoundInfoRequestError(t *testing.T) { - roundInfo := &data.RoundInfo{} + roundInfo := &outport.RoundInfo{} localError := errors.New("local err") arguments := createMockElasticProcessorArgs() dbWriter := &mock.DatabaseWriterStub{ @@ -549,7 +581,7 @@ func TestElasticsearch_saveRoundInfoRequestError(t *testing.T) { } elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) - err := elasticDatabase.SaveRoundsInfo([]*data.RoundInfo{roundInfo}) + err := elasticDatabase.SaveRoundsInfo(&outport.RoundsInfo{RoundsInfo: []*outport.RoundInfo{roundInfo}}) require.Equal(t, localError, err) } @@ -625,7 +657,15 @@ func TestElasticProcessor_IndexEpochInfoData(t *testing.T) { body := &dataBlock.Body{} metaHeader := &dataBlock.MetaBlock{} - err = elasticSearchProc.SaveHeader([]byte("hh"), metaHeader, nil, body, nil, outport.HeaderGasConsumption{}, 0, &outport.Pool{}) + err = elasticSearchProc.SaveHeader(&outport.OutportBlockWithHeader{ + Header: metaHeader, + OutportBlock: &outport.OutportBlock{ + BlockData: &outport.BlockData{ + Body: body, + }, + HeaderGasConsumption: &outport.HeaderGasConsumption{}, + }, + }) require.Nil(t, err) require.True(t, called) } @@ -634,7 +674,7 @@ func TestElasticProcessor_SaveTransactionNoDataShouldNotDoRequest(t *testing.T) called := false arguments := createMockElasticProcessorArgs() arguments.TransactionsProc = &mock.DBTransactionProcessorStub{ - PrepareTransactionsForDatabaseCalled: func(body *dataBlock.Body, header coreData.HeaderHandler, pool *outport.Pool) *data.PreparedResults { + PrepareTransactionsForDatabaseCalled: func(body *dataBlock.Body, header coreData.HeaderHandler, pool *outport.TransactionPool) *data.PreparedResults { return &data.PreparedResults{ Transactions: nil, ScResults: nil, @@ -655,7 +695,14 @@ func TestElasticProcessor_SaveTransactionNoDataShouldNotDoRequest(t *testing.T) elasticSearchProc := newElasticsearchProcessor(dbWriter, arguments) elasticSearchProc.enabledIndexes[dataindexer.ScResultsIndex] = struct{}{} - err := elasticSearchProc.SaveTransactions(&dataBlock.Body{}, &dataBlock.Header{}, &outport.Pool{}, nil, false, 3) + err := elasticSearchProc.SaveTransactions(&outport.OutportBlockWithHeader{ + Header: &dataBlock.Header{}, + OutportBlock: &outport.OutportBlock{ + BlockData: &outport.BlockData{ + Body: &dataBlock.Body{}, + }, + }, + }) require.Nil(t, err) require.False(t, called) } diff --git a/process/elasticproc/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go index eb0b3ffb..d6986a87 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -95,7 +95,7 @@ func (proc *smartContractResultsProcessor) processSCRsFromMiniblock( indexerSCR := proc.prepareSmartContractResult(scrHash, mbHash, scrInfo, header, mb.SenderShardID, mb.ReceiverShardID, numOfShards) indexerSCRs = append(indexerSCRs, indexerSCR) - delete(scrs, string(scrHash)) + delete(scrs, hex.EncodeToString(scrHash)) } return indexerSCRs diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index 2b341311..4ef1a835 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -147,12 +147,12 @@ func (dtb *dbTransactionBuilder) prepareRewardTransaction( } func (dtb *dbTransactionBuilder) prepareReceipt( - recHash string, + recHashHex string, rec *receipt.Receipt, header coreData.HeaderHandler, ) *data.Receipt { return &data.Receipt{ - Hash: hex.EncodeToString([]byte(recHash)), + Hash: recHashHex, Value: rec.Value.String(), Sender: dtb.addressPubkeyConverter.Encode(rec.SndAddr), Data: string(rec.Data), diff --git a/process/elasticproc/transactions/transactionDBBuilder_test.go b/process/elasticproc/transactions/transactionDBBuilder_test.go index 2411ce84..f1ae0896 100644 --- a/process/elasticproc/transactions/transactionDBBuilder_test.go +++ b/process/elasticproc/transactions/transactionDBBuilder_test.go @@ -9,6 +9,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/rewardTx" "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-es-indexer-go/data" @@ -53,6 +54,16 @@ func TestGetMoveBalanceTransaction(t *testing.T) { SndUserName: []byte("snd"), } + txInfo := &outport.TxInfo{ + Transaction: tx, + FeeInfo: &outport.FeeInfo{ + GasUsed: 500, + Fee: big.NewInt(100), + InitialPaidFee: big.NewInt(100), + }, + ExecutionOrder: 0, + } + expectedTx := &data.Transaction{ Hash: hex.EncodeToString(txHash), MBHash: hex.EncodeToString(mbHash), @@ -82,7 +93,7 @@ func TestGetMoveBalanceTransaction(t *testing.T) { Receivers: []string{}, } - dbTx := cp.prepareTransaction(tx, txHash, mbHash, mb, header, status, big.NewInt(100), 500, big.NewInt(100), 3) + dbTx := cp.prepareTransaction(txInfo, txHash, mbHash, mb, header, status, 3) require.Equal(t, expectedTx, dbTx) } @@ -172,6 +183,16 @@ func TestGetMoveBalanceTransactionInvalid(t *testing.T) { ESDTValuesNum: []float64{}, } - dbTx := cp.prepareTransaction(tx, txHash, mbHash, mb, header, status, big.NewInt(100), 500, big.NewInt(100), 3) + txInfo := &outport.TxInfo{ + Transaction: tx, + FeeInfo: &outport.FeeInfo{ + GasUsed: 500, + Fee: big.NewInt(100), + InitialPaidFee: big.NewInt(100), + }, + ExecutionOrder: 0, + } + + dbTx := cp.prepareTransaction(txInfo, txHash, mbHash, mb, header, status, 3) require.Equal(t, expectedTx, dbTx) } diff --git a/process/elasticproc/transactions/transactionsGrouper.go b/process/elasticproc/transactions/transactionsGrouper.go index 01734259..c6073914 100644 --- a/process/elasticproc/transactions/transactionsGrouper.go +++ b/process/elasticproc/transactions/transactionsGrouper.go @@ -190,7 +190,7 @@ func (tg *txsGrouper) prepareInvalidTxForDB( header coreData.HeaderHandler, numOfShards uint32, ) (*data.Transaction, bool) { - txInfo, okGet := txs[string(txHash)] + txInfo, okGet := txs[hex.EncodeToString(txHash)] if !okGet { return nil, false } @@ -210,8 +210,8 @@ func (tg *txsGrouper) shouldIndex(destinationShardID uint32, isImportDB bool, se func (tg *txsGrouper) groupReceipts(header coreData.HeaderHandler, txsPool map[string]*receipt.Receipt) []*data.Receipt { dbReceipts := make([]*data.Receipt, 0) - for hash, rec := range txsPool { - dbReceipts = append(dbReceipts, tg.txBuilder.prepareReceipt(hash, rec, header)) + for hashHex, rec := range txsPool { + dbReceipts = append(dbReceipts, tg.txBuilder.prepareReceipt(hashHex, rec, header)) } return dbReceipts diff --git a/process/elasticproc/transactions/transactionsGrouper_test.go b/process/elasticproc/transactions/transactionsGrouper_test.go index be81979a..12c6b75e 100644 --- a/process/elasticproc/transactions/transactionsGrouper_test.go +++ b/process/elasticproc/transactions/transactionsGrouper_test.go @@ -1,10 +1,9 @@ package transactions import ( - "math/big" + "encoding/hex" "testing" - coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/receipt" @@ -30,15 +29,21 @@ func TestGroupNormalTxs(t *testing.T) { Type: block.TxBlock, } header := &block.Header{} - txs := map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash1): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - SndAddr: []byte("sender1"), - RcvAddr: []byte("receiver1"), - }, 0, big.NewInt(0)), - string(txHash2): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - SndAddr: []byte("sender2"), - RcvAddr: []byte("receiver2"), - }, 0, big.NewInt(0)), + txs := map[string]*outport.TxInfo{ + hex.EncodeToString(txHash1): { + Transaction: &transaction.Transaction{ + SndAddr: []byte("sender1"), + RcvAddr: []byte("receiver1"), + }, + FeeInfo: &outport.FeeInfo{}, + }, + hex.EncodeToString(txHash2): { + Transaction: &transaction.Transaction{ + SndAddr: []byte("sender2"), + RcvAddr: []byte("receiver2"), + }, + FeeInfo: &outport.FeeInfo{}, + }, } normalTxs, _ := grouper.groupNormalTxs(0, mb, header, txs, false, 3) @@ -60,13 +65,13 @@ func TestGroupRewardsTxs(t *testing.T) { Type: block.RewardsBlock, } header := &block.Header{} - txs := map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash1): outport.NewTransactionHandlerWithGasAndFee(&rewardTx.RewardTx{ + txs := map[string]*outport.RewardInfo{ + hex.EncodeToString(txHash1): {Reward: &rewardTx.RewardTx{ RcvAddr: []byte("receiver1"), - }, 0, big.NewInt(0)), - string(txHash2): outport.NewTransactionHandlerWithGasAndFee(&rewardTx.RewardTx{ + }}, + hex.EncodeToString(txHash2): {Reward: &rewardTx.RewardTx{ RcvAddr: []byte("receiver2"), - }, 0, big.NewInt(0)), + }}, } normalTxs, _ := grouper.groupRewardsTxs(0, mb, header, txs, false) @@ -88,15 +93,17 @@ func TestGroupInvalidTxs(t *testing.T) { Type: block.InvalidBlock, } header := &block.Header{} - txs := map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash1): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - SndAddr: []byte("sender1"), - RcvAddr: []byte("receiver1"), - }, 0, big.NewInt(0)), - string(txHash2): outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - SndAddr: []byte("sender2"), - RcvAddr: []byte("receiver2"), - }, 0, big.NewInt(0)), + txs := map[string]*outport.TxInfo{ + hex.EncodeToString(txHash1): { + Transaction: &transaction.Transaction{ + SndAddr: []byte("sender1"), + RcvAddr: []byte("receiver1"), + }, FeeInfo: &outport.FeeInfo{}}, + hex.EncodeToString(txHash2): { + Transaction: &transaction.Transaction{ + SndAddr: []byte("sender2"), + RcvAddr: []byte("receiver2"), + }, FeeInfo: &outport.FeeInfo{}}, } normalTxs, _ := grouper.groupInvalidTxs(0, mb, header, txs, 3) @@ -114,15 +121,15 @@ func TestGroupReceipts(t *testing.T) { txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") header := &block.Header{} - txs := map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash1): outport.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ + txs := map[string]*receipt.Receipt{ + hex.EncodeToString(txHash1): { SndAddr: []byte("sender1"), - }, 0, big.NewInt(0)), - string(txHash2): outport.NewTransactionHandlerWithGasAndFee(&receipt.Receipt{ + }, + hex.EncodeToString(txHash2): { SndAddr: []byte("sender2"), - }, 0, big.NewInt(0)), + }, } - normalTxs := grouper.groupReceipts(header, txs) - require.Len(t, normalTxs, 2) + receipts := grouper.groupReceipts(header, txs) + require.Len(t, receipts, 2) } diff --git a/process/elasticproc/transactions/transactionsProcessor_test.go b/process/elasticproc/transactions/transactionsProcessor_test.go index d4caac9c..2273e089 100644 --- a/process/elasticproc/transactions/transactionsProcessor_test.go +++ b/process/elasticproc/transactions/transactionsProcessor_test.go @@ -1,6 +1,7 @@ package transactions import ( + "encoding/hex" "math/big" "testing" @@ -124,6 +125,9 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { PrevTxHash: txHash1, GasLimit: 1, }, + FeeInfo: &outport.FeeInfo{ + Fee: big.NewInt(0), + }, } scHash2 := []byte("scHash2") scResult2 := &outport.SCRInfo{ @@ -132,6 +136,9 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { PrevTxHash: txHash1, GasLimit: 1, }, + FeeInfo: &outport.FeeInfo{ + Fee: big.NewInt(0), + }, } scHash3 := []byte("scHash3") scResult3 := &outport.SCRInfo{ @@ -139,6 +146,9 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { OriginalTxHash: txHash3, Data: []byte("@" + "6F6B"), }, + FeeInfo: &outport.FeeInfo{ + Fee: big.NewInt(0), + }, } body := &block.Body{ @@ -177,26 +187,26 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { pool := &outport.TransactionPool{ Transactions: map[string]*outport.TxInfo{ - string(txHash1): tx1, - string(txHash2): tx2, - string(txHash3): tx3, - string(txHash4): tx4, + hex.EncodeToString(txHash1): tx1, + hex.EncodeToString(txHash2): tx2, + hex.EncodeToString(txHash3): tx3, + hex.EncodeToString(txHash4): tx4, }, SmartContractResults: map[string]*outport.SCRInfo{ - string(scHash1): scResult1, - string(scHash2): scResult2, - string(scHash3): scResult3, + hex.EncodeToString(scHash1): scResult1, + hex.EncodeToString(scHash2): scResult2, + hex.EncodeToString(scHash3): scResult3, }, Rewards: map[string]*outport.RewardInfo{ - string(rTx1Hash): rTx1, - string(rTx2Hash): rTx2, + hex.EncodeToString(rTx1Hash): rTx1, + hex.EncodeToString(rTx2Hash): rTx2, }, InvalidTxs: map[string]*outport.TxInfo{ - string(txHash5): tx5, + hex.EncodeToString(txHash5): tx5, }, Receipts: map[string]*receipt.Receipt{ - string(recHash1): rec1, - string(recHash2): rec2, + hex.EncodeToString(recHash1): rec1, + hex.EncodeToString(recHash2): rec2, }, } @@ -211,24 +221,27 @@ func TestRelayedTransactions(t *testing.T) { t.Parallel() txHash1 := []byte("txHash1") - tx1 := &outport.TxInfo{Transaction: &transaction.Transaction{ - GasLimit: 100, - GasPrice: 100, - Data: []byte("relayedTx@blablabllablalba"), - }} + tx1 := &outport.TxInfo{ + Transaction: &transaction.Transaction{ + GasLimit: 100, + GasPrice: 100, + Data: []byte("relayedTx@blablabllablalba"), + }, FeeInfo: &outport.FeeInfo{}} scHash1 := []byte("scHash1") - scResult1 := &outport.SCRInfo{SmartContractResult: &smartContractResult.SmartContractResult{ - OriginalTxHash: txHash1, - PrevTxHash: txHash1, - GasLimit: 1, - }} + scResult1 := &outport.SCRInfo{ + SmartContractResult: &smartContractResult.SmartContractResult{ + OriginalTxHash: txHash1, + PrevTxHash: txHash1, + GasLimit: 1, + }, FeeInfo: &outport.FeeInfo{}} scHash2 := []byte("scHash2") - scResult2 := &outport.SCRInfo{SmartContractResult: &smartContractResult.SmartContractResult{ - OriginalTxHash: txHash1, - PrevTxHash: txHash1, - GasLimit: 1, - }} + scResult2 := &outport.SCRInfo{ + SmartContractResult: &smartContractResult.SmartContractResult{ + OriginalTxHash: txHash1, + PrevTxHash: txHash1, + GasLimit: 1, + }, FeeInfo: &outport.FeeInfo{}} body := &block.Body{ MiniBlocks: []*block.MiniBlock{ @@ -247,11 +260,11 @@ func TestRelayedTransactions(t *testing.T) { pool := &outport.TransactionPool{ Transactions: map[string]*outport.TxInfo{ - string(txHash1): tx1, + hex.EncodeToString(txHash1): tx1, }, SmartContractResults: map[string]*outport.SCRInfo{ - string(scHash1): scResult1, - string(scHash2): scResult2, + hex.EncodeToString(scHash1): scResult1, + hex.EncodeToString(scHash2): scResult2, }, } @@ -361,10 +374,10 @@ func TestCheckGasUsedInvalidTransaction(t *testing.T) { pool := &outport.TransactionPool{ InvalidTxs: map[string]*outport.TxInfo{ - string(txHash1): tx1, + hex.EncodeToString(txHash1): tx1, }, Receipts: map[string]*receipt.Receipt{ - string(recHash1): rec1, + hex.EncodeToString(recHash1): rec1, }, } @@ -469,9 +482,12 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseInvalidTxWithSCR(t * FeeInfo: &outport.FeeInfo{GasUsed: 100}, } scResHash1 := []byte("scResHash1") - scRes1 := &outport.SCRInfo{SmartContractResult: &smartContractResult.SmartContractResult{ - OriginalTxHash: txHash1, - }} + scRes1 := &outport.SCRInfo{ + SmartContractResult: &smartContractResult.SmartContractResult{ + OriginalTxHash: txHash1, + }, + FeeInfo: &outport.FeeInfo{}, + } body := &block.Body{ MiniBlocks: []*block.MiniBlock{ @@ -490,10 +506,10 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseInvalidTxWithSCR(t * pool := &outport.TransactionPool{ InvalidTxs: map[string]*outport.TxInfo{ - string(txHash1): tx1, + hex.EncodeToString(txHash1): tx1, }, SmartContractResults: map[string]*outport.SCRInfo{ - string(scResHash1): scRes1, + hex.EncodeToString(scResHash1): scRes1, }, } @@ -523,12 +539,14 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseESDTNFTTransfer(t *t FeeInfo: &outport.FeeInfo{GasUsed: 100}, } scResHash1 := []byte("scResHash1") - scRes1 := &outport.SCRInfo{SmartContractResult: &smartContractResult.SmartContractResult{ - Nonce: 1, - Data: []byte("@" + okHexEncoded), - OriginalTxHash: txHash1, - PrevTxHash: txHash1, - }} + scRes1 := &outport.SCRInfo{ + SmartContractResult: &smartContractResult.SmartContractResult{ + Nonce: 1, + Data: []byte("@" + okHexEncoded), + OriginalTxHash: txHash1, + PrevTxHash: txHash1, + }, + FeeInfo: &outport.FeeInfo{}} body := &block.Body{ MiniBlocks: []*block.MiniBlock{ @@ -547,7 +565,7 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseESDTNFTTransfer(t *t pool := &outport.TransactionPool{ Transactions: map[string]*outport.TxInfo{ - string(txHash1): tx1, + hex.EncodeToString(txHash1): tx1, }, SmartContractResults: map[string]*outport.SCRInfo{ string(scResHash1): scRes1, @@ -598,25 +616,25 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { } pool := &outport.TransactionPool{ Transactions: map[string]*outport.TxInfo{ - "t1": {Transaction: &transaction.Transaction{ + hex.EncodeToString([]byte("t1")): {Transaction: &transaction.Transaction{ SndAddr: decodeBech32("erd1dglncxk6sl9a3xumj78n6z2xux4ghp5c92cstv5zsn56tjgtdwpsk46qrs"), RcvAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), Data: []byte("issue@4141414141@41414141414141@0186a0@01@63616e467265657a65@74727565@63616e57697065@74727565@63616e5061757365@74727565@63616e4d696e74@74727565@63616e4275726e@74727565@63616e4368616e67654f776e6572@74727565@63616e55706772616465@74727565"), - }}, + }, FeeInfo: &outport.FeeInfo{}}, }, SmartContractResults: map[string]*outport.SCRInfo{ - "scr1": {SmartContractResult: &smartContractResult.SmartContractResult{ + hex.EncodeToString([]byte("scr1")): {SmartContractResult: &smartContractResult.SmartContractResult{ OriginalTxHash: []byte("t1"), Data: []byte("ESDTTransfer@414141414141412d323436626461@0186a0"), SndAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), RcvAddr: decodeBech32("erd1dglncxk6sl9a3xumj78n6z2xux4ghp5c92cstv5zsn56tjgtdwpsk46qrs"), - }}, - "scr2": {SmartContractResult: &smartContractResult.SmartContractResult{ + }, FeeInfo: &outport.FeeInfo{}}, + hex.EncodeToString([]byte("scr2")): {SmartContractResult: &smartContractResult.SmartContractResult{ OriginalTxHash: []byte("t1"), Data: []byte("@6f6b"), SndAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), RcvAddr: decodeBech32("erd1dglncxk6sl9a3xumj78n6z2xux4ghp5c92cstv5zsn56tjgtdwpsk46qrs"), - }}, + }, FeeInfo: &outport.FeeInfo{}}, }, } @@ -627,19 +645,19 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { // transaction fail pool = &outport.TransactionPool{ Transactions: map[string]*outport.TxInfo{ - "t1": {Transaction: &transaction.Transaction{ + hex.EncodeToString([]byte("t1")): {Transaction: &transaction.Transaction{ SndAddr: decodeBech32("erd1dglncxk6sl9a3xumj78n6z2xux4ghp5c92cstv5zsn56tjgtdwpsk46qrs"), RcvAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), Data: []byte("issue@4141414141@41414141414141@0186a0@01@63616e467265657a65@74727565@63616e57697065@74727565@63616e5061757365@74727565@63616e4d696e74@74727565@63616e4275726e@74727565@63616e4368616e67654f776e6572@74727565@63616e55706772616465@74727565"), - }}, + }, FeeInfo: &outport.FeeInfo{}}, }, SmartContractResults: map[string]*outport.SCRInfo{ - "scr1": {SmartContractResult: &smartContractResult.SmartContractResult{ + hex.EncodeToString([]byte("scr1")): {SmartContractResult: &smartContractResult.SmartContractResult{ OriginalTxHash: []byte("t1"), Data: []byte("75736572206572726f72"), SndAddr: decodeBech32("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u"), RcvAddr: decodeBech32("erd1dglncxk6sl9a3xumj78n6z2xux4ghp5c92cstv5zsn56tjgtdwpsk46qrs"), - }}, + }, FeeInfo: &outport.FeeInfo{}}, }, } From e891b0e4b8e2421f350aab4f98512ba0c1d00a61 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 15 Mar 2023 15:23:00 +0200 Subject: [PATCH 019/189] fix unit tests 3 --- .../elasticproc/block/blockProcessor_test.go | 35 +++++++++-------- process/elasticproc/elasticProcessor_test.go | 39 +------------------ .../logsevents/logsAndEventsProcessor_test.go | 17 ++++---- .../miniblocks/miniblocksProcessor_test.go | 12 +++--- process/factory/indexerFactory.go | 2 +- process/factory/indexerFactory_test.go | 10 +++++ 6 files changed, 46 insertions(+), 69 deletions(-) diff --git a/process/elasticproc/block/blockProcessor_test.go b/process/elasticproc/block/blockProcessor_test.go index 3a90f280..f4f0daf6 100644 --- a/process/elasticproc/block/blockProcessor_test.go +++ b/process/elasticproc/block/blockProcessor_test.go @@ -88,9 +88,9 @@ func TestBlockProcessor_PrepareBlockForDBShouldWork(t *testing.T) { Validators: []uint64{0x0, 0x1, 0x2}, EpochStartBlock: false, SearchOrder: 0x3fc, - MiniBlocksHashes: []string{"c57392e53257b4861f5e406349a8deb89c6dbc2127564ee891a41a188edbf01a", "28fda294dc987e5099d75e53cd6f87a9a42b96d55242a634385b5d41175c0c21"}, + MiniBlocksHashes: []string{"0796d34e8d443fd31bf4d9ec4051421b4d5d0e8c1db9ff942d6f4dc3a9ca2803", "4cc379ab1f0aef6602e85a0a7ffabb5bc9a2ba646dc0fd720028e06527bf873f"}, NotarizedBlocksHashes: []string(nil), - Size: 104, + Size: 230, AccumulatedFees: "0", DeveloperFees: "0", } @@ -115,6 +115,9 @@ func TestBlockProcessor_PrepareBlockForDBNilBody(t *testing.T) { outportBlockWithHeader := &outport.OutportBlockWithHeader{ Header: &dataBlock.MetaBlock{}, + OutportBlock: &outport.OutportBlock{ + BlockData: &outport.BlockData{}, + }, } dbBlock, err := bp.PrepareBlockForDB(outportBlockWithHeader) require.Equal(t, indexer.ErrNilBlockBody, err) @@ -190,7 +193,7 @@ func TestBlockProcessor_ComputeHeaderHash(t *testing.T) { header := &dataBlock.Header{} hashBytes, err := bp.ComputeHeaderHash(header) require.Nil(t, err) - require.Equal(t, "c7c81a1b22b67680f35837b474387ddfe10f67e104034c80f94ab9e5a0a089fb", hex.EncodeToString(hashBytes)) + require.Equal(t, "96f7d09988eafbc99b45dfce0eaf9df1d02def2ae678d88bd154ebffa3247b2a", hex.EncodeToString(hashBytes)) } func TestBlockProcessor_PrepareBlockForDBEpochStartMeta(t *testing.T) { @@ -264,12 +267,12 @@ func TestBlockProcessor_PrepareBlockForDBEpochStartMeta(t *testing.T) { Round: 0, Epoch: 0, Hash: "68617368", - MiniBlocksHashes: []string{"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"}, + MiniBlocksHashes: []string{"8748c4677b01f7db984004fa8465afbf55feaab4b573174c8c0afa282941b9e4", "8748c4677b01f7db984004fa8465afbf55feaab4b573174c8c0afa282941b9e4"}, NotarizedBlocksHashes: nil, Proposer: 0, Validators: nil, PubKeyBitmap: "", - Size: 643, + Size: 898, SizeTxs: 0, Timestamp: 0, StateRootHash: "", @@ -397,27 +400,26 @@ func TestBlockProcessor_PrepareBlockForDBMiniBlocksDetails(t *testing.T) { }, }, }, - SignersIndexes: []uint64{0, 1, 2}, TransactionPool: &outport.TransactionPool{ Transactions: map[string]*outport.TxInfo{ - txHash: { + hex.EncodeToString([]byte(txHash)): { ExecutionOrder: 2, }, - notExecutedTxHash: { + hex.EncodeToString([]byte(notExecutedTxHash)): { ExecutionOrder: 0, }, }, Rewards: map[string]*outport.RewardInfo{ - rewardsTxHash: { + hex.EncodeToString([]byte(rewardsTxHash)): { ExecutionOrder: 3, }, }, InvalidTxs: map[string]*outport.TxInfo{ - invalidTxHash: { + hex.EncodeToString([]byte(invalidTxHash)): { ExecutionOrder: 1, }}, SmartContractResults: map[string]*outport.SCRInfo{ - scrHash: { + hex.EncodeToString([]byte(scrHash)): { ExecutionOrder: 0, }, }, @@ -431,16 +433,17 @@ func TestBlockProcessor_PrepareBlockForDBMiniBlocksDetails(t *testing.T) { require.Equal(t, &data.Block{ Hash: "68617368", - Size: int64(341), + Size: int64(723), + SizeTxs: 94, AccumulatedFees: "0", DeveloperFees: "0", TxCount: uint32(5), SearchOrder: uint64(1020), MiniBlocksHashes: []string{ - "ee29d9b4a5017b7351974110d6a3f28ce6612476582f16b7849e3e87c647fc2d", - "c067de5b3c0031a14578699b1c3cdb9a19039e4a7b3fae6a94932ad3f70cf375", - "758f925b254ea0a6ad1bcbe3ddfcc73418ed4c8712506aafddc4da703295ad63", - "28a96506c2999838923f5310b3bb1d6849b5a259b429790d9eeb21c2a1402f82", + "8987edec270eb942d8ea9051fe301673aea29890919f5849882617aabcc7a248", + "1183f422a5b76c3cb7b439334f1fe7235c8d09f577e0f1e15e62cd05b9a81950", + "b24e307f3917e84603d3ebfb9c03c8fc651b62cb68ca884c3ff015b66a610a79", + "c0a855563172b2f72be569963d26d4fae38d4371342e2bf3ded93466a72f36f3", }, MiniBlocksDetails: []*data.MiniBlocksDetails{ { diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index 33e78c66..0e401ee0 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "io/ioutil" - "math/big" "strconv" "strings" "testing" @@ -80,43 +79,6 @@ func createMockElasticProcessorArgs() *ArgElasticProcessor { } } -func newTestTxPool() map[string]coreData.TransactionHandlerWithGasUsedAndFee { - txPool := map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - "tx1": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - Nonce: uint64(1), - Value: big.NewInt(1), - RcvAddr: []byte("receiver_address1"), - SndAddr: []byte("sender_address1"), - GasPrice: uint64(10000), - GasLimit: uint64(1000), - Data: []byte("tx_data1"), - Signature: []byte("signature1"), - }, 0, big.NewInt(0)), - "tx2": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - Nonce: uint64(2), - Value: big.NewInt(2), - RcvAddr: []byte("receiver_address2"), - SndAddr: []byte("sender_address2"), - GasPrice: uint64(10000), - GasLimit: uint64(1000), - Data: []byte("tx_data2"), - Signature: []byte("signature2"), - }, 0, big.NewInt(0)), - "tx3": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - Nonce: uint64(3), - Value: big.NewInt(3), - RcvAddr: []byte("receiver_address3"), - SndAddr: []byte("sender_address3"), - GasPrice: uint64(10000), - GasLimit: uint64(1000), - Data: []byte("tx_data3"), - Signature: []byte("signature3"), - }, 0, big.NewInt(0)), - } - - return txPool -} - func newTestBlockBody() *dataBlock.Body { return &dataBlock.Body{ MiniBlocks: []*dataBlock.MiniBlock{ @@ -701,6 +663,7 @@ func TestElasticProcessor_SaveTransactionNoDataShouldNotDoRequest(t *testing.T) BlockData: &outport.BlockData{ Body: &dataBlock.Body{}, }, + TransactionPool: &outport.TransactionPool{}, }, }) require.Nil(t, err) diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go index 51347c0d..3327677e 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go @@ -1,6 +1,7 @@ package logsevents import ( + "encoding/hex" "math/big" "testing" "time" @@ -57,8 +58,8 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) t.Parallel() logsAndEvents := map[string]*transaction.Log{ - "wrong": nil, - "h3": { + hex.EncodeToString([]byte("wrong")): nil, + hex.EncodeToString([]byte("h3")): { Events: []*transaction.Event{ { Address: []byte("addr"), @@ -68,7 +69,7 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) }, }, - "h1": { + hex.EncodeToString([]byte("h1")): { Address: []byte("address"), Events: []*transaction.Event{ { @@ -79,7 +80,7 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) }, }, - "h2": { + hex.EncodeToString([]byte("h2")): { Events: []*transaction.Event{ { Address: []byte("addr"), @@ -89,7 +90,7 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) nil, }, }, - "h4": { + hex.EncodeToString([]byte("h4")): { Events: []*transaction.Event{ { Address: []byte("addr"), @@ -99,7 +100,7 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) nil, }, }, - "h5": { + hex.EncodeToString([]byte("h5")): { Address: []byte("contract"), Events: []*transaction.Event{ { @@ -109,7 +110,7 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) }, }, }, - "h6": { + hex.EncodeToString([]byte("h6")): { Address: []byte("contract-second"), Events: []*transaction.Event{ { @@ -191,7 +192,7 @@ func TestLogsAndEventsProcessor_PrepareLogsForDB(t *testing.T) { logsAndEvents := map[string]*transaction.Log{ "wrong": nil, - "txHash": &transaction.Log{ + "txHash": { Address: []byte("address"), Events: []*transaction.Event{ { diff --git a/process/elasticproc/miniblocks/miniblocksProcessor_test.go b/process/elasticproc/miniblocks/miniblocksProcessor_test.go index 3ca8f4b2..c5b80ccb 100644 --- a/process/elasticproc/miniblocks/miniblocksProcessor_test.go +++ b/process/elasticproc/miniblocks/miniblocksProcessor_test.go @@ -136,9 +136,9 @@ func TestMiniblocksProcessor_GetMiniblocksHashesHexEncoded(t *testing.T) { } expectedHashes := []string{ - "c57392e53257b4861f5e406349a8deb89c6dbc2127564ee891a41a188edbf01a", - "28fda294dc987e5099d75e53cd6f87a9a42b96d55242a634385b5d41175c0c21", - "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "0796d34e8d443fd31bf4d9ec4051421b4d5d0e8c1db9ff942d6f4dc3a9ca2803", + "4cc379ab1f0aef6602e85a0a7ffabb5bc9a2ba646dc0fd720028e06527bf873f", + "8748c4677b01f7db984004fa8465afbf55feaab4b573174c8c0afa282941b9e4", } miniblocksHashes := mp.GetMiniblocksHashesHexEncoded(header, body) require.Equal(t, expectedHashes, miniblocksHashes) @@ -177,9 +177,9 @@ func TestMiniblocksProcessor_GetMiniblocksHashesHexEncodedImportDBMode(t *testin } expectedHashes := []string{ - "3acf97c324f5e8cd1e2d87de862b3105a9f08262c7914be2e186ced2a1cf1124", - "40a551b2ebc5e4b5a55e73d49ec056c72af6314606850c4d54dadfad3a7e23e5", - "4a270e1ddac6b429c14c7ebccdcdd53e4f68aeebfc41552c775a7f5a5c35d06d", + "11a1bb4065e16a2e93b2b5ac5957b7b69f1cfba7579b170b24f30dab2d3162e0", + "68e9a4360489ab7a6e99f92e05d1a3f06a982b7b157ac23fdf39f2392bf88e15", + "d1fd2a5c95c8899ebbaad035b6b0f77c5103b3aacfe630b1a7c51468d682bb1b", } miniblocksHashes := mp.GetMiniblocksHashesHexEncoded(header, body) require.Equal(t, expectedHashes, miniblocksHashes) diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index 91ff15b2..02771456 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -123,7 +123,7 @@ func checkDataIndexerParams(arguments ArgsIndexerFactory) error { return dataindexer.ErrNilHasher } if check.IfNil(arguments.HeaderMarshaller) { - return fmt.Errorf("%w header marshaller", dataindexer.ErrNilMarshalizer) + return fmt.Errorf("%w: header marshaller", dataindexer.ErrNilMarshalizer) } return nil diff --git a/process/factory/indexerFactory_test.go b/process/factory/indexerFactory_test.go index 43043541..b4de4c94 100644 --- a/process/factory/indexerFactory_test.go +++ b/process/factory/indexerFactory_test.go @@ -21,6 +21,7 @@ func createMockIndexerFactoryArgs() ArgsIndexerFactory { UserName: "", Password: "", Marshalizer: &mock.MarshalizerMock{}, + HeaderMarshaller: &mock.MarshalizerMock{}, Hasher: &mock.HasherMock{}, AddressPubkeyConverter: mock.NewPubkeyConverterMock(32), ValidatorPubkeyConverter: &mock.PubkeyConverterMock{}, @@ -80,6 +81,15 @@ func TestNewIndexerFactory(t *testing.T) { }, exError: dataindexer.ErrNilHasher, }, + { + name: "NilHeaderMarshaller", + argsFunc: func() ArgsIndexerFactory { + args := createMockIndexerFactoryArgs() + args.HeaderMarshaller = nil + return args + }, + exError: dataindexer.ErrNilMarshalizer, + }, { name: "EmptyUrl", argsFunc: func() ArgsIndexerFactory { From 8e7e2541446a98de1a7ea56ca238cfbd08d90f38 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 15 Mar 2023 18:45:55 +0200 Subject: [PATCH 020/189] fix integration tests 1 --- .../accountsBalanceNftTransfer_test.go | 80 +++++---- .../accountsBalanceWithLowerTimestamp_test.go | 80 +++++---- integrationtests/accountsESDTRollback_test.go | 32 ++-- .../accountsESDTWithTokenType_test.go | 107 ++++++------ integrationtests/claimRewards_test.go | 43 ++--- integrationtests/createNFTWithTags_test.go | 79 ++++----- integrationtests/delegators_test.go | 102 +++++------ integrationtests/esdtTransfer_test.go | 26 +-- .../issueTokenAndSetRoles_test.go | 159 ++++++++--------- integrationtests/issueToken_test.go | 94 +++++----- integrationtests/logsCrossShard_test.go | 89 +++++----- ...sferWithScCallAndErrorSignaledBySC_test.go | 63 +++---- integrationtests/nftIssueCreateBurn_test.go | 73 ++++---- .../nftTransferCrossShard_test.go | 127 ++++++++------ .../nftTransferCrossWithScCall_test.go | 28 +-- integrationtests/relayedTx_test.go | 62 ++++--- integrationtests/scCallIntraShard_test.go | 59 ++++--- integrationtests/transactions_test.go | 25 ++- integrationtests/updateNFT_test.go | 163 ++++++++---------- scripts/script.sh | 2 +- 20 files changed, 748 insertions(+), 745 deletions(-) diff --git a/integrationtests/accountsBalanceNftTransfer_test.go b/integrationtests/accountsBalanceNftTransfer_test.go index 1c66a240..23b0b342 100644 --- a/integrationtests/accountsBalanceNftTransfer_test.go +++ b/integrationtests/accountsBalanceNftTransfer_test.go @@ -3,12 +3,14 @@ package integrationtests import ( + "encoding/hex" "fmt" "math/big" "testing" "github.com/multiversx/mx-chain-core-go/core" coreData "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/data/alteredAccount" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/transaction" @@ -16,6 +18,28 @@ import ( "github.com/stretchr/testify/require" ) +func createOutportBlockWithHeader( + body *dataBlock.Body, + header coreData.HeaderHandler, + pool *outport.TransactionPool, + coreAlteredAccounts map[string]*alteredAccount.AlteredAccount, + importDD bool, + numOfShards uint32, +) *outport.OutportBlockWithHeader { + return &outport.OutportBlockWithHeader{ + OutportBlock: &outport.OutportBlock{ + BlockData: &outport.BlockData{ + Body: body, + }, + TransactionPool: pool, + AlteredAccounts: coreAlteredAccounts, + NumberOfShards: numOfShards, + IsImportDB: importDD, + }, + Header: header, + } +} + func TestAccountBalanceNFTTransfer(t *testing.T) { setLogLevelDebug() @@ -36,28 +60,25 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { ShardID: 1, } - pool := &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(addr), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte("NFT-abcdef"), big.NewInt(7440483).Bytes(), big.NewInt(1).Bytes()}, - }, - nil, + pool := &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(addr), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte("NFT-abcdef"), big.NewInt(7440483).Bytes(), big.NewInt(1).Bytes()}, }, + nil, }, }, }, } - coreAlteredAccounts := map[string]*outport.AlteredAccount{ + coreAlteredAccounts := map[string]*alteredAccount.AlteredAccount{ addr: { Address: addr, - Tokens: []*outport.AccountTokenData{ + Tokens: []*alteredAccount.AccountTokenData{ { Identifier: "NFT-abcdef", Nonce: 7440483, @@ -67,7 +88,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) require.Nil(t, err) ids := []string{fmt.Sprintf("%s-NFT-abcdef-718863", addr)} @@ -85,19 +106,16 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { ShardID: 1, } - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: []byte("test-address-balance-1"), - Identifier: []byte(core.BuiltInFunctionESDTNFTTransfer), - Topics: [][]byte{[]byte("NFT-abcdef"), big.NewInt(7440483).Bytes(), big.NewInt(1).Bytes(), decodeAddress(addrReceiver)}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: []byte("test-address-balance-1"), + Identifier: []byte(core.BuiltInFunctionESDTNFTTransfer), + Topics: [][]byte{[]byte("NFT-abcdef"), big.NewInt(7440483).Bytes(), big.NewInt(1).Bytes(), decodeAddress(addrReceiver)}, }, + nil, }, }, }, @@ -106,10 +124,10 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { esProc, err = CreateElasticProcessor(esClient) require.Nil(t, err) - coreAlteredAccounts = map[string]*outport.AlteredAccount{ + coreAlteredAccounts = map[string]*alteredAccount.AlteredAccount{ addr: { Address: addr, - Tokens: []*outport.AccountTokenData{ + Tokens: []*alteredAccount.AccountTokenData{ { Identifier: "NFT-abcdef", Nonce: 7440483, @@ -119,7 +137,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { }, addrReceiver: { Address: addrReceiver, - Tokens: []*outport.AccountTokenData{ + Tokens: []*alteredAccount.AccountTokenData{ { Identifier: "NFT-abcdef", Nonce: 7440483, @@ -128,7 +146,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { }, }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) require.Nil(t, err) ids = []string{fmt.Sprintf("%s-NFT-abcdef-718863", addr)} diff --git a/integrationtests/accountsBalanceWithLowerTimestamp_test.go b/integrationtests/accountsBalanceWithLowerTimestamp_test.go index 30ab7673..baef6736 100644 --- a/integrationtests/accountsBalanceWithLowerTimestamp_test.go +++ b/integrationtests/accountsBalanceWithLowerTimestamp_test.go @@ -3,12 +3,13 @@ package integrationtests import ( + "encoding/hex" "fmt" "math/big" "testing" "github.com/multiversx/mx-chain-core-go/core" - coreData "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/data/alteredAccount" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/esdt" "github.com/multiversx/mx-chain-core-go/data/outport" @@ -33,10 +34,10 @@ func TestIndexAccountsBalance(t *testing.T) { addr := "erd17umc0uvel62ng30k5uprqcxh3ue33hq608njejaqljuqzqlxtzuqeuzlcv" addr2 := "erd1m2pyjudsqt8gn0tnsstht35gfqcfx8ku5utz07mf2r6pq3sfxjzszhcx6w" - alteredAccount := &outport.AlteredAccount{ + account := &alteredAccount.AlteredAccount{ Address: addr, Balance: "0", - Tokens: []*outport.AccountTokenData{ + Tokens: []*alteredAccount.AccountTokenData{ { Identifier: "TTTT-abcd", Balance: "1000", @@ -45,9 +46,9 @@ func TestIndexAccountsBalance(t *testing.T) { }, } - coreAlteredAccounts := map[string]*outport.AlteredAccount{ - addr: alteredAccount, - addr2: alteredAccount, + coreAlteredAccounts := map[string]*alteredAccount.AlteredAccount{ + addr: account, + addr2: account, } esProc, err := CreateElasticProcessor(esClient) @@ -59,25 +60,22 @@ func TestIndexAccountsBalance(t *testing.T) { ShardID: 2, } - pool := &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: []byte("eeeebbbb"), - Identifier: []byte(core.BuiltInFunctionESDTTransfer), - Topics: [][]byte{[]byte("TTTT-abcd"), nil, big.NewInt(1).Bytes()}, - }, - nil, + pool := &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: []byte("eeeebbbb"), + Identifier: []byte(core.BuiltInFunctionESDTTransfer), + Topics: [][]byte{[]byte("TTTT-abcd"), nil, big.NewInt(1).Bytes()}, }, + nil, }, }, }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) require.Nil(t, err) ids := []string{addr} @@ -100,7 +98,7 @@ func TestIndexAccountsBalance(t *testing.T) { ShardID: 2, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) require.Nil(t, err) ids = []string{addr} @@ -123,28 +121,28 @@ func TestIndexAccountsBalance(t *testing.T) { } coreAlteredAccounts[addr].Balance = "2000" - coreAlteredAccounts[addr].AdditionalData = &outport.AdditionalAccountData{ + coreAlteredAccounts[addr].AdditionalData = &alteredAccount.AdditionalAccountData{ IsSender: true, BalanceChanged: true, } - pool = &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - "h1": outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ - SndAddr: []byte(addr), - }, 0, big.NewInt(0)), + pool = &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + hex.EncodeToString([]byte("h1")): { + Transaction: &transaction.Transaction{ + SndAddr: []byte(addr), + }, + FeeInfo: &outport.FeeInfo{}, + }, }, - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(addr2), - Identifier: []byte(core.BuiltInFunctionESDTTransfer), - Topics: [][]byte{[]byte("TTTT-abcd"), nil, big.NewInt(1).Bytes()}, - }, - nil, + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(addr2), + Identifier: []byte(core.BuiltInFunctionESDTTransfer), + Topics: [][]byte{[]byte("TTTT-abcd"), nil, big.NewInt(1).Bytes()}, }, + nil, }, }, }, @@ -159,7 +157,7 @@ func TestIndexAccountsBalance(t *testing.T) { }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) require.Nil(t, err) ids = []string{addr} @@ -188,13 +186,13 @@ func TestIndexAccountsBalance(t *testing.T) { coreAlteredAccounts[addr].Balance = "2000" coreAlteredAccounts[addr].Tokens[0].Balance = "0" - coreAlteredAccounts[addr].AdditionalData = &outport.AdditionalAccountData{ + coreAlteredAccounts[addr].AdditionalData = &alteredAccount.AdditionalAccountData{ IsSender: false, BalanceChanged: false, } - pool.Txs = make(map[string]coreData.TransactionHandlerWithGasUsedAndFee) - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) + pool.Transactions = make(map[string]*outport.TxInfo) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) require.Nil(t, err) ids = []string{addr} diff --git a/integrationtests/accountsESDTRollback_test.go b/integrationtests/accountsESDTRollback_test.go index d33d7a2a..0717dec2 100644 --- a/integrationtests/accountsESDTRollback_test.go +++ b/integrationtests/accountsESDTRollback_test.go @@ -3,13 +3,14 @@ package integrationtests import ( + "encoding/hex" "encoding/json" "fmt" "math/big" "testing" "github.com/multiversx/mx-chain-core-go/core" - coreData "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/data/alteredAccount" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/esdt" "github.com/multiversx/mx-chain-core-go/data/outport" @@ -32,15 +33,15 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { }, } addr := "erd1sqy2ywvswp09ef7qwjhv8zwr9kzz3xas6y2ye5nuryaz0wcnfzzsnq0am3" - coreAlteredAccounts := map[string]*outport.AlteredAccount{ + coreAlteredAccounts := map[string]*alteredAccount.AlteredAccount{ addr: { Address: addr, - Tokens: []*outport.AccountTokenData{ + Tokens: []*alteredAccount.AccountTokenData{ { Identifier: "TOKEN-eeee", Nonce: 2, Balance: "1000", - MetaData: &outport.TokenMetaData{ + MetaData: &alteredAccount.TokenMetaData{ Creator: "creator", }, Properties: "3032", @@ -54,19 +55,16 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { // CREATE SEMI-FUNGIBLE TOKEN esdtDataBytes, _ := json.Marshal(esdtToken) - pool := &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(addr), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte("TOKEN-eeee"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, - }, - nil, + pool := &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(addr), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte("TOKEN-eeee"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, }, + nil, }, }, }, @@ -79,7 +77,7 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { ShardID: 2, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) require.Nil(t, err) ids := []string{fmt.Sprintf("%s-TOKEN-eeee-02", addr)} diff --git a/integrationtests/accountsESDTWithTokenType_test.go b/integrationtests/accountsESDTWithTokenType_test.go index 77d0139e..b8f51136 100644 --- a/integrationtests/accountsESDTWithTokenType_test.go +++ b/integrationtests/accountsESDTWithTokenType_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "encoding/hex" "encoding/json" "fmt" "math/big" @@ -10,7 +11,7 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/core" - coreData "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/data/alteredAccount" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/esdt" "github.com/multiversx/mx-chain-core-go/data/outport" @@ -37,25 +38,22 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { } address := "erd1sqy2ywvswp09ef7qwjhv8zwr9kzz3xas6y2ye5nuryaz0wcnfzzsnq0am3" - pool := &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte("issueSemiFungible"), - Topics: [][]byte{[]byte("SEMI-abcd"), []byte("SEMI-token"), []byte("SEM"), []byte(core.SemiFungibleESDT)}, - }, - nil, + pool := &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte("issueSemiFungible"), + Topics: [][]byte{[]byte("SEMI-abcd"), []byte("SEMI-token"), []byte("SEM"), []byte(core.SemiFungibleESDT)}, }, + nil, }, }, }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) require.Nil(t, err) ids := []string{"SEMI-abcd"} @@ -65,17 +63,17 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/token-after-issue.json"), string(genericResponse.Docs[0].Source)) // ################ CREATE SEMI FUNGIBLE TOKEN ########################## - coreAlteredAccounts := map[string]*outport.AlteredAccount{ + coreAlteredAccounts := map[string]*alteredAccount.AlteredAccount{ address: { Address: address, Balance: "1000", - Tokens: []*outport.AccountTokenData{ + Tokens: []*alteredAccount.AccountTokenData{ { Identifier: "SEMI-abcd", Balance: "1000", Nonce: 2, Properties: "3032", - MetaData: &outport.TokenMetaData{ + MetaData: &alteredAccount.TokenMetaData{ Creator: "creator", }, }, @@ -98,25 +96,22 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { } esdtDataBytes, _ := json.Marshal(esdtData) - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte("SEMI-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte("SEMI-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, }, + nil, }, }, }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) require.Nil(t, err) ids = []string{fmt.Sprintf("%s-SEMI-abcd-02", address)} @@ -137,17 +132,17 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) body := &dataBlock.Body{} address := "erd1l29zsl2dqq988kvr2y0xlfv9ydgnvhzkatfd8ccalpag265pje8qn8lslf" - coreAlteredAccounts := map[string]*outport.AlteredAccount{ + coreAlteredAccounts := map[string]*alteredAccount.AlteredAccount{ address: { Address: address, Balance: "1000", - Tokens: []*outport.AccountTokenData{ + Tokens: []*alteredAccount.AccountTokenData{ { Identifier: "TTTT-abcd", Nonce: 2, Balance: "1000", Properties: "3032", - MetaData: &outport.TokenMetaData{ + MetaData: &alteredAccount.TokenMetaData{ Creator: "erd1l29zsl2dqq988kvr2y0xlfv9ydgnvhzkatfd8ccalpag265pje8qn8lslf", }, }, @@ -170,25 +165,22 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) } esdtDataBytes, _ := json.Marshal(esdtData) - pool := &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte("TTTT-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, - }, - nil, + pool := &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte("TTTT-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, }, + nil, }, }, }, } - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) require.Nil(t, err) ids := []string{fmt.Sprintf("%s-TTTT-abcd-02", address)} @@ -209,25 +201,22 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) esProc, err = CreateElasticProcessor(esClient) require.Nil(t, err) - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte("issueSemiFungible"), - Topics: [][]byte{[]byte("TTTT-abcd"), []byte("TTTT-token"), []byte("SEM"), []byte(core.SemiFungibleESDT)}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte("issueSemiFungible"), + Topics: [][]byte{[]byte("TTTT-abcd"), []byte("TTTT-token"), []byte("SEM"), []byte(core.SemiFungibleESDT)}, }, + nil, }, }, }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) require.Nil(t, err) ids = []string{"TTTT-abcd"} diff --git a/integrationtests/claimRewards_test.go b/integrationtests/claimRewards_test.go index 38a093e6..bc27113c 100644 --- a/integrationtests/claimRewards_test.go +++ b/integrationtests/claimRewards_test.go @@ -8,7 +8,6 @@ import ( "testing" "github.com/multiversx/mx-chain-core-go/core" - coreData "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" @@ -88,34 +87,38 @@ func TestTransactionWithClaimRewardsGasRefund(t *testing.T) { Value: big.NewInt(0), } - tx := outport.NewTransactionHandlerWithGasAndFee(tx1, 1068000, big.NewInt(78000000000000)) - tx.SetInitialPaidFee(big.NewInt(127320000000000)) + txInfo := &outport.TxInfo{ + Transaction: tx1, + FeeInfo: &outport.FeeInfo{ + GasUsed: 1068000, + Fee: big.NewInt(78000000000000), + InitialPaidFee: big.NewInt(127320000000000), + }, + ExecutionOrder: 0, + } - pool := &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): tx, + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + hex.EncodeToString(txHash): txInfo, }, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), - string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(scrHash2): {SmartContractResult: scr2, FeeInfo: &outport.FeeInfo{}}, + hex.EncodeToString(scrHash1): {SmartContractResult: scr1, FeeInfo: &outport.FeeInfo{}}, }, - Logs: []*coreData.LogData{ - { - TxHash: string(txHash), - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(addressSender), - Identifier: []byte("writeLog"), - Topics: [][]byte{[]byte("something")}, - }, + Logs: map[string]*transaction.Log{ + hex.EncodeToString(txHash): { + Events: []*transaction.Event{ + { + Address: decodeAddress(addressSender), + Identifier: []byte("writeLog"), + Topics: [][]byte{[]byte("something")}, }, }, }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/createNFTWithTags_test.go b/integrationtests/createNFTWithTags_test.go index fdcad119..2b71508e 100644 --- a/integrationtests/createNFTWithTags_test.go +++ b/integrationtests/createNFTWithTags_test.go @@ -10,7 +10,7 @@ import ( "testing" "github.com/multiversx/mx-chain-core-go/core" - coreData "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/data/alteredAccount" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/esdt" "github.com/multiversx/mx-chain-core-go/data/outport" @@ -47,38 +47,35 @@ func TestCreateNFTWithTags(t *testing.T) { // CREATE A FIRST NFT WITH THE TAGS address1 := "erd1v7e552pz9py4hv6raan0c4jflez3e6csdmzcgrncg0qrnk4tywvsqx0h5j" - pool := &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte("DESK-abcd"), big.NewInt(1).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, - }, - nil, + pool := &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte("DESK-abcd"), big.NewInt(1).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, }, + nil, }, }, }, } - coreAlteredAccounts := map[string]*outport.AlteredAccount{ + coreAlteredAccounts := map[string]*alteredAccount.AlteredAccount{ address1: { Address: address1, Balance: "0", - Tokens: []*outport.AccountTokenData{ + Tokens: []*alteredAccount.AccountTokenData{ { - AdditionalData: &outport.AdditionalAccountTokenData{ + AdditionalData: &alteredAccount.AdditionalAccountTokenData{ IsNFTCreate: true, }, Identifier: "DESK-abcd", Nonce: 1, Balance: "1000", Properties: "3032", - MetaData: &outport.TokenMetaData{ + MetaData: &alteredAccount.TokenMetaData{ Creator: "creator", Attributes: []byte("tags:hello,something,do,music,art,gallery;metadata:QmZ2QqaGq4bqsEzs5JLTjRmmvR2GAR4qXJZBN8ibfDdaud"), }, @@ -88,7 +85,7 @@ func TestCreateNFTWithTags(t *testing.T) { } body := &dataBlock.Body{} - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) require.Nil(t, err) ids := []string{fmt.Sprintf("%s-DESK-abcd-01", address1)} @@ -115,19 +112,16 @@ func TestCreateNFTWithTags(t *testing.T) { require.Equal(t, len(ids), tagsChecked) // CREATE A SECOND NFT WITH THE SAME TAGS - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte("DESK-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte("DESK-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, }, + nil, }, }, }, @@ -135,7 +129,7 @@ func TestCreateNFTWithTags(t *testing.T) { coreAlteredAccounts[address1].Tokens[0].Nonce = 2 body = &dataBlock.Body{} - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) require.Nil(t, err) genericResponse = &GenericResponse{} @@ -164,26 +158,23 @@ func TestCreateNFTWithTags(t *testing.T) { esProc, err = CreateElasticProcessor(esClient) require.Nil(t, err) - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte("DESK-abcd"), big.NewInt(3).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte("DESK-abcd"), big.NewInt(3).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, }, + nil, }, }, }, } body = &dataBlock.Body{} - err = esProc.SaveTransactions(body, header, pool, coreAlteredAccounts, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) require.Nil(t, err) ids = append(ids, "XFxcXFxcXFxcXFxcXFxcXFxcXA==", "JycnJw==", "PDw8Pj4+JiYmJiYmJiYmJiYmJiYm") diff --git a/integrationtests/delegators_test.go b/integrationtests/delegators_test.go index 79f00107..c26883d9 100644 --- a/integrationtests/delegators_test.go +++ b/integrationtests/delegators_test.go @@ -3,12 +3,12 @@ package integrationtests import ( + "encoding/hex" "math/big" "testing" "time" "github.com/multiversx/mx-chain-core-go/core" - coreData "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/transaction" @@ -40,26 +40,23 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { // delegate delegatedValue, _ := big.NewInt(0).SetString("200000000000000000000", 10) - pool := &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Address: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqhllllsajxzat"), - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("delegate"), - Topics: [][]byte{delegatedValue.Bytes(), delegatedValue.Bytes(), big.NewInt(10).Bytes(), delegatedValue.Bytes()}, - }, - nil, + pool := &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Address: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqhllllsajxzat"), + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("delegate"), + Topics: [][]byte{delegatedValue.Bytes(), delegatedValue.Bytes(), big.NewInt(10).Bytes(), delegatedValue.Bytes()}, }, + nil, }, }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids := []string{"9v/pLAXxUZJ4Oy1U+x5al/Xg5sebh1dYCRTeZwg/u68="} @@ -71,27 +68,24 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { // unDelegate 1 unDelegatedValue, _ := big.NewInt(0).SetString("50000000000000000000", 10) totalDelegation, _ := big.NewInt(0).SetString("150000000000000000000", 10) - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h2", - LogHandler: &transaction.Log{ - Address: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqhllllsajxzat"), - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("unDelegate"), - Topics: [][]byte{unDelegatedValue.Bytes(), totalDelegation.Bytes(), big.NewInt(10).Bytes(), totalDelegation.Bytes(), []byte("1")}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h2")): { + Address: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqhllllsajxzat"), + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("unDelegate"), + Topics: [][]byte{unDelegatedValue.Bytes(), totalDelegation.Bytes(), big.NewInt(10).Bytes(), totalDelegation.Bytes(), []byte("1")}, }, + nil, }, }, }, } header.TimeStamp = 5050 - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse) @@ -101,27 +95,24 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { // unDelegate 2 unDelegatedValue, _ = big.NewInt(0).SetString("25500000000000000000", 10) totalDelegation, _ = big.NewInt(0).SetString("124500000000000000000", 10) - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h3", - LogHandler: &transaction.Log{ - Address: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqhllllsajxzat"), - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("unDelegate"), - Topics: [][]byte{unDelegatedValue.Bytes(), totalDelegation.Bytes(), big.NewInt(10).Bytes(), totalDelegation.Bytes(), []byte("2")}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h3")): { + Address: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqhllllsajxzat"), + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("unDelegate"), + Topics: [][]byte{unDelegatedValue.Bytes(), totalDelegation.Bytes(), big.NewInt(10).Bytes(), totalDelegation.Bytes(), []byte("2")}, }, + nil, }, }, }, } header.TimeStamp = 5060 - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse) @@ -141,27 +132,24 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { // withdraw withdrawValue, _ := big.NewInt(0).SetString("725500000000000000000", 10) - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h4", - LogHandler: &transaction.Log{ - Address: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqhllllsajxzat"), - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("withdraw"), - Topics: [][]byte{withdrawValue.Bytes(), totalDelegation.Bytes(), big.NewInt(10).Bytes(), totalDelegation.Bytes(), []byte("false"), []byte("1"), []byte("2")}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h4")): { + Address: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqhllllsajxzat"), + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("withdraw"), + Topics: [][]byte{withdrawValue.Bytes(), totalDelegation.Bytes(), big.NewInt(10).Bytes(), totalDelegation.Bytes(), []byte("false"), []byte("1"), []byte("2")}, }, + nil, }, }, }, } header.TimeStamp = 5070 - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse) diff --git a/integrationtests/esdtTransfer_test.go b/integrationtests/esdtTransfer_test.go index 81c98778..13a6e5e8 100644 --- a/integrationtests/esdtTransfer_test.go +++ b/integrationtests/esdtTransfer_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - coreData "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" @@ -84,19 +83,26 @@ func TestESDTTransferTooMuchGasProvided(t *testing.T) { } initialPaidFee, _ := big.NewInt(0).SetString("104000110000000", 10) - tx := outport.NewTransactionHandlerWithGasAndFee(txESDT, 104011, big.NewInt(104000110000000)) - tx.SetInitialPaidFee(initialPaidFee) + txInfo := &outport.TxInfo{ + Transaction: txESDT, + FeeInfo: &outport.FeeInfo{ + GasUsed: 104011, + Fee: initialPaidFee, + InitialPaidFee: big.NewInt(104000110000000), + }, + ExecutionOrder: 0, + } - pool := &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): tx, + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + hex.EncodeToString(txHash): txInfo, }, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), - string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(scrHash2): {SmartContractResult: scr2, FeeInfo: &outport.FeeInfo{}}, + hex.EncodeToString(scrHash1): {SmartContractResult: scr1, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/issueTokenAndSetRoles_test.go b/integrationtests/issueTokenAndSetRoles_test.go index 8e4f8a42..a598f9f4 100644 --- a/integrationtests/issueTokenAndSetRoles_test.go +++ b/integrationtests/issueTokenAndSetRoles_test.go @@ -3,11 +3,12 @@ package integrationtests import ( + "encoding/hex" "math/big" "testing" "github.com/multiversx/mx-chain-core-go/core" - coreData "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/data/alteredAccount" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/transaction" @@ -33,30 +34,27 @@ func TestIssueTokenAndSetRole(t *testing.T) { address1 := "erd1k04pxr6c0gvlcx4rd5fje0a4uy33axqxwz0fpcrgtfdy3nrqauqqgvxprv" address2 := "erd1suhxyflu4w4pqdxmushpxzc6a3qszr89m8uswzqcvyh0mh9mzxwqdwkm0x" - pool := &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("issueSemiFungible"), - Topics: [][]byte{[]byte("TOK-abcd"), []byte("semi-token"), []byte("SEMI"), []byte(core.SemiFungibleESDT)}, - }, - { - Address: decodeAddress(address1), - Identifier: []byte("upgradeProperties"), - Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), []byte("canUpgrade"), []byte("true")}, - }, - nil, + pool := &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("issueSemiFungible"), + Topics: [][]byte{[]byte("TOK-abcd"), []byte("semi-token"), []byte("SEMI"), []byte(core.SemiFungibleESDT)}, }, + { + Address: decodeAddress(address1), + Identifier: []byte("upgradeProperties"), + Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), []byte("canUpgrade"), []byte("true")}, + }, + nil, }, }, }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) require.Nil(t, err) ids := []string{"TOK-abcd"} @@ -66,26 +64,23 @@ func TestIssueTokenAndSetRole(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-issue-ok.json"), string(genericResponse.Docs[0].Source)) // SET ROLES - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionSetESDTRole), - Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte(core.ESDTRoleNFTCreate), []byte(core.ESDTRoleNFTBurn)}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionSetESDTRole), + Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte(core.ESDTRoleNFTCreate), []byte(core.ESDTRoleNFTBurn)}, }, + nil, }, }, }, } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -95,22 +90,19 @@ func TestIssueTokenAndSetRole(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-set-role.json"), string(genericResponse.Docs[0].Source)) // TRANSFER ROLE - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreateRoleTransfer), - Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte("false")}, - }, - { - Address: decodeAddress(address2), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreateRoleTransfer), - Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte("true")}, - }, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreateRoleTransfer), + Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte("false")}, + }, + { + Address: decodeAddress(address2), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreateRoleTransfer), + Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte("true")}, }, }, }, @@ -118,7 +110,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -128,26 +120,23 @@ func TestIssueTokenAndSetRole(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-transfer-role.json"), string(genericResponse.Docs[0].Source)) // UNSET ROLES - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionUnSetESDTRole), - Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte(core.ESDTRoleNFTBurn)}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionUnSetESDTRole), + Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte(core.ESDTRoleNFTBurn)}, }, + nil, }, }, }, } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -175,26 +164,23 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { address1 := "erd1k04pxr6c0gvlcx4rd5fje0a4uy33axqxwz0fpcrgtfdy3nrqauqqgvxprv" // SET ROLES - pool := &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionSetESDTRole), - Topics: [][]byte{[]byte("TTT-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte(core.ESDTRoleNFTCreate), []byte(core.ESDTRoleNFTBurn)}, - }, - nil, + pool := &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionSetESDTRole), + Topics: [][]byte{[]byte("TTT-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte(core.ESDTRoleNFTCreate), []byte(core.ESDTRoleNFTBurn)}, }, + nil, }, }, }, } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) require.Nil(t, err) ids := []string{"TTT-abcd"} @@ -204,25 +190,22 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-set-roles-first.json"), string(genericResponse.Docs[0].Source)) // ISSUE - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("issueSemiFungible"), - Topics: [][]byte{[]byte("TTT-abcd"), []byte("semi-token"), []byte("SEMI"), []byte(core.SemiFungibleESDT)}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("issueSemiFungible"), + Topics: [][]byte{[]byte("TTT-abcd"), []byte("semi-token"), []byte("SEMI"), []byte(core.SemiFungibleESDT)}, }, + nil, }, }, }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) require.Nil(t, err) ids = []string{"TTT-abcd"} diff --git a/integrationtests/issueToken_test.go b/integrationtests/issueToken_test.go index fadfdee4..2c321dde 100644 --- a/integrationtests/issueToken_test.go +++ b/integrationtests/issueToken_test.go @@ -3,11 +3,11 @@ package integrationtests import ( + "encoding/hex" "math/big" "testing" "github.com/multiversx/mx-chain-core-go/core" - coreData "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/transaction" @@ -33,25 +33,22 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { address1 := "erd1v7e552pz9py4hv6raan0c4jflez3e6csdmzcgrncg0qrnk4tywvsqx0h5j" address2 := "erd1acjlnuhkd8773sqhmw85r0ur4lcyuqgm0n69h9ttxh0gwxtuuzxq4lckh6" - pool := &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("issueSemiFungible"), - Topics: [][]byte{[]byte("SSSS-abcd"), []byte("semi-token"), []byte("SSSS"), []byte(core.SemiFungibleESDT), big.NewInt(18).Bytes()}, - }, - nil, + pool := &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("issueSemiFungible"), + Topics: [][]byte{[]byte("SSSS-abcd"), []byte("semi-token"), []byte("SSSS"), []byte(core.SemiFungibleESDT), big.NewInt(18).Bytes()}, }, + nil, }, }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids := []string{"SSSS-abcd"} @@ -65,26 +62,23 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/issueToken/token-semi.json"), string(genericResponse.Docs[0].Source)) // transfer ownership - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("transferOwnership"), - Topics: [][]byte{[]byte("SSSS-abcd"), []byte("semi-token"), []byte("SSSS"), []byte(core.SemiFungibleESDT), decodeAddress(address2)}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("transferOwnership"), + Topics: [][]byte{[]byte("SSSS-abcd"), []byte("semi-token"), []byte("SSSS"), []byte(core.SemiFungibleESDT), decodeAddress(address2)}, }, + nil, }, }, }, } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) @@ -96,26 +90,23 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/issueToken/token-semi-after-transfer-ownership.json"), string(genericResponse.Docs[0].Source)) // do pause - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("ESDTPause"), - Topics: [][]byte{[]byte("SSSS-abcd")}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("ESDTPause"), + Topics: [][]byte{[]byte("SSSS-abcd")}, }, + nil, }, }, }, } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) @@ -123,26 +114,23 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/issueToken/token-semi-after-pause.json"), string(genericResponse.Docs[0].Source)) // do unPause - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("ESDTUnPause"), - Topics: [][]byte{[]byte("SSSS-abcd")}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("ESDTUnPause"), + Topics: [][]byte{[]byte("SSSS-abcd")}, }, + nil, }, }, }, } header.TimeStamp = 10000 - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) diff --git a/integrationtests/logsCrossShard_test.go b/integrationtests/logsCrossShard_test.go index cb3a3087..0cd4dd0a 100644 --- a/integrationtests/logsCrossShard_test.go +++ b/integrationtests/logsCrossShard_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/multiversx/mx-chain-core-go/core" - coreData "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/data/alteredAccount" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/transaction" @@ -37,25 +37,22 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { logID := hex.EncodeToString([]byte("cross-log")) // index on source - pool := &outport.Pool{ - Logs: []*coreData.LogData{ - { - LogHandler: &transaction.Log{ - Address: decodeAddress(address1), - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionESDTTransfer), - Topics: [][]byte{[]byte("ESDT-abcd"), big.NewInt(0).Bytes(), big.NewInt(1).Bytes()}, - }, - nil, + pool := &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("cross-log")): { + Address: decodeAddress(address1), + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTTransfer), + Topics: [][]byte{[]byte("ESDT-abcd"), big.NewInt(0).Bytes(), big.NewInt(1).Bytes()}, }, + nil, }, - TxHash: "cross-log", }, }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) require.Nil(t, err) ids := []string{logID} @@ -72,31 +69,28 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { Round: 50, TimeStamp: 6040, } - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - LogHandler: &transaction.Log{ - Address: decodeAddress(address1), - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionESDTTransfer), - Topics: [][]byte{[]byte("ESDT-abcd"), big.NewInt(0).Bytes(), big.NewInt(1).Bytes()}, - }, - { - - Address: decodeAddress(address2), - Identifier: []byte("do-something"), - Topics: [][]byte{[]byte("topic1"), []byte("topic2")}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("cross-log")): { + Address: decodeAddress(address1), + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTTransfer), + Topics: [][]byte{[]byte("ESDT-abcd"), big.NewInt(0).Bytes(), big.NewInt(1).Bytes()}, }, + { + + Address: decodeAddress(address2), + Identifier: []byte("do-something"), + Topics: [][]byte{[]byte("topic1"), []byte("topic2")}, + }, + nil, }, - TxHash: "cross-log", }, }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse) @@ -111,25 +105,22 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { Round: 50, TimeStamp: 5000, } - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - LogHandler: &transaction.Log{ - Address: decodeAddress(address1), - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionESDTTransfer), - Topics: [][]byte{[]byte("ESDT-abcd"), big.NewInt(0).Bytes(), big.NewInt(1).Bytes()}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("cross-log")): { + Address: decodeAddress(address1), + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTTransfer), + Topics: [][]byte{[]byte("ESDT-abcd"), big.NewInt(0).Bytes(), big.NewInt(1).Bytes()}, }, + nil, }, - TxHash: "cross-log", }, }, } - err = esProc.SaveTransactions(body, header, pool, map[string]*outport.AlteredAccount{}, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse) diff --git a/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go b/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go index e5ef68ac..6fb7392f 100644 --- a/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go +++ b/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - coreData "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" @@ -64,7 +63,7 @@ func TestMultiTransferCrossShardAndScCallErrorSignaledBySC(t *testing.T) { OriginalTxHash: txHash, } - tx := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + tx := &transaction.Transaction{ Nonce: 79, SndAddr: decodeAddress(address1), RcvAddr: decodeAddress(address1), @@ -72,18 +71,27 @@ func TestMultiTransferCrossShardAndScCallErrorSignaledBySC(t *testing.T) { GasPrice: 1000000000, Data: []byte("MultiESDTNFTTransfer@000000000000000005005ebeb3515cb42056a81d42adaf756a3f63a360bfb055@02@5745474c442d626434643739@@38e62046fb1a0000@584d45582d666461333535@07@048907e58284c28e898e29@6164644c697175696469747950726f7879@00000000000000000500ebd304c2f34a6b3f6a57c133ab7b8c6f81dc40155483@38d78f595785c000@0487deac313c6f6b111906"), Value: big.NewInt(0), - }, 150000000, big.NewInt(1904415000000000)) - tx.SetInitialPaidFee(big.NewInt(1904415000000000)) + } + + txInfo := &outport.TxInfo{ + Transaction: tx, + FeeInfo: &outport.FeeInfo{ + GasUsed: 150000000, + Fee: big.NewInt(1904415000000000), + InitialPaidFee: big.NewInt(1904415000000000), + }, + ExecutionOrder: 0, + } - pool := &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): tx, + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + hex.EncodeToString(txHash): txInfo, }, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(scrHash1): {SmartContractResult: scr1, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} @@ -126,32 +134,29 @@ func TestMultiTransferCrossShardAndScCallErrorSignaledBySC(t *testing.T) { ReturnMessage: []byte("error signalled by smartcontract"), } - pool = &outport.Pool{ - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), - string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), + pool = &outport.TransactionPool{ + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(scrHash1): {SmartContractResult: scr1, FeeInfo: &outport.FeeInfo{}}, + hex.EncodeToString(scrHash2): {SmartContractResult: scr2, FeeInfo: &outport.FeeInfo{}}, }, - Logs: []*coreData.LogData{ - { - TxHash: string(scrHash1), - LogHandler: &transaction.Log{ - Address: decodeAddress(address2), - Events: []*transaction.Event{ - { - Address: decodeAddress(address2), - Identifier: []byte("signalError"), - }, - { - Address: decodeAddress(address1), - Identifier: []byte("internalVMErrors"), - }, + Logs: map[string]*transaction.Log{ + hex.EncodeToString(scrHash1): { + Address: decodeAddress(address2), + Events: []*transaction.Event{ + { + Address: decodeAddress(address2), + Identifier: []byte("signalError"), + }, + { + Address: decodeAddress(address1), + Identifier: []byte("internalVMErrors"), }, }, }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids = []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/nftIssueCreateBurn_test.go b/integrationtests/nftIssueCreateBurn_test.go index 341ed977..55ce802e 100644 --- a/integrationtests/nftIssueCreateBurn_test.go +++ b/integrationtests/nftIssueCreateBurn_test.go @@ -3,12 +3,12 @@ package integrationtests import ( + "encoding/hex" "encoding/json" "math/big" "testing" "github.com/multiversx/mx-chain-core-go/core" - coreData "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/esdt" "github.com/multiversx/mx-chain-core-go/data/outport" @@ -36,25 +36,22 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { } address1 := "erd1ju8pkvg57cwdmjsjx58jlmnuf4l9yspstrhr9tgsrt98n9edpm2qtlgy99" - pool := &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("issueNonFungible"), - Topics: [][]byte{[]byte("NON-abcd"), []byte("NON-token"), []byte("NON"), []byte(core.NonFungibleESDT)}, - }, - nil, + pool := &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("issueNonFungible"), + Topics: [][]byte{[]byte("NON-abcd"), []byte("NON-token"), []byte("NON"), []byte(core.NonFungibleESDT)}, }, + nil, }, }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids := []string{"NON-abcd"} @@ -80,26 +77,23 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { } esdtDataBytes, _ := json.Marshal(esdtData) - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Address: decodeAddress(address1), - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte("NON-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Address: decodeAddress(address1), + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte("NON-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, }, + nil, }, }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids = []string{"NON-abcd-02"} @@ -116,25 +110,22 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { ShardID: 0, } - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - TxHash: "h1", - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionESDTNFTBurn), - Topics: [][]byte{[]byte("NON-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), decodeAddress(address1)}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTNFTBurn), + Topics: [][]byte{[]byte("NON-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), decodeAddress(address1)}, }, + nil, }, }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids = []string{"NON-abcd-02"} diff --git a/integrationtests/nftTransferCrossShard_test.go b/integrationtests/nftTransferCrossShard_test.go index 0fc53429..bf02fbfc 100644 --- a/integrationtests/nftTransferCrossShard_test.go +++ b/integrationtests/nftTransferCrossShard_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - coreData "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" @@ -63,7 +62,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { OriginalTxHash: txHash, } - tx := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + tx := &transaction.Transaction{ Nonce: 79, SndAddr: decodeAddress(address1), RcvAddr: decodeAddress(address1), @@ -71,18 +70,28 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { GasPrice: 1000000000, Data: []byte("ESDTNFTTransfer@4c4b4641524d2d336634663962@016534@6f1e6f01bc7627f5ae@00000000000000000500f1c8f2fdc58a63c6b201fc2ed629962d3dfa33fe7ceb@636f6d706f756e645265776172647350726f7879@000000000000000005004f79ec44bb13372b5ac9d996d749120f476427627ceb"), Value: big.NewInt(0), - }, 150000000, big.NewInt(1904415000000000)) - tx.SetInitialPaidFee(big.NewInt(1904415000000000)) + } + + txInfo := &outport.TxInfo{ + Transaction: tx, + FeeInfo: &outport.FeeInfo{ + GasUsed: 150000000, + Fee: big.NewInt(1904415000000000), + InitialPaidFee: big.NewInt(1904415000000000), + }, + ExecutionOrder: 0, + } - pool := &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): tx, + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + hex.EncodeToString(txHash): txInfo, }, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(scrHash1): {SmartContractResult: scr1, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -108,21 +117,21 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { } scrWithRefund := []byte("scrWithRefund") refundValueBig, _ := big.NewInt(0).SetString("101676480000000", 10) - poolDstShard := &outport.Pool{ - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), - string(scrWithRefund): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + poolDstShard := &outport.TransactionPool{ + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(scrHash1): {SmartContractResult: scr1, FeeInfo: &outport.FeeInfo{}}, + hex.EncodeToString(scrWithRefund): {SmartContractResult: &smartContractResult.SmartContractResult{ SndAddr: decodeAddress(address2), RcvAddr: decodeAddress(address1), PrevTxHash: []byte("f639cb7a0231191e04ec19dcb1359bd93a03fe8dc4a28a80d00835c5d1c988f8"), OriginalTxHash: txHash, Value: refundValueBig, Data: []byte("@6f6b@017d15@0000000e4d45584641524d2d6239336536300000000000017d15000000097045173cc97554b65d@0178af"), - }, 139832352, big.NewInt(1802738520000000)), + }, FeeInfo: &outport.FeeInfo{GasUsed: 139832352, Fee: big.NewInt(1802738520000000)}}, }, } - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, false, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -187,7 +196,7 @@ func TestNFTTransferCrossShard(t *testing.T) { OriginalTxHash: txHash, } - tx := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + tx := &transaction.Transaction{ Nonce: 79, SndAddr: decodeAddress(address1), RcvAddr: decodeAddress(address1), @@ -195,16 +204,25 @@ func TestNFTTransferCrossShard(t *testing.T) { GasPrice: 1000000000, Data: []byte("ESDTNFTTransfer@536f6d657468696e672d616263646566@01@01@00000000000000000500a7a02771aa07090e607f02b25f4d6d241bff32b990a2"), Value: big.NewInt(0), - }, 963500, big.NewInt(235850000000000)) - tx.SetInitialPaidFee(big.NewInt(276215000000000)) + } + + txInfo := &outport.TxInfo{ + Transaction: tx, + FeeInfo: &outport.FeeInfo{ + GasUsed: 963500, + Fee: big.NewInt(235850000000000), + InitialPaidFee: big.NewInt(276215000000000), + }, + ExecutionOrder: 0, + } refundValueBig, _ := big.NewInt(0).SetString("40365000000000", 10) - pool := &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): tx, + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + hex.EncodeToString(txHash): txInfo, }, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(scrHash1): {SmartContractResult: &smartContractResult.SmartContractResult{ Nonce: 80, Value: refundValueBig, GasPrice: 1000000000, @@ -213,11 +231,11 @@ func TestNFTTransferCrossShard(t *testing.T) { Data: []byte("@6f6b"), PrevTxHash: txHash, OriginalTxHash: txHash, - }, 0, big.NewInt(0)), - string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), + }, FeeInfo: &outport.FeeInfo{}}, + hex.EncodeToString(scrHash2): {SmartContractResult: scr2, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -242,20 +260,20 @@ func TestNFTTransferCrossShard(t *testing.T) { }, } scr3WithErrHash := []byte("scrWithError") - poolDstShard := &outport.Pool{ - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), - string(scr3WithErrHash): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + poolDstShard := &outport.TransactionPool{ + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(scrHash2): {SmartContractResult: scr2, FeeInfo: &outport.FeeInfo{}}, + hex.EncodeToString(scr3WithErrHash): {SmartContractResult: &smartContractResult.SmartContractResult{ SndAddr: decodeAddress(address2), RcvAddr: decodeAddress(address1), PrevTxHash: []byte("1546eb9970a6dc1710b6528274e75d5095c1349706f4ff70f52a1f58e1156316"), OriginalTxHash: txHash, Data: []byte("ESDTNFTTransfer@434f4c45435449452d323663313838@01@01@08011202000122e50108011204434f4f4c1a20e0f3ecf555f63f2d101241dfc98b4614aff9284edd50b46a1c6e36b83558744d20c4132a2e516d5a7961565631786a7866446255575a503178655a7676544d3156686f61346f594752444d706d4a727a52435a324368747470733a2f2f697066732e696f2f697066732f516d5a7961565631786a7866446255575a503178655a7676544d3156686f61346f594752444d706d4a727a52435a3a41746167733a436f6f6c3b6d657461646174613a516d5869417850396e535948515954546143357358717a4d32645856334142516145355241725932777a4e686179@75736572206572726f72"), - }, 0, big.NewInt(0)), + }, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, false, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -308,22 +326,22 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t }, } scr3WithErrHash := []byte("scrWithError") - poolDstShard := &outport.Pool{ - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), - string(scr3WithErrHash): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + poolDstShard := &outport.TransactionPool{ + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(scrHash2): {SmartContractResult: scr2, FeeInfo: &outport.FeeInfo{}}, + hex.EncodeToString(scr3WithErrHash): {SmartContractResult: &smartContractResult.SmartContractResult{ SndAddr: decodeAddress(address2), RcvAddr: decodeAddress(address1), PrevTxHash: []byte("1546eb9970a6dc1710b6528274e75d5095c1349706f4ff70f52a1f58e1156316"), OriginalTxHash: txHash, Data: []byte("ESDTNFTTransfer@434f4c4c454354494f4e2d323663313838@01@01@08011202000122e50108011204434f4f4c1a20e0f3ecf555f63f2d101241dfc98b4614aff9284edd50b46a1c6e36b83558744d20c4132a2e516d5a7961565631786a7866446255575a503178655a7676544d3156686f61346f594752444d706d4a727a52435a324368747470733a2f2f697066732e696f2f697066732f516d5a7961565631786a7866446255575a503178655a7676544d3156686f61346f594752444d706d4a727a52435a3a41746167733a436f6f6c3b6d657461646174613a516d5869417850396e535948515954546143357358717a4d32645856334142516145355241725932777a4e686179@75736572206572726f72"), - }, 0, big.NewInt(0)), + }, FeeInfo: &outport.FeeInfo{}}, }, } ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, false, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -353,7 +371,7 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t }, } - tx := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + tx := &transaction.Transaction{ Nonce: 79, SndAddr: decodeAddress(address1), RcvAddr: decodeAddress(address1), @@ -361,16 +379,25 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t GasPrice: 1000000000, Data: []byte("ESDTNFTTransfer@434f4c4c454354494f4e2d323663313838@01@01@00000000000000000500a7a02771aa07090e607f02b25f4d6d241bff32b990a2"), Value: big.NewInt(0), - }, 963500, big.NewInt(238820000000000)) - tx.SetInitialPaidFee(big.NewInt(595490000000000)) + } + + txInfo := &outport.TxInfo{ + Transaction: tx, + FeeInfo: &outport.FeeInfo{ + GasUsed: 963500, + Fee: big.NewInt(238820000000000), + InitialPaidFee: big.NewInt(595490000000000), + }, + ExecutionOrder: 0, + } refundValueBig, _ := big.NewInt(0).SetString("40365000000000", 10) - pool := &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): tx, + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + hex.EncodeToString(txHash): txInfo, }, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(scrHash1): {SmartContractResult: &smartContractResult.SmartContractResult{ Nonce: 80, Value: refundValueBig, GasPrice: 1000000000, @@ -379,11 +406,11 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t Data: []byte("@6f6b"), PrevTxHash: txHash, OriginalTxHash: txHash, - }, 0, big.NewInt(0)), - string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), + }, FeeInfo: &outport.FeeInfo{}}, + hex.EncodeToString(scrHash2): {SmartContractResult: scr2, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) diff --git a/integrationtests/nftTransferCrossWithScCall_test.go b/integrationtests/nftTransferCrossWithScCall_test.go index d725495f..5d83fdda 100644 --- a/integrationtests/nftTransferCrossWithScCall_test.go +++ b/integrationtests/nftTransferCrossWithScCall_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - coreData "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" @@ -60,7 +59,7 @@ func TestNFTTransferCrossShardWithScCall(t *testing.T) { OriginalTxHash: txHash, } - tx := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + tx := &transaction.Transaction{ Nonce: 79, SndAddr: decodeAddress(address1), RcvAddr: decodeAddress(address1), @@ -68,18 +67,27 @@ func TestNFTTransferCrossShardWithScCall(t *testing.T) { GasPrice: 1000000000, Data: []byte("ESDTNFTTransfer@4d45584641524d2d636362323532@078b@0347543e5b59c9be8670@00000000000000000500a7a02771aa07090e607f02b25f4d6d241bff32b990a2@636c61696d52657761726473"), Value: big.NewInt(0), - }, 5000000, big.NewInt(595490000000000)) - tx.SetInitialPaidFee(big.NewInt(595490000000000)) + } + + txInfo := &outport.TxInfo{ + Transaction: tx, + FeeInfo: &outport.FeeInfo{ + GasUsed: 5000000, + Fee: big.NewInt(595490000000000), + InitialPaidFee: big.NewInt(595490000000000), + }, + ExecutionOrder: 0, + } - pool := &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): tx, + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + hex.EncodeToString(txHash): txInfo, }, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(scrHash2): {SmartContractResult: scr2, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/relayedTx_test.go b/integrationtests/relayedTx_test.go index 9943a4de..b1883d9c 100644 --- a/integrationtests/relayedTx_test.go +++ b/integrationtests/relayedTx_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - coreData "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" @@ -73,18 +72,25 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { OriginalTxHash: txHash, } - tx := outport.NewTransactionHandlerWithGasAndFee(initialTx, 16610000, big.NewInt(1760000000000000)) - tx.SetInitialPaidFee(big.NewInt(1760000000000000)) + txInfo := &outport.TxInfo{ + Transaction: initialTx, + FeeInfo: &outport.FeeInfo{ + GasUsed: 16610000, + Fee: big.NewInt(1760000000000000), + InitialPaidFee: big.NewInt(1760000000000000), + }, + ExecutionOrder: 0, + } - pool := &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): tx, + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + hex.EncodeToString(txHash): txInfo, }, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(scrHash1): {SmartContractResult: scr1, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -110,10 +116,10 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { } scrWithRefund := []byte("scrWithRefund") refundValueBig, _ := big.NewInt(0).SetString("86271830000000", 10) - poolDstShard := &outport.Pool{ - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), - string(scrWithRefund): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + poolDstShard := &outport.TransactionPool{ + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(scrHash1): {SmartContractResult: scr1, FeeInfo: &outport.FeeInfo{}}, + hex.EncodeToString(scrWithRefund): {SmartContractResult: &smartContractResult.SmartContractResult{ Nonce: 3, SndAddr: decodeAddress(address3), RcvAddr: decodeAddress(address1), @@ -122,11 +128,11 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { Value: refundValueBig, Data: []byte(""), ReturnMessage: []byte("gas refund for relayer"), - }, 7982817, big.NewInt(1673728170000000)), + }, FeeInfo: &outport.FeeInfo{GasUsed: 7982817, Fee: big.NewInt(1673728170000000)}}, }, } - err = esProc.SaveTransactions(bodyDstShard, header, poolDstShard, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, false, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -206,18 +212,26 @@ func TestRelayedTransactionIntraShard(t *testing.T) { Value: refundValueBig, } - tx := outport.NewTransactionHandlerWithGasAndFee(initialTx, 10556000, big.NewInt(2257820000000000)) - tx.SetInitialPaidFee(big.NewInt(2306320000000000)) - pool := &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): tx, + txInfo := &outport.TxInfo{ + Transaction: initialTx, + FeeInfo: &outport.FeeInfo{ + GasUsed: 10556000, + Fee: big.NewInt(2257820000000000), + InitialPaidFee: big.NewInt(2306320000000000), + }, + ExecutionOrder: 0, + } + + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + hex.EncodeToString(txHash): txInfo, }, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(scr1, 0, big.NewInt(0)), - string(scrHash2): outport.NewTransactionHandlerWithGasAndFee(scr2, 0, big.NewInt(0)), + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(scrHash1): {SmartContractResult: scr1, FeeInfo: &outport.FeeInfo{}}, + hex.EncodeToString(scrHash2): {SmartContractResult: scr2, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/scCallIntraShard_test.go b/integrationtests/scCallIntraShard_test.go index e9f44e4d..2dbce180 100644 --- a/integrationtests/scCallIntraShard_test.go +++ b/integrationtests/scCallIntraShard_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - coreData "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" @@ -45,7 +44,7 @@ func TestTransactionWithSCCallFail(t *testing.T) { address1 := "erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv" address2 := "erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqfhllllscrt56r" refundValueBig, _ := big.NewInt(0).SetString("5000000000000000000", 10) - tx := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + tx := &transaction.Transaction{ Nonce: 46, SndAddr: decodeAddress(address1), RcvAddr: decodeAddress(address2), @@ -53,15 +52,24 @@ func TestTransactionWithSCCallFail(t *testing.T) { GasPrice: 1000000000, Data: []byte("delegate"), Value: refundValueBig, - }, 12000000, big.NewInt(181380000000000)) - tx.SetInitialPaidFee(big.NewInt(181380000000000)) + } + + txInfo := &outport.TxInfo{ + Transaction: tx, + FeeInfo: &outport.FeeInfo{ + GasUsed: 12000000, + Fee: big.NewInt(181380000000000), + InitialPaidFee: big.NewInt(181380000000000), + }, + ExecutionOrder: 0, + } - pool := &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): tx, + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + hex.EncodeToString(txHash): txInfo, }, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(scrHash1): {SmartContractResult: &smartContractResult.SmartContractResult{ Nonce: 46, Value: refundValueBig, GasPrice: 0, @@ -71,10 +79,10 @@ func TestTransactionWithSCCallFail(t *testing.T) { PrevTxHash: txHash, OriginalTxHash: txHash, ReturnMessage: []byte("total delegation cap reached"), - }, 0, big.NewInt(0)), + }, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -116,7 +124,7 @@ func TestTransactionWithScCallSuccess(t *testing.T) { address1 := "erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv" address2 := "erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqfhllllscrt56r" - tx := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + tx := &transaction.Transaction{ Nonce: 101, SndAddr: decodeAddress(address1), RcvAddr: decodeAddress(address2), @@ -124,16 +132,25 @@ func TestTransactionWithScCallSuccess(t *testing.T) { GasPrice: 1000000000, Data: []byte("claimRewards"), Value: big.NewInt(0), - }, 33891715, big.NewInt(406237150000000)) - tx.SetInitialPaidFee(big.NewInt(2567320000000000)) + } + + txInfo := &outport.TxInfo{ + Transaction: tx, + FeeInfo: &outport.FeeInfo{ + GasUsed: 33891715, + Fee: big.NewInt(406237150000000), + InitialPaidFee: big.NewInt(2567320000000000), + }, + ExecutionOrder: 0, + } refundValueBig, _ := big.NewInt(0).SetString("2161082850000000", 10) - pool := &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): tx, + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + string(txHash): txInfo, }, - Scrs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(scrHash1): outport.NewTransactionHandlerWithGasAndFee(&smartContractResult.SmartContractResult{ + SmartContractResults: map[string]*outport.SCRInfo{ + string(scrHash1): {SmartContractResult: &smartContractResult.SmartContractResult{ Nonce: 102, Value: refundValueBig, GasPrice: 1000000000, @@ -142,10 +159,10 @@ func TestTransactionWithScCallSuccess(t *testing.T) { Data: []byte("@6f6b"), PrevTxHash: txHash, OriginalTxHash: txHash, - }, 0, big.NewInt(0)), + }, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/transactions_test.go b/integrationtests/transactions_test.go index 05574386..77b8b694 100644 --- a/integrationtests/transactions_test.go +++ b/integrationtests/transactions_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - coreData "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/transaction" @@ -39,7 +38,7 @@ func TestElasticIndexerSaveTransactions(t *testing.T) { }, }, } - tx := outport.NewTransactionHandlerWithGasAndFee(&transaction.Transaction{ + tx := &transaction.Transaction{ Nonce: 1, SndAddr: decodeAddress("erd1w7jyzuj6cv4ngw8luhlkakatjpmjh3ql95lmxphd3vssc4vpymks6k5th7"), RcvAddr: decodeAddress("erd1ahmy0yjhjg87n755yv99nzla22zzwfud55sa69gk3anyxyyucq9q2hgxww"), @@ -47,14 +46,24 @@ func TestElasticIndexerSaveTransactions(t *testing.T) { GasPrice: 1000000000, Data: []byte("transfer"), Value: big.NewInt(1234), - }, 62000, big.NewInt(62000000000000)) - tx.SetInitialPaidFee(big.NewInt(62080000000000)) - pool := &outport.Pool{ - Txs: map[string]coreData.TransactionHandlerWithGasUsedAndFee{ - string(txHash): tx, + } + + txInfo := &outport.TxInfo{ + Transaction: tx, + FeeInfo: &outport.FeeInfo{ + GasUsed: 62000, + Fee: big.NewInt(62000000000000), + InitialPaidFee: big.NewInt(62080000000000), + }, + ExecutionOrder: 0, + } + + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + hex.EncodeToString(txHash): txInfo, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/updateNFT_test.go b/integrationtests/updateNFT_test.go index fe629c83..47cfc6e5 100644 --- a/integrationtests/updateNFT_test.go +++ b/integrationtests/updateNFT_test.go @@ -3,12 +3,12 @@ package integrationtests import ( + "encoding/hex" "encoding/json" "math/big" "testing" "github.com/multiversx/mx-chain-core-go/core" - coreData "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/esdt" "github.com/multiversx/mx-chain-core-go/data/outport" @@ -42,24 +42,21 @@ func TestNFTUpdateMetadata(t *testing.T) { // CREATE NFT data address := "erd1w7jyzuj6cv4ngw8luhlkakatjpmjh3ql95lmxphd3vssc4vpymks6k5th7" - pool := &outport.Pool{ - Logs: []*coreData.LogData{ - { - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(1).Bytes(), marshalizedCreate}, - }, - nil, + pool := &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(1).Bytes(), marshalizedCreate}, }, + nil, }, - TxHash: "h1", }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids := []string{"NFT-abcd-0e"} @@ -69,45 +66,39 @@ func TestNFTUpdateMetadata(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/updateNFT/token.json"), string(genericResponse.Docs[0].Source)) // Add URIS 1 - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte(core.BuiltInFunctionESDTNFTAddURI), - Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("uri1"), []byte("uri2")}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte(core.BuiltInFunctionESDTNFTAddURI), + Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("uri1"), []byte("uri2")}, }, + nil, }, - TxHash: "h1", }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) // Add URIS 2 --- results should be the same - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte(core.BuiltInFunctionESDTNFTAddURI), - Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("uri1"), []byte("uri2")}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte(core.BuiltInFunctionESDTNFTAddURI), + Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("uri1"), []byte("uri2")}, }, + nil, }, - TxHash: "h1", }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) // Update attributes 1 @@ -117,24 +108,21 @@ func TestNFTUpdateMetadata(t *testing.T) { require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/updateNFT/token-after-add-uris.json"), string(genericResponse.Docs[0].Source)) - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte(core.BuiltInFunctionESDTNFTUpdateAttributes), - Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("tags:test,free,fun;description:This is a test description for an awesome nft;metadata:metadata-test")}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte(core.BuiltInFunctionESDTNFTUpdateAttributes), + Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("tags:test,free,fun;description:This is a test description for an awesome nft;metadata:metadata-test")}, }, + nil, }, - TxHash: "h1", }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids = []string{"NFT-abcd-0e"} @@ -145,24 +133,21 @@ func TestNFTUpdateMetadata(t *testing.T) { // Update attributes 2 - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte(core.BuiltInFunctionESDTNFTUpdateAttributes), - Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("something")}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte(core.BuiltInFunctionESDTNFTUpdateAttributes), + Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("something")}, }, + nil, }, - TxHash: "h1", }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids = []string{"NFT-abcd-0e"} @@ -172,24 +157,21 @@ func TestNFTUpdateMetadata(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/updateNFT/token-after-update-attributes-second.json"), string(genericResponse.Docs[0].Source)) // Freeze nft - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte(core.BuiltInFunctionESDTFreeze), - Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("something")}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte(core.BuiltInFunctionESDTFreeze), + Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("something")}, }, + nil, }, - TxHash: "h1", }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids = []string{"NFT-abcd-0e"} genericResponse = &GenericResponse{} @@ -198,24 +180,21 @@ func TestNFTUpdateMetadata(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/updateNFT/token-after-freeze.json"), string(genericResponse.Docs[0].Source)) // UnFreeze nft - pool = &outport.Pool{ - Logs: []*coreData.LogData{ - { - LogHandler: &transaction.Log{ - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte(core.BuiltInFunctionESDTUnFreeze), - Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("something")}, - }, - nil, + pool = &outport.TransactionPool{ + Logs: map[string]*transaction.Log{ + hex.EncodeToString([]byte("h1")): { + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte(core.BuiltInFunctionESDTUnFreeze), + Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("something")}, }, + nil, }, - TxHash: "h1", }, }, } - err = esProc.SaveTransactions(body, header, pool, nil, false, testNumOfShards) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) require.Nil(t, err) ids = []string{"NFT-abcd-0e"} genericResponse = &GenericResponse{} diff --git a/scripts/script.sh b/scripts/script.sh index bba2b141..f19a14cc 100755 --- a/scripts/script.sh +++ b/scripts/script.sh @@ -1,6 +1,6 @@ IMAGE_NAME=elastic-container DEFAULT_ES_VERSION=7.16.2 -INDICES_LIST=("rating" "transactions" "blocks" "validators" "miniblocks" "rounds" "accounts" "accountshistory" "receipts" "scresults" "accountsesdt" "accountsesdthistory" "epochinfo" "scdeploys" "tokens" "tags" "logs" "delegators" "operations", "esdts") +INDICES_LIST=("rating" "transactions" "blocks" "validators" "miniblocks" "rounds" "accounts" "accountshistory" "receipts" "scresults" "accountsesdt" "accountsesdthistory" "epochinfo" "scdeploys" "tokens" "tags" "logs" "delegators" "operations" "esdts") start() { From 2cbf2050298d880d341bf77799f3eda7681198a8 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 16 Mar 2023 10:11:33 +0200 Subject: [PATCH 021/189] fix integration tests 2 --- templates/noKibana/esdts.go | 3 +++ templates/withKibana/esdts.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/templates/noKibana/esdts.go b/templates/noKibana/esdts.go index c2d1e9a1..072d215a 100644 --- a/templates/noKibana/esdts.go +++ b/templates/noKibana/esdts.go @@ -49,6 +49,9 @@ var ESDTs = Object{ }, }, }, + "paused": Object{ + "type": "boolean", + }, "properties": Object{ "properties": Object{ "canMint": Object{ diff --git a/templates/withKibana/esdts.go b/templates/withKibana/esdts.go index 466f8f33..cc27f793 100644 --- a/templates/withKibana/esdts.go +++ b/templates/withKibana/esdts.go @@ -49,6 +49,9 @@ var ESDTs = Object{ }, }, }, + "paused": Object{ + "type": "boolean", + }, "properties": Object{ "properties": Object{ "canMint": Object{ From 07675da8f0ba22e69335333aab8b800aab00e8de Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 16 Mar 2023 10:40:12 +0200 Subject: [PATCH 022/189] fix integration tests and unit tests 1 --- integrationtests/logsCrossShard_test.go | 6 +++--- integrationtests/scCallIntraShard_test.go | 4 ++-- .../testdata/claimRewards/tx-claim-rewards.json | 2 +- integrationtests/testdata/esdtTransfer/esdt-transfer.json | 2 +- .../transaction-after-execution-of-scr-dst-shard.json | 2 +- .../transaction-executed-on-source.json | 2 +- .../op-nft-transfer-sc-call-after-refund.json | 2 +- .../nftTransferCrossShard/tx-complete-with-status.json | 2 +- .../tx-nft-transfer-failed-on-dst.json | 2 +- .../tx-nft-transfer-sc-call-after-refund.json | 2 +- .../tx-nft-transfer-sc-call-source.json | 2 +- .../testdata/nftTransferCrossShard/tx-nft-transfer.json | 2 +- .../cross-shard-transfer-with-sc-call.json | 2 +- .../testdata/relayedTx/relayed-tx-after-refund.json | 2 +- integrationtests/testdata/relayedTx/relayed-tx-intra.json | 2 +- .../testdata/relayedTx/relayed-tx-source.json | 2 +- .../testdata/scCallIntraShard/claim-rewards.json | 2 +- .../testdata/scCallIntraShard/sc-call-fail.json | 2 +- integrationtests/testdata/transactions/move-balance.json | 2 +- process/elasticproc/logsevents/logsAndEventsProcessor.go | 8 +++----- 20 files changed, 25 insertions(+), 27 deletions(-) diff --git a/integrationtests/logsCrossShard_test.go b/integrationtests/logsCrossShard_test.go index 0cd4dd0a..496239d2 100644 --- a/integrationtests/logsCrossShard_test.go +++ b/integrationtests/logsCrossShard_test.go @@ -39,7 +39,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { // index on source pool := &outport.TransactionPool{ Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("cross-log")): { + logID: { Address: decodeAddress(address1), Events: []*transaction.Event{ { @@ -71,7 +71,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { } pool = &outport.TransactionPool{ Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("cross-log")): { + logID: { Address: decodeAddress(address1), Events: []*transaction.Event{ { @@ -107,7 +107,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { } pool = &outport.TransactionPool{ Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("cross-log")): { + logID: { Address: decodeAddress(address1), Events: []*transaction.Event{ { diff --git a/integrationtests/scCallIntraShard_test.go b/integrationtests/scCallIntraShard_test.go index 2dbce180..30296ff7 100644 --- a/integrationtests/scCallIntraShard_test.go +++ b/integrationtests/scCallIntraShard_test.go @@ -147,10 +147,10 @@ func TestTransactionWithScCallSuccess(t *testing.T) { refundValueBig, _ := big.NewInt(0).SetString("2161082850000000", 10) pool := &outport.TransactionPool{ Transactions: map[string]*outport.TxInfo{ - string(txHash): txInfo, + hex.EncodeToString(txHash): txInfo, }, SmartContractResults: map[string]*outport.SCRInfo{ - string(scrHash1): {SmartContractResult: &smartContractResult.SmartContractResult{ + hex.EncodeToString(scrHash1): {SmartContractResult: &smartContractResult.SmartContractResult{ Nonce: 102, Value: refundValueBig, GasPrice: 1000000000, diff --git a/integrationtests/testdata/claimRewards/tx-claim-rewards.json b/integrationtests/testdata/claimRewards/tx-claim-rewards.json index b4a76975..c11ef56c 100644 --- a/integrationtests/testdata/claimRewards/tx-claim-rewards.json +++ b/integrationtests/testdata/claimRewards/tx-claim-rewards.json @@ -1,5 +1,5 @@ { - "miniBlockHash": "582fecdda564e76162bbb07d797c9ee3780a82fab226f5faced83a6cce2cf5d9", + "miniBlockHash": "d5f20802b232c47bd738db31df9046a0c3703d692f72104c4caf0e0a9506373f", "nonce": 617, "round": 50, "value": "0", diff --git a/integrationtests/testdata/esdtTransfer/esdt-transfer.json b/integrationtests/testdata/esdtTransfer/esdt-transfer.json index b23c9761..871e60a6 100644 --- a/integrationtests/testdata/esdtTransfer/esdt-transfer.json +++ b/integrationtests/testdata/esdtTransfer/esdt-transfer.json @@ -1,5 +1,5 @@ { - "miniBlockHash": "1ecea6dff9ab9a785a2d55720e88c1bbd7d9c56310a035d16163e373879cd0e1", + "miniBlockHash": "0787323c7f992341d261e87d3ec662b79772732832e6bcf8d1a8a9949006b6af", "nonce": 6, "round": 50, "value": "0", diff --git a/integrationtests/testdata/multiTransferWithScCallAndErrorSignaledBySC/transaction-after-execution-of-scr-dst-shard.json b/integrationtests/testdata/multiTransferWithScCallAndErrorSignaledBySC/transaction-after-execution-of-scr-dst-shard.json index e475e9f7..1956ce24 100644 --- a/integrationtests/testdata/multiTransferWithScCallAndErrorSignaledBySC/transaction-after-execution-of-scr-dst-shard.json +++ b/integrationtests/testdata/multiTransferWithScCallAndErrorSignaledBySC/transaction-after-execution-of-scr-dst-shard.json @@ -18,7 +18,7 @@ "initialPaidFee": "1904415000000000", "gasLimit": 150000000, "gasUsed": 150000000, - "miniBlockHash": "2ea5bfe2968c98f3fcb059c6dcb3b0b42863ebe957748649f566db12548d2091", + "miniBlockHash": "71a5272ba7198b74f8a59cb9db4a270d2cfb1a7aefacbe17a27d1aa6f0d60b57", "receivers": [ "erd1qqqqqqqqqqqqqpgqt6ltx52ukss9d2qag2k67at28a36xc9lkp2sr06394", "erd1qqqqqqqqqqqqqpgqt6ltx52ukss9d2qag2k67at28a36xc9lkp2sr06394" diff --git a/integrationtests/testdata/multiTransferWithScCallAndErrorSignaledBySC/transaction-executed-on-source.json b/integrationtests/testdata/multiTransferWithScCallAndErrorSignaledBySC/transaction-executed-on-source.json index b638fbe5..a86b0649 100644 --- a/integrationtests/testdata/multiTransferWithScCallAndErrorSignaledBySC/transaction-executed-on-source.json +++ b/integrationtests/testdata/multiTransferWithScCallAndErrorSignaledBySC/transaction-executed-on-source.json @@ -18,7 +18,7 @@ "initialPaidFee": "1904415000000000", "gasLimit": 150000000, "gasUsed": 150000000, - "miniBlockHash": "2ea5bfe2968c98f3fcb059c6dcb3b0b42863ebe957748649f566db12548d2091", + "miniBlockHash": "71a5272ba7198b74f8a59cb9db4a270d2cfb1a7aefacbe17a27d1aa6f0d60b57", "receivers": [ "erd1qqqqqqqqqqqqqpgqt6ltx52ukss9d2qag2k67at28a36xc9lkp2sr06394", "erd1qqqqqqqqqqqqqpgqt6ltx52ukss9d2qag2k67at28a36xc9lkp2sr06394" diff --git a/integrationtests/testdata/nftTransferCrossShard/op-nft-transfer-sc-call-after-refund.json b/integrationtests/testdata/nftTransferCrossShard/op-nft-transfer-sc-call-after-refund.json index 4cb70b38..32c4ff66 100644 --- a/integrationtests/testdata/nftTransferCrossShard/op-nft-transfer-sc-call-after-refund.json +++ b/integrationtests/testdata/nftTransferCrossShard/op-nft-transfer-sc-call-after-refund.json @@ -1,5 +1,5 @@ { - "miniBlockHash": "b30aaa656bf101a7fb87f6c02a9da9e70cd053a79de24f5d14276232757d9766", + "miniBlockHash": "d5a0138c5a29200f398cc8cb168a5b1cb8d4a16890746381b5cbed14c7bb379f", "nonce": 79, "round": 50, "value": "0", diff --git a/integrationtests/testdata/nftTransferCrossShard/tx-complete-with-status.json b/integrationtests/testdata/nftTransferCrossShard/tx-complete-with-status.json index 9826d540..0fb5adbd 100644 --- a/integrationtests/testdata/nftTransferCrossShard/tx-complete-with-status.json +++ b/integrationtests/testdata/nftTransferCrossShard/tx-complete-with-status.json @@ -16,7 +16,7 @@ "nonce": 79, "gasLimit": 5000000, "gasUsed": 963500, - "miniBlockHash": "db7161a83f08489cba131e55f042536ee49116b622e33e70335a13e51a6c268c", + "miniBlockHash": "6d6986fad945a4587927692450372cc0d7aea2a7c1b5326b485fe0a9da0c0065", "round": 50, "hasScResults": true, "sender": "erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv", diff --git a/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-failed-on-dst.json b/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-failed-on-dst.json index 45c02f7b..af8b4ee9 100644 --- a/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-failed-on-dst.json +++ b/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-failed-on-dst.json @@ -1,5 +1,5 @@ { - "miniBlockHash": "83c60064098aa89220b5adc9d71f22b489bfc78cb3dcb516381102d7fec959e8", + "miniBlockHash": "1b39a9702116ba9020d928bf681c49ee4b462fcc8e3209fbf256459f97e6ef84", "nonce": 79, "round": 50, "value": "0", diff --git a/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-sc-call-after-refund.json b/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-sc-call-after-refund.json index f276884c..1223f2e7 100644 --- a/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-sc-call-after-refund.json +++ b/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-sc-call-after-refund.json @@ -1,5 +1,5 @@ { - "miniBlockHash": "b30aaa656bf101a7fb87f6c02a9da9e70cd053a79de24f5d14276232757d9766", + "miniBlockHash": "d5a0138c5a29200f398cc8cb168a5b1cb8d4a16890746381b5cbed14c7bb379f", "nonce": 79, "round": 50, "value": "0", diff --git a/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-sc-call-source.json b/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-sc-call-source.json index a5fa15f7..0342ec98 100644 --- a/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-sc-call-source.json +++ b/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-sc-call-source.json @@ -16,7 +16,7 @@ "nonce": 79, "gasLimit": 150000000, "gasUsed": 150000000, - "miniBlockHash": "b30aaa656bf101a7fb87f6c02a9da9e70cd053a79de24f5d14276232757d9766", + "miniBlockHash": "d5a0138c5a29200f398cc8cb168a5b1cb8d4a16890746381b5cbed14c7bb379f", "round": 50, "hasScResults": true, "sender": "erd1ef9xx3k3m89azf4c4xc98wpcdnx5h0cnxy6em47r6dc4alud0uwqx24f50", diff --git a/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer.json b/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer.json index bf3d4021..dc2e7dc7 100644 --- a/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer.json +++ b/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer.json @@ -17,7 +17,7 @@ "nonce": 79, "gasLimit": 5000000, "gasUsed": 963500, - "miniBlockHash": "83c60064098aa89220b5adc9d71f22b489bfc78cb3dcb516381102d7fec959e8", + "miniBlockHash": "1b39a9702116ba9020d928bf681c49ee4b462fcc8e3209fbf256459f97e6ef84", "round": 50, "hasScResults": true, "sender": "erd1ure7ea247clj6yqjg80unz6xzjhlj2zwm4gtg6sudcmtsd2cw3xs74hasv", diff --git a/integrationtests/testdata/nftTransferCrossShardWithScCall/cross-shard-transfer-with-sc-call.json b/integrationtests/testdata/nftTransferCrossShardWithScCall/cross-shard-transfer-with-sc-call.json index e2e13d6c..07f16691 100644 --- a/integrationtests/testdata/nftTransferCrossShardWithScCall/cross-shard-transfer-with-sc-call.json +++ b/integrationtests/testdata/nftTransferCrossShardWithScCall/cross-shard-transfer-with-sc-call.json @@ -1,5 +1,5 @@ { - "miniBlockHash": "99a07aab4f6722a1473b33bd7bb35e339c69339c400737b14a94ad8bceaa1734", + "miniBlockHash": "60cb6c0185ada3fd5814611a0bc3bd63b47b4bffc78ecbce43adb4f32a19eac6", "nonce": 79, "round": 50, "value": "0", diff --git a/integrationtests/testdata/relayedTx/relayed-tx-after-refund.json b/integrationtests/testdata/relayedTx/relayed-tx-after-refund.json index a01337f9..dd85dc43 100644 --- a/integrationtests/testdata/relayedTx/relayed-tx-after-refund.json +++ b/integrationtests/testdata/relayedTx/relayed-tx-after-refund.json @@ -1,5 +1,5 @@ { - "miniBlockHash": "fed7c174a849c30b88c36a26453407f1b95970941d0872e603e641c5c804104a", + "miniBlockHash": "333145179043d02bb47bacd7a1af58640fcffae7be284cb30b2295b41159bf04", "nonce": 1196667, "round": 50, "value": "0", diff --git a/integrationtests/testdata/relayedTx/relayed-tx-intra.json b/integrationtests/testdata/relayedTx/relayed-tx-intra.json index 6b77adc6..cf066e85 100644 --- a/integrationtests/testdata/relayedTx/relayed-tx-intra.json +++ b/integrationtests/testdata/relayedTx/relayed-tx-intra.json @@ -1,5 +1,5 @@ { - "miniBlockHash": "2709174224d13e49fd76a70b48bd3db7838ca715bcfe09be59cef043241d7ef3", + "miniBlockHash": "be053e3ccb556eb39be6f388fb0ab5256d394d634887be06a6ed996560cbcae7", "nonce": 1196665, "round": 50, "value": "0", diff --git a/integrationtests/testdata/relayedTx/relayed-tx-source.json b/integrationtests/testdata/relayedTx/relayed-tx-source.json index c644e3e1..9c907820 100644 --- a/integrationtests/testdata/relayedTx/relayed-tx-source.json +++ b/integrationtests/testdata/relayedTx/relayed-tx-source.json @@ -1,5 +1,5 @@ { - "miniBlockHash": "fed7c174a849c30b88c36a26453407f1b95970941d0872e603e641c5c804104a", + "miniBlockHash": "333145179043d02bb47bacd7a1af58640fcffae7be284cb30b2295b41159bf04", "nonce": 1196667, "round": 50, "value": "0", diff --git a/integrationtests/testdata/scCallIntraShard/claim-rewards.json b/integrationtests/testdata/scCallIntraShard/claim-rewards.json index 2124bb2c..6de4372a 100644 --- a/integrationtests/testdata/scCallIntraShard/claim-rewards.json +++ b/integrationtests/testdata/scCallIntraShard/claim-rewards.json @@ -1,5 +1,5 @@ { - "miniBlockHash": "60b38b11110d28d1b361359f9688bb041bb9180219a612a83ff00dcc0db4d607", + "miniBlockHash": "b94c93965b411ffd412af1f52e3feab3b222d50e5f96fc69f9b6b8b53d926ac0", "nonce": 101, "round": 50, "value": "0", diff --git a/integrationtests/testdata/scCallIntraShard/sc-call-fail.json b/integrationtests/testdata/scCallIntraShard/sc-call-fail.json index 5e1cbab2..dbffafcb 100644 --- a/integrationtests/testdata/scCallIntraShard/sc-call-fail.json +++ b/integrationtests/testdata/scCallIntraShard/sc-call-fail.json @@ -1,5 +1,5 @@ { - "miniBlockHash": "5d04f80b044352bfbbde123702323eae07fdd8ca77f24f256079006058b6e7b4", + "miniBlockHash": "367a92162b8a241d2b30253520e93021c8dd2247bd30d8a21108feb2827db45e", "nonce": 46, "round": 50, "value": "5000000000000000000", diff --git a/integrationtests/testdata/transactions/move-balance.json b/integrationtests/testdata/transactions/move-balance.json index 4a2bb764..266cfe1c 100644 --- a/integrationtests/testdata/transactions/move-balance.json +++ b/integrationtests/testdata/transactions/move-balance.json @@ -1,5 +1,5 @@ { - "miniBlockHash": "24c374c9405540e88a36959ea83eede6ad50f6872f82d2e2a2280975615e1811", + "miniBlockHash": "ef17c864ce0ceca1f51171c79a50317d40dd2a71c016761332ed15ddce9b2cf6", "nonce": 1, "round": 50, "value": "1234", diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor.go b/process/elasticproc/logsevents/logsAndEventsProcessor.go index 4997f3ae..80034447 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor.go @@ -1,7 +1,6 @@ package logsevents import ( - "encoding/hex" "time" "github.com/multiversx/mx-chain-core-go/core" @@ -197,20 +196,19 @@ func (lep *logsAndEventsProcessor) PrepareLogsForDB( } func (lep *logsAndEventsProcessor) prepareLogsForDB( - id string, + logHashHex string, logHandler coreData.LogHandler, timestamp uint64, ) *data.Logs { - encodedID := hex.EncodeToString([]byte(id)) originalTxHash := "" - scr, ok := lep.logsData.scrsMap[encodedID] + scr, ok := lep.logsData.scrsMap[logHashHex] if ok { originalTxHash = scr.OriginalTxHash } events := logHandler.GetLogEvents() logsDB := &data.Logs{ - ID: encodedID, + ID: logHashHex, OriginalTxHash: originalTxHash, Address: lep.pubKeyConverter.Encode(logHandler.GetAddress()), Timestamp: time.Duration(timestamp), From 50d411005073aa5940b53ca0123c42bf7c4b3ea1 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 16 Mar 2023 10:42:54 +0200 Subject: [PATCH 023/189] small fix --- process/elasticproc/logsevents/logsAndEventsProcessor_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go index 3327677e..f99a0c2f 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go @@ -190,9 +190,9 @@ func TestLogsAndEventsProcessor_PrepareLogsForDB(t *testing.T) { t.Parallel() logsAndEvents := map[string]*transaction.Log{ - "wrong": nil, + hex.EncodeToString([]byte("wrong")): nil, - "txHash": { + hex.EncodeToString([]byte("txHash")): { Address: []byte("address"), Events: []*transaction.Event{ { From 5ed7b015d8553c054d4184e98b8b73930e687a69 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 16 Mar 2023 10:59:13 +0200 Subject: [PATCH 024/189] esdts index --- cmd/elasticindexer/config/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/elasticindexer/config/config.toml b/cmd/elasticindexer/config/config.toml index 713f0e00..7fff1c1f 100644 --- a/cmd/elasticindexer/config/config.toml +++ b/cmd/elasticindexer/config/config.toml @@ -2,7 +2,7 @@ available-indices = [ "rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", - "logs", "delegators", "operations" + "logs", "delegators", "operations", "esdts" ] [config.address-converter] length = 32 From 87b2eb6dd345769d6010939cab9fa02192e33d60 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 17 Mar 2023 12:56:36 +0200 Subject: [PATCH 025/189] extend interface --- process/dataindexer/dataIndexer.go | 6 +++--- process/dataindexer/dataIndexer_test.go | 1 - process/dataindexer/interface.go | 3 ++- process/factory/indexerFactory_test.go | 1 - 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/process/dataindexer/dataIndexer.go b/process/dataindexer/dataIndexer.go index e8930516..b9de6d4b 100644 --- a/process/dataindexer/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -138,9 +138,9 @@ func (di *dataIndexer) FinalizedBlock(_ *outport.FinalizedBlock) error { return nil } -// IsNilIndexer will return a bool value that signals if the indexer's implementation is a NilIndexer -func (di *dataIndexer) IsNilIndexer() bool { - return di.isNilIndexer +// GetMarshaller return the marshaller +func (di *dataIndexer) GetMarshaller() marshal.Marshalizer { + return di.headerMarshaller } // IsInterfaceNil returns true if there is no value under the interface diff --git a/process/dataindexer/dataIndexer_test.go b/process/dataindexer/dataIndexer_test.go index 30b73f05..46b04ab4 100644 --- a/process/dataindexer/dataIndexer_test.go +++ b/process/dataindexer/dataIndexer_test.go @@ -54,7 +54,6 @@ func TestDataIndexer_NewIndexerWithCorrectParamsShouldWork(t *testing.T) { require.Nil(t, err) require.False(t, check.IfNil(ei)) - require.False(t, ei.IsNilIndexer()) } func TestDataIndexer_SaveBlock(t *testing.T) { diff --git a/process/dataindexer/interface.go b/process/dataindexer/interface.go index f50d98a4..2cfd3188 100644 --- a/process/dataindexer/interface.go +++ b/process/dataindexer/interface.go @@ -6,6 +6,7 @@ import ( coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" ) @@ -61,9 +62,9 @@ type Indexer interface { SaveValidatorsRating(ratingData *outport.ValidatorsRating) error SaveAccounts(accountsData *outport.Accounts) error FinalizedBlock(finalizedBlock *outport.FinalizedBlock) error + GetMarshaller() marshal.Marshalizer Close() error IsInterfaceNil() bool - IsNilIndexer() bool } // BalanceConverter defines what a balance converter should be able to do diff --git a/process/factory/indexerFactory_test.go b/process/factory/indexerFactory_test.go index b4de4c94..3f153017 100644 --- a/process/factory/indexerFactory_test.go +++ b/process/factory/indexerFactory_test.go @@ -126,7 +126,6 @@ func TestIndexerFactoryCreate_ElasticIndexer(t *testing.T) { err = elasticIndexer.Close() require.NoError(t, err) - require.False(t, elasticIndexer.IsNilIndexer()) err = elasticIndexer.Close() require.NoError(t, err) From 20f684fd4f6019a2915e135f82ebae5b7bb4faf6 Mon Sep 17 00:00:00 2001 From: schimih Date: Mon, 20 Mar 2023 13:17:57 +0200 Subject: [PATCH 026/189] update mx-chain-core-go and vm-common-go versions --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index bfa0718b..bd7b19af 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,9 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gorilla/websocket v1.5.0 - github.com/multiversx/mx-chain-core-go v1.1.34-0.20230215164556-5ec8f51e96d3 + github.com/multiversx/mx-chain-core-go v1.2.0 github.com/multiversx/mx-chain-logger-go v1.0.11 - github.com/multiversx/mx-chain-vm-common-go v1.3.37-0.20230216122352-6dd6ff58ca2d + github.com/multiversx/mx-chain-vm-common-go v1.4.0 github.com/stretchr/testify v1.7.0 github.com/tidwall/gjson v1.14.0 github.com/urfave/cli v1.22.10 diff --git a/go.sum b/go.sum index bc7200d3..e1f7b242 100644 --- a/go.sum +++ b/go.sum @@ -70,12 +70,12 @@ github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.1.34-0.20230215164556-5ec8f51e96d3 h1:68ooIEnoUO69QGWXDa0VdDarOA4CaF9IiEMg9q/kisc= -github.com/multiversx/mx-chain-core-go v1.1.34-0.20230215164556-5ec8f51e96d3/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= +github.com/multiversx/mx-chain-core-go v1.2.0 h1:K539hKZKcHjBiFQpowFbA3BUd95Fe5+FLC+rKBOFZF0= +github.com/multiversx/mx-chain-core-go v1.2.0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= -github.com/multiversx/mx-chain-vm-common-go v1.3.37-0.20230216122352-6dd6ff58ca2d h1:n2qx7CceoqbEqZXWOsjLccS34zLPh5KNriIXTlNSIc4= -github.com/multiversx/mx-chain-vm-common-go v1.3.37-0.20230216122352-6dd6ff58ca2d/go.mod h1:Y9ggiJtjGCPK/8WEzFO91JLlppMa/uUMobMmsogeiRw= +github.com/multiversx/mx-chain-vm-common-go v1.4.0 h1:0i0cJZJOXGzqYzwtKFHSr2yGmnFAdizOuISK8HgsnYo= +github.com/multiversx/mx-chain-vm-common-go v1.4.0/go.mod h1:odBJC92ANA8zLtPh/wwajUUGJOaS88F5QYGf0t8Wgzw= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= From 41e5f00caf869a268a155b349d3f339bb19118ad Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 20 Mar 2023 16:22:22 +0200 Subject: [PATCH 027/189] modify python3 scripts --- scripts/observers/.env | 5 +- scripts/observers/config.py | 88 ++++++++++++++++++++++++------ scripts/observers/requirements.txt | 3 +- scripts/observers/start.py | 23 ++++++-- scripts/observers/stop.py | 1 + 5 files changed, 93 insertions(+), 27 deletions(-) diff --git a/scripts/observers/.env b/scripts/observers/.env index 95d2c92c..6fd3a970 100644 --- a/scripts/observers/.env +++ b/scripts/observers/.env @@ -1,8 +1,7 @@ -NODE_CONFIG_URL="https://github.com/multiversx/mx-chain-testnet-config" -NODE_CONFIG_BRANCH="" +MX_CHAIN_DEPLOY_GO_URL=https://github.com/multiversx/mx-chain-deploy-go NODE_GO_URL="https://github.com/multiversx/mx-chain-go" -NODE_GO_BRANCH="" +NODE_GO_BRANCH="MX-13968-refactor-outport-interface" WORKING_DIRECTORY="IndexerObservers" OBSERVER_DIR_PREFIX="observer_shard_" diff --git a/scripts/observers/config.py b/scripts/observers/config.py index dc23cf24..7d7cb89e 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -72,36 +72,89 @@ def prepare_observer(shard_id, working_dir, config_folder): update_toml_indexer(indexer_config, shard_id) +def generate_new_config(working_dir): + mx_chain_go_folder = working_dir / "mx-chain-go" / "scripts" / "testnet" + + with open(mx_chain_go_folder/"local.sh", "w") as file: + file.write("export SHARDCOUNT=3\n") + file.write("export SHARD_VALIDATORCOUNT=1\n") + file.write("export SHARD_OBSERVERCOUNT=0\n") + file.write("export SHARD_CONSENSUS_SIZE=1\n") + file.write("export META_VALIDATORCOUNT=1\n") + file.write("export META_OBSERVERCOUNT=0\n") + file.write("export META_CONSENSUS_SIZE=1\n") + file.write('export LOGLEVEL="*:DEBUG"\n') + file.write('export OBSERVERS_ANTIFLOOD_DISABLE=0\n') + file.write('export USETMUX=0\n') + file.write('export USE_PROXY=0\n') + + +def clone_mx_chain_go(working_dir): + print("cloning mx-chain-go....") + mx_chain_go_folder = working_dir / "mx-chain-go" + if not os.path.isdir(mx_chain_go_folder): + Repo.clone_from(os.getenv('NODE_GO_URL'), mx_chain_go_folder) + + repo_mx_chain_go = Repo(mx_chain_go_folder) + repo_mx_chain_go.git.checkout(os.getenv('NODE_GO_BRANCH')) + + +def clone_dependencies(working_dir): + print("cloning dependencies") + mx_chain_deploy_folder = working_dir / "mx-chain-deploy-go" + if not os.path.isdir(mx_chain_deploy_folder): + Repo.clone_from(os.getenv('MX_CHAIN_DEPLOY_GO_URL'), mx_chain_deploy_folder) + + +def prepare_seed_node(working_dir): + print("preparing seed node") + seed_node = Path.home() / "MultiversX/testnet/seednode" + shutil.copytree(seed_node, working_dir/"seednode") + + mx_chain_go_folder = working_dir / "mx-chain-go" + subprocess.check_call(["go", "build"], cwd=mx_chain_go_folder / "cmd/seednode") + + seed_node_exec = mx_chain_go_folder / "cmd/seednode/seednode" + shutil.copyfile(seed_node_exec, working_dir/"seednode/seednode") + + st = os.stat(working_dir/"seednode/seednode") + os.chmod(working_dir/"seednode/seednode", st.st_mode | stat.S_IEXEC) + + +def generate_config_for_local_testnet(working_dir): + mx_chain_local_testnet_scripts = working_dir / "mx-chain-go/scripts/testnet" + subprocess.check_call(["./clean.sh"], cwd=mx_chain_local_testnet_scripts) + subprocess.check_call(["./config.sh"], cwd=mx_chain_local_testnet_scripts) + + config_folder = Path.home() / "MultiversX/testnet/node/config" + os.rename(config_folder / "config_validator.toml", config_folder / "config.toml") + shutil.copytree(config_folder, working_dir/"config") + + def main(): load_dotenv() working_dir = get_working_dir() try: os.makedirs(working_dir) except FileExistsError: + print("something") print(f"working directory {working_dir} already exists") print("use `python3 clean.py` command first") sys.exit() - # CLONE config - print("cloning config....") - config_folder = working_dir / "config" - if not os.path.isdir(config_folder): - Repo.clone_from(os.getenv('NODE_CONFIG_URL'), config_folder) - - repo_cfg = Repo(config_folder) - repo_cfg.git.checkout(os.getenv('NODE_CONFIG_BRANCH')) - - # CLONE mx-chain-go - print("cloning mx-chain-go....") - mx_chain_go_folder = working_dir / "mx-chain-go" - if not os.path.isdir(mx_chain_go_folder): - Repo.clone_from(os.getenv('NODE_GO_URL'), mx_chain_go_folder) - - repo_mx_chain_go = Repo(mx_chain_go_folder) - repo_mx_chain_go.git.checkout(os.getenv('NODE_GO_BRANCH')) + # clone mx-chain-go + clone_mx_chain_go(working_dir) + # clone dependencies + clone_dependencies(working_dir) + # generate configs + generate_new_config(working_dir) + generate_config_for_local_testnet(working_dir) + # prepare seednode + prepare_seed_node(working_dir) # build binary mx-chain-go print("building node...") + mx_chain_go_folder = working_dir / "mx-chain-go" subprocess.check_call(["go", "build"], cwd=mx_chain_go_folder / "cmd/node") # build binary indexer @@ -109,6 +162,7 @@ def main(): subprocess.check_call(["go", "build"], cwd="../../cmd/elasticindexer") # prepare observers + config_folder = working_dir / "config" print("preparing config...") prepare_observer(0, working_dir, config_folder) prepare_observer(1, working_dir, config_folder) diff --git a/scripts/observers/requirements.txt b/scripts/observers/requirements.txt index 0899c299..45c66bdf 100644 --- a/scripts/observers/requirements.txt +++ b/scripts/observers/requirements.txt @@ -1 +1,2 @@ -git \ No newline at end of file +python-dotenv +GitPython \ No newline at end of file diff --git a/scripts/observers/start.py b/scripts/observers/start.py index e57ff4de..81f82941 100644 --- a/scripts/observers/start.py +++ b/scripts/observers/start.py @@ -2,14 +2,24 @@ from utils import * -def start_observer(shard_id, working_dir): +def start_seed_node(working_dir): + current_directory = os.getcwd() + working_dir_seed_node = working_dir/"seednode" + os.chdir(working_dir_seed_node) + command = "./seednode" + os.system("screen -d -m -S seednode" + " " + command) + + os.chdir(current_directory) + + +def start_observer(shard_id, working_dir, sk_index): current_observer = str(os.getenv('OBSERVER_DIR_PREFIX')) + str(shard_id) working_dir_observer = working_dir / current_observer current_directory = os.getcwd() # start observer os.chdir(working_dir_observer / "node") - command = "./node" + " --log-level *:DEBUG --no-key --log-save" + command = "./node" + " --log-level *:DEBUG --log-save --sk-index " + str(sk_index) os.system("screen -d -m -S obs" + str(shard_id) + " " + command) # start indexer @@ -29,10 +39,11 @@ def main(): print("staring observers and indexers....") - start_observer(0, working_dir) - start_observer(1, working_dir) - start_observer(2, working_dir) - start_observer(METACHAIN, working_dir) + start_seed_node(working_dir) + start_observer(0, working_dir, 0) + start_observer(1, working_dir, 1) + start_observer(2, working_dir, 2) + start_observer(METACHAIN, working_dir, 3) print("done") diff --git a/scripts/observers/stop.py b/scripts/observers/stop.py index b2d6362f..063f12ae 100644 --- a/scripts/observers/stop.py +++ b/scripts/observers/stop.py @@ -2,6 +2,7 @@ def main(): + os.system("screen -X -S seednode quit") os.system("screen -X -S obs0 quit") os.system("screen -X -S obs1 quit") os.system("screen -X -S obs2 quit") From f9b00a89cf8ca9d7bf716aa2a253e5bff5e011bd Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 21 Mar 2023 14:52:46 +0200 Subject: [PATCH 028/189] fixes after review --- go.mod | 2 +- go.sum | 8 +- .../dataindexer/workItems/workItemBlock.go | 7 +- process/elasticproc/block/blockProcessor.go | 39 +++++++- .../elasticproc/block/blockProcessor_test.go | 2 +- process/elasticproc/elasticProcessor_test.go | 96 +++++++------------ .../smartContractResultsProcessor.go | 13 ++- .../transactions/transactionDBBuilder.go | 20 +++- .../validators/validatorsProcessor.go | 2 +- .../validators/validatorsProcessor_test.go | 23 ++++- 10 files changed, 125 insertions(+), 87 deletions(-) diff --git a/go.mod b/go.mod index d0a08eb2..e10ef15e 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gorilla/websocket v1.5.0 - github.com/multiversx/mx-chain-core-go v1.1.35-0.20230314140225-cdd3af9ba37e + github.com/multiversx/mx-chain-core-go v1.1.36-0.20230320105458-1ac37b30d984 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.3.34 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index 1d07e500..f5ecb5cc 100644 --- a/go.sum +++ b/go.sum @@ -70,12 +70,8 @@ github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.1.35-0.20230306111403-9272fdab3455 h1:C96Si2yl6pn98SPvJM6cD33bCapT0JZ7NeUlPicljws= -github.com/multiversx/mx-chain-core-go v1.1.35-0.20230306111403-9272fdab3455/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.1.35-0.20230309140731-222b64e21ca6 h1:+J4X/BmNQPz7+/bhQ/mSss3Gl70GpwabZObFDUK39SE= -github.com/multiversx/mx-chain-core-go v1.1.35-0.20230309140731-222b64e21ca6/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.1.35-0.20230314140225-cdd3af9ba37e h1:B7Da37AgvQ1u82tjPv2L6EAhJ+PVPuU/9wq7v2aa0Xs= -github.com/multiversx/mx-chain-core-go v1.1.35-0.20230314140225-cdd3af9ba37e/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= +github.com/multiversx/mx-chain-core-go v1.1.36-0.20230320105458-1ac37b30d984 h1:vqjSw8MKp99Ku3r5rqm24JT9q9tQmZ+SFdhvB0t6eoI= +github.com/multiversx/mx-chain-core-go v1.1.36-0.20230320105458-1ac37b30d984/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= github.com/multiversx/mx-chain-vm-common-go v1.3.34 h1:6tSshEeVJpgFZmP8JV2wrPyRtL477oTYw1tjFcHTtv4= diff --git a/process/dataindexer/workItems/workItemBlock.go b/process/dataindexer/workItems/workItemBlock.go index eb34fb05..64bda3da 100644 --- a/process/dataindexer/workItems/workItemBlock.go +++ b/process/dataindexer/workItems/workItemBlock.go @@ -1,6 +1,7 @@ package workItems import ( + "encoding/hex" "fmt" "time" @@ -53,7 +54,7 @@ func (wib *itemBlock) Save() error { err := wib.indexer.SaveHeader(wib.outportBlockWithHeader) if err != nil { return fmt.Errorf("%w when saving header block, hash %s, nonce %d", - err, wib.outportBlockWithHeader.BlockData.HeaderHash, wib.outportBlockWithHeader.Header.GetNonce()) + err, hex.EncodeToString(wib.outportBlockWithHeader.BlockData.HeaderHash), wib.outportBlockWithHeader.Header.GetNonce()) } if len(wib.outportBlockWithHeader.BlockData.Body.MiniBlocks) == 0 { @@ -63,13 +64,13 @@ func (wib *itemBlock) Save() error { err = wib.indexer.SaveMiniblocks(wib.outportBlockWithHeader.Header, wib.outportBlockWithHeader.BlockData.Body) if err != nil { return fmt.Errorf("%w when saving miniblocks, block hash %s, nonce %d", - err, wib.outportBlockWithHeader.BlockData.HeaderHash, wib.outportBlockWithHeader.Header.GetNonce()) + err, hex.EncodeToString(wib.outportBlockWithHeader.BlockData.HeaderHash), wib.outportBlockWithHeader.Header.GetNonce()) } err = wib.indexer.SaveTransactions(wib.outportBlockWithHeader) if err != nil { return fmt.Errorf("%w when saving transactions, block hash %s, nonce %d", - err, wib.outportBlockWithHeader.BlockData.HeaderHash, wib.outportBlockWithHeader.Header.GetNonce()) + err, hex.EncodeToString(wib.outportBlockWithHeader.BlockData.HeaderHash), wib.outportBlockWithHeader.Header.GetNonce()) } return nil diff --git a/process/elasticproc/block/blockProcessor.go b/process/elasticproc/block/blockProcessor.go index 6aab62cc..cc91a8c7 100644 --- a/process/elasticproc/block/blockProcessor.go +++ b/process/elasticproc/block/blockProcessor.go @@ -2,6 +2,7 @@ package block import ( "encoding/hex" + "errors" "fmt" "strconv" "time" @@ -25,7 +26,11 @@ const ( notFound = -2 ) -var log = logger.GetOrCreate("indexer/process/block") +var ( + log = logger.GetOrCreate("indexer/process/block") + errNilBlockData = errors.New("nil block data") + errNilHeaderGasConsumed = errors.New("nil header gas consumed data") +) type blockProcessor struct { hasher hashing.Hasher @@ -52,18 +57,22 @@ func (bp *blockProcessor) PrepareBlockForDB(obh *outport.OutportBlockWithHeader) if check.IfNil(obh.Header) { return nil, indexer.ErrNilHeaderHandler } + if obh.BlockData == nil { + return nil, errNilBlockData + } if obh.BlockData.Body == nil { return nil, indexer.ErrNilBlockBody } + if obh.HeaderGasConsumption == nil { + return nil, errNilHeaderGasConsumed + } blockSizeInBytes, err := bp.computeBlockSize(obh.Header, obh.BlockData.Body) if err != nil { return nil, err } - // TODO compute size of transactions - sizeTxs := obh.OutportBlock.TransactionPool.Size() - + sizeTxs := computeSizeOfTransactions(obh.TransactionPool) miniblocksHashes := bp.getEncodedMBSHashes(obh.BlockData.Body) leaderIndex := bp.getLeaderIndex(obh.SignersIndexes) @@ -360,3 +369,25 @@ func hexEncodeSlice(slice [][]byte) []string { } return res } + +func computeSizeOfTransactions(pool *outport.TransactionPool) int { + if pool == nil { + return 0 + } + + txsSize := 0 + for _, txInfo := range pool.Transactions { + txsSize += txInfo.Transaction.Size() + } + for _, rewardInfo := range pool.Rewards { + txsSize += rewardInfo.Reward.Size() + } + for _, invalidTxInfo := range pool.InvalidTxs { + txsSize += invalidTxInfo.Transaction.Size() + } + for _, scrInfo := range pool.SmartContractResults { + txsSize += scrInfo.SmartContractResult.Size() + } + + return txsSize +} diff --git a/process/elasticproc/block/blockProcessor_test.go b/process/elasticproc/block/blockProcessor_test.go index f4f0daf6..efa3cf8e 100644 --- a/process/elasticproc/block/blockProcessor_test.go +++ b/process/elasticproc/block/blockProcessor_test.go @@ -434,7 +434,7 @@ func TestBlockProcessor_PrepareBlockForDBMiniBlocksDetails(t *testing.T) { require.Equal(t, &data.Block{ Hash: "68617368", Size: int64(723), - SizeTxs: 94, + SizeTxs: 0, AccumulatedFees: "0", DeveloperFees: "0", TxCount: uint32(5), diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index 0e401ee0..5c6385c5 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -47,6 +47,22 @@ func newElasticsearchProcessor(elasticsearchWriter DatabaseClientHandler, argume } } +func createEmptyOutportBlockWithHeader() *outport.OutportBlockWithHeader { + signerIndexes := []uint64{0, 1} + header := &dataBlock.Header{Nonce: 1} + return &outport.OutportBlockWithHeader{ + Header: header, + OutportBlock: &outport.OutportBlock{ + BlockData: &outport.BlockData{ + Body: &dataBlock.Body{}, + }, + SignersIndexes: signerIndexes, + HeaderGasConsumption: &outport.HeaderGasConsumption{}, + TransactionPool: &outport.TransactionPool{}, + }, + } +} + func createMockElasticProcessorArgs() *ArgElasticProcessor { balanceConverter, _ := converters.NewBalanceConverter(10) @@ -311,8 +327,7 @@ func TestElasticProcessor_RemoveMiniblocks(t *testing.T) { func TestElasticseachDatabaseSaveHeader_RequestError(t *testing.T) { localErr := errors.New("localErr") - header := &dataBlock.Header{Nonce: 1} - signerIndexes := []uint64{0, 1} + arguments := createMockElasticProcessorArgs() dbWriter := &mock.DatabaseWriterStub{ DoBulkRequestCalled: func(buff *bytes.Buffer, index string) error { @@ -322,24 +337,12 @@ func TestElasticseachDatabaseSaveHeader_RequestError(t *testing.T) { arguments.BlockProc, _ = block.NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) - err := elasticDatabase.SaveHeader(&outport.OutportBlockWithHeader{ - Header: header, - OutportBlock: &outport.OutportBlock{ - BlockData: &outport.BlockData{ - Body: &dataBlock.Body{}, - }, - SignersIndexes: signerIndexes, - HeaderGasConsumption: &outport.HeaderGasConsumption{}, - }, - }) + err := elasticDatabase.SaveHeader(createEmptyOutportBlockWithHeader()) require.Equal(t, localErr, err) } func TestElasticseachDatabaseSaveHeader_CheckRequestBody(t *testing.T) { - header := &dataBlock.Header{ - Nonce: 1, - } - signerIndexes := []uint64{0, 1} + outportBlock := createEmptyOutportBlockWithHeader() miniBlock := &dataBlock.MiniBlock{ Type: dataBlock.TxBlock, @@ -349,6 +352,7 @@ func TestElasticseachDatabaseSaveHeader_CheckRequestBody(t *testing.T) { miniBlock, }, } + outportBlock.BlockData.Body = blockBody arguments := createMockElasticProcessorArgs() @@ -362,9 +366,9 @@ func TestElasticseachDatabaseSaveHeader_CheckRequestBody(t *testing.T) { var bl data.Block blockBytes, _ := ioutil.ReadAll(req.Body) _ = json.Unmarshal(blockBytes, &bl) - require.Equal(t, header.Nonce, bl.Nonce) + require.Equal(t, outportBlock.Header.GetNonce(), bl.Nonce) require.Equal(t, hexEncodedHash, bl.MiniBlocksHashes[0]) - require.Equal(t, signerIndexes, bl.Validators) + require.Equal(t, outportBlock.SignersIndexes, bl.Validators) return nil }, @@ -372,16 +376,7 @@ func TestElasticseachDatabaseSaveHeader_CheckRequestBody(t *testing.T) { arguments.BlockProc, _ = block.NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) - err := elasticDatabase.SaveHeader(&outport.OutportBlockWithHeader{ - Header: header, - OutportBlock: &outport.OutportBlock{ - BlockData: &outport.BlockData{ - Body: blockBody, - }, - SignersIndexes: signerIndexes, - HeaderGasConsumption: &outport.HeaderGasConsumption{}, - }, - }) + err := elasticDatabase.SaveHeader(outportBlock) require.Nil(t, err) } @@ -407,21 +402,15 @@ func TestElasticseachSaveTransactions(t *testing.T) { txDbProc, _ := transactions.NewTransactionsProcessor(args) arguments.TransactionsProc = txDbProc + outportBlock := createEmptyOutportBlockWithHeader() + outportBlock.Header = header + outportBlock.BlockData.Body = body + outportBlock.TransactionPool.Transactions = map[string]*outport.TxInfo{ + hex.EncodeToString([]byte("tx1")): {Transaction: &transaction.Transaction{}, FeeInfo: &outport.FeeInfo{}}, + } + elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) - err := elasticDatabase.SaveTransactions(&outport.OutportBlockWithHeader{ - Header: header, - OutportBlock: &outport.OutportBlock{ - BlockData: &outport.BlockData{ - Body: body, - }, - HeaderGasConsumption: &outport.HeaderGasConsumption{}, - TransactionPool: &outport.TransactionPool{ - Transactions: map[string]*outport.TxInfo{ - hex.EncodeToString([]byte("tx1")): {Transaction: &transaction.Transaction{}, FeeInfo: &outport.FeeInfo{}}, - }, - }, - }, - }) + err := elasticDatabase.SaveTransactions(outportBlock) require.Equal(t, localErr, err) } @@ -616,18 +605,7 @@ func TestElasticProcessor_IndexEpochInfoData(t *testing.T) { err := elasticSearchProc.indexEpochInfoData(shardHeader, buffSlice) require.True(t, errors.Is(err, dataindexer.ErrHeaderTypeAssertion)) - body := &dataBlock.Body{} - metaHeader := &dataBlock.MetaBlock{} - - err = elasticSearchProc.SaveHeader(&outport.OutportBlockWithHeader{ - Header: metaHeader, - OutportBlock: &outport.OutportBlock{ - BlockData: &outport.BlockData{ - Body: body, - }, - HeaderGasConsumption: &outport.HeaderGasConsumption{}, - }, - }) + err = elasticSearchProc.SaveHeader(createEmptyOutportBlockWithHeader()) require.Nil(t, err) require.True(t, called) } @@ -657,15 +635,7 @@ func TestElasticProcessor_SaveTransactionNoDataShouldNotDoRequest(t *testing.T) elasticSearchProc := newElasticsearchProcessor(dbWriter, arguments) elasticSearchProc.enabledIndexes[dataindexer.ScResultsIndex] = struct{}{} - err := elasticSearchProc.SaveTransactions(&outport.OutportBlockWithHeader{ - Header: &dataBlock.Header{}, - OutportBlock: &outport.OutportBlock{ - BlockData: &outport.BlockData{ - Body: &dataBlock.Body{}, - }, - TransactionPool: &outport.TransactionPool{}, - }, - }) + err := elasticSearchProc.SaveTransactions(createEmptyOutportBlockWithHeader()) require.Nil(t, err) require.False(t, called) } diff --git a/process/elasticproc/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go index d6986a87..bcd70b9e 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -2,6 +2,7 @@ package transactions import ( "encoding/hex" + "math/big" "strconv" "time" @@ -149,6 +150,14 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( esdtValues = res.ESDTValues } + feeInfo := &outport.FeeInfo{ + Fee: big.NewInt(0), + InitialPaidFee: big.NewInt(0), + } + if scrInfo.FeeInfo != nil { + feeInfo = scrInfo.FeeInfo + } + return &indexerData.ScResult{ Hash: hex.EncodeToString(scrHash), MBHash: hexEncodedMBHash, @@ -181,8 +190,8 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( ReceiversShardIDs: res.ReceiversShardID, IsRelayed: res.IsRelayed, OriginalSender: originalSenderAddr, - InitialTxFee: scrInfo.FeeInfo.Fee.String(), - InitialTxGasUsed: scrInfo.FeeInfo.GasUsed, + InitialTxFee: feeInfo.Fee.String(), + InitialTxGasUsed: feeInfo.GasUsed, } } diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index 4ef1a835..a1a3dfab 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -3,6 +3,7 @@ package transactions import ( "encoding/hex" "fmt" + "math/big" "time" "github.com/multiversx/mx-chain-core-go/core" @@ -59,9 +60,18 @@ func (dtb *dbTransactionBuilder) prepareTransaction( log.Warn("dbTransactionBuilder.prepareTransaction: cannot compute value as num", "value", tx.Value, "hash", txHash, "error", err) } - feeNum, err := dtb.balanceConverter.ComputeESDTBalanceAsFloat(txInfo.FeeInfo.Fee) + + feeInfo := &outport.FeeInfo{ + Fee: big.NewInt(0), + InitialPaidFee: big.NewInt(0), + } + if txInfo.FeeInfo != nil { + feeInfo = txInfo.FeeInfo + } + + feeNum, err := dtb.balanceConverter.ComputeESDTBalanceAsFloat(feeInfo.Fee) if err != nil { - log.Warn("dbTransactionBuilder.prepareTransaction: cannot compute transaction fee as num", "fee", txInfo.FeeInfo.Fee, + log.Warn("dbTransactionBuilder.prepareTransaction: cannot compute transaction fee as num", "fee", feeInfo.Fee, "hash", txHash, "error", err) } esdtValuesNum, err := dtb.balanceConverter.ComputeSliceOfStringsAsFloat(res.ESDTValues) @@ -92,9 +102,9 @@ func (dtb *dbTransactionBuilder) prepareTransaction( Signature: hex.EncodeToString(tx.Signature), Timestamp: time.Duration(header.GetTimeStamp()), Status: txStatus, - GasUsed: txInfo.FeeInfo.GasUsed, - InitialPaidFee: txInfo.FeeInfo.InitialPaidFee.String(), - Fee: txInfo.FeeInfo.Fee.String(), + GasUsed: feeInfo.GasUsed, + InitialPaidFee: feeInfo.InitialPaidFee.String(), + Fee: feeInfo.Fee.String(), FeeNum: feeNum, ReceiverUserName: tx.RcvUserName, SenderUserName: tx.SndUserName, diff --git a/process/elasticproc/validators/validatorsProcessor.go b/process/elasticproc/validators/validatorsProcessor.go index e70ef815..6f85cfa7 100644 --- a/process/elasticproc/validators/validatorsProcessor.go +++ b/process/elasticproc/validators/validatorsProcessor.go @@ -29,7 +29,7 @@ func NewValidatorsProcessor(validatorPubkeyConverter core.PubkeyConverter, bulkS }, nil } -// PrepareAnSerializeValidatorsPubKeys will prepare validators public keys and serialize validators public keys +// PrepareAnSerializeValidatorsPubKeys will prepare validators public keys and serialize them func (vp *validatorsProcessor) PrepareAnSerializeValidatorsPubKeys(validatorsPubKeys *outport.ValidatorsPubKeys) ([]*bytes.Buffer, error) { buffSlice := data.NewBufferSlice(vp.bulkSizeMaxSize) diff --git a/process/elasticproc/validators/validatorsProcessor_test.go b/process/elasticproc/validators/validatorsProcessor_test.go index ef2de6b6..eaf89a59 100644 --- a/process/elasticproc/validators/validatorsProcessor_test.go +++ b/process/elasticproc/validators/validatorsProcessor_test.go @@ -3,6 +3,8 @@ package validators import ( "testing" + "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/stretchr/testify/require" ) @@ -15,4 +17,23 @@ func TestNewValidatorsProcessor(t *testing.T) { require.Equal(t, dataindexer.ErrNilPubkeyConverter, err) } -// TODO add unit tests +func TestValidatorsProcessor_PrepareAnSerializeValidatorsPubKeys(t *testing.T) { + t.Parallel() + + vp, err := NewValidatorsProcessor(&mock.PubkeyConverterMock{}, 0) + require.Nil(t, err) + + validators := &outport.ValidatorsPubKeys{ + Epoch: 30, + ShardValidatorsPubKeys: map[uint32]*outport.PubKeys{ + 0: {Keys: [][]byte{[]byte("k1"), []byte("k2")}}, + 1: {Keys: [][]byte{[]byte("k3"), []byte("k4")}}, + }, + } + res, err := vp.PrepareAnSerializeValidatorsPubKeys(validators) + require.Equal(t, `{ "index" : { "_id" : "0_30" } } +{"publicKeys":["6b31","6b32"]} +{ "index" : { "_id" : "1_30" } } +{"publicKeys":["6b33","6b34"]} +`, res[0].String()) +} From 62076807b905eaaeb282ecdd29108c426e055bea Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 21 Mar 2023 15:07:45 +0200 Subject: [PATCH 029/189] fixes after merge --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index ec4bf323..001137f1 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gorilla/websocket v1.5.0 - github.com/multiversx/mx-chain-core-go v1.1.36-0.20230320105458-1ac37b30d984 + github.com/multiversx/mx-chain-core-go v1.2.1-0.20230321130308-19c4c93dcae5 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.0 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index e1f7b242..dfba78bc 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,9 @@ github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.0 h1:K539hKZKcHjBiFQpowFbA3BUd95Fe5+FLC+rKBOFZF0= github.com/multiversx/mx-chain-core-go v1.2.0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230321130308-19c4c93dcae5 h1:jMxFHf7I5QdHknAzCADFerVZIinHwoAShgM2OaDWCHY= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230321130308-19c4c93dcae5/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= github.com/multiversx/mx-chain-vm-common-go v1.4.0 h1:0i0cJZJOXGzqYzwtKFHSr2yGmnFAdizOuISK8HgsnYo= From ac69eb89daf6f5cc28c9843412ae81603263684d Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 21 Mar 2023 15:12:36 +0200 Subject: [PATCH 030/189] fix linter issue --- process/elasticproc/validators/validatorsProcessor_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/process/elasticproc/validators/validatorsProcessor_test.go b/process/elasticproc/validators/validatorsProcessor_test.go index eaf89a59..0eb6dbeb 100644 --- a/process/elasticproc/validators/validatorsProcessor_test.go +++ b/process/elasticproc/validators/validatorsProcessor_test.go @@ -31,6 +31,7 @@ func TestValidatorsProcessor_PrepareAnSerializeValidatorsPubKeys(t *testing.T) { }, } res, err := vp.PrepareAnSerializeValidatorsPubKeys(validators) + require.Nil(t, err) require.Equal(t, `{ "index" : { "_id" : "0_30" } } {"publicKeys":["6b31","6b32"]} { "index" : { "_id" : "1_30" } } From a6148020e2399e5660aefa745b98527d11769722 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 21 Mar 2023 16:38:59 +0200 Subject: [PATCH 031/189] marshaller type --- scripts/observers/.env | 3 +++ scripts/observers/config.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/scripts/observers/.env b/scripts/observers/.env index 6fd3a970..6d1d9bc5 100644 --- a/scripts/observers/.env +++ b/scripts/observers/.env @@ -5,3 +5,6 @@ NODE_GO_BRANCH="MX-13968-refactor-outport-interface" WORKING_DIRECTORY="IndexerObservers" OBSERVER_DIR_PREFIX="observer_shard_" + +# marshaller types: `json` or `gogo proto` +WS_MARSHALLER_TYPE="gogo protobuf" diff --git a/scripts/observers/config.py b/scripts/observers/config.py index 7d7cb89e..b46da265 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -16,6 +16,8 @@ def update_toml_indexer(path, shard_id): prefs_data['config']['web-socket']['server-url'] = "localhost:" + str(WS_PORT_BASE + shard_id) else: prefs_data['config']['web-socket']['server-url'] = "localhost:" + str(WS_METACHAIN_PORT) + prefs_data['config']['web-socket']['data-marshaller-type'] = str(os.getenv('WS_MARSHALLER_TYPE')) + f = open(path_prefs, 'w') toml.dump(prefs_data, f) f.close() @@ -38,6 +40,8 @@ def update_toml_node(path, shard_id): external_data['WebSocketConnector']['URL'] = "localhost:" + str(WS_PORT_BASE + shard_id) else: external_data['WebSocketConnector']['URL'] = "localhost:" + str(WS_METACHAIN_PORT) + + external_data['WebSocketConnector']['MarshallerType'] = str(os.getenv('WS_MARSHALLER_TYPE')) f = open(path_external, 'w') toml.dump(external_data, f) f.close() From f37049f26a4018767545181608e8e590e8bd7f3a Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 22 Mar 2023 11:47:51 +0200 Subject: [PATCH 032/189] update mx-chain-core-go and fix unit tests --- go.mod | 2 +- go.sum | 4 +- process/elasticproc/elasticProcessor.go | 3 +- process/elasticproc/interface.go | 5 +- .../logsevents/logsAndEventsProcessor.go | 19 +-- .../logsevents/logsAndEventsProcessor_test.go | 148 ++++++++++-------- 6 files changed, 101 insertions(+), 80 deletions(-) diff --git a/go.mod b/go.mod index 001137f1..658897a9 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gorilla/websocket v1.5.0 - github.com/multiversx/mx-chain-core-go v1.2.1-0.20230321130308-19c4c93dcae5 + github.com/multiversx/mx-chain-core-go v1.2.1-0.20230322093158-35195fa155c0 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.0 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index dfba78bc..216c3679 100644 --- a/go.sum +++ b/go.sum @@ -71,8 +71,8 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230321130308-19c4c93dcae5 h1:jMxFHf7I5QdHknAzCADFerVZIinHwoAShgM2OaDWCHY= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230321130308-19c4c93dcae5/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230322093158-35195fa155c0 h1:Hr0bAMUJigh5xGFm2qoKKglEtpIXckxCLjCDBUYv1DM= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230322093158-35195fa155c0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= github.com/multiversx/mx-chain-vm-common-go v1.4.0 h1:0i0cJZJOXGzqYzwtKFHSr2yGmnFAdizOuISK8HgsnYo= diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index 8e845b6a..283b4cd9 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -11,7 +11,6 @@ import ( "github.com/multiversx/mx-chain-core-go/data/alteredAccount" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" - "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-es-indexer-go/data" elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" @@ -518,7 +517,7 @@ func (ei *elasticProcessor) indexTransactionsFeeData(txsHashFeeData map[string]* return ei.transactionsProc.SerializeTransactionsFeeData(txsHashFeeData, buffSlice, elasticIndexer.OperationsIndex) } -func (ei *elasticProcessor) prepareAndIndexLogs(logsAndEvents map[string]*transaction.Log, timestamp uint64, buffSlice *data.BufferSlice) error { +func (ei *elasticProcessor) prepareAndIndexLogs(logsAndEvents []*outport.LogData, timestamp uint64, buffSlice *data.BufferSlice) error { if !ei.isIndexEnabled(elasticIndexer.LogsIndex) { return nil } diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index beb641d4..eeb12cd4 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -8,7 +8,6 @@ import ( "github.com/multiversx/mx-chain-core-go/data/alteredAccount" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" - "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokeninfo" ) @@ -93,9 +92,9 @@ type DBValidatorsHandler interface { // DBLogsAndEventsHandler defines the actions that a logs and events handler should do type DBLogsAndEventsHandler interface { - PrepareLogsForDB(logsAndEvents map[string]*transaction.Log, timestamp uint64) []*data.Logs + PrepareLogsForDB(logsAndEvents []*outport.LogData, timestamp uint64) []*data.Logs ExtractDataFromLogs( - logsAndEvents map[string]*transaction.Log, + logsAndEvents []*outport.LogData, preparedResults *data.PreparedResults, timestamp uint64, shardID uint32, diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor.go b/process/elasticproc/logsevents/logsAndEventsProcessor.go index 5bac9874..2efbced0 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor.go @@ -6,6 +6,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" coreData "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" @@ -86,7 +87,7 @@ func createEventsProcessors(args ArgsLogsAndEventsProcessor) []eventsProcessor { // ExtractDataFromLogs will extract data from the provided logs and events and put in altered addresses func (lep *logsAndEventsProcessor) ExtractDataFromLogs( - logsAndEvents map[string]*transaction.Log, + logsAndEvents []*outport.LogData, preparedResults *data.PreparedResults, timestamp uint64, shardID uint32, @@ -94,20 +95,20 @@ func (lep *logsAndEventsProcessor) ExtractDataFromLogs( ) *data.PreparedLogsResults { lep.logsData = newLogsData(timestamp, preparedResults.Transactions, preparedResults.ScResults) - for txHashHexEncoded, txLog := range logsAndEvents { + for _, txLog := range logsAndEvents { if txLog == nil { continue } - events := txLog.Events - lep.processEvents(txHashHexEncoded, txLog.Address, events, shardID, numOfShards) + events := txLog.Log.Events + lep.processEvents(txLog.TxHash, txLog.Log.Address, events, shardID, numOfShards) - tx, ok := lep.logsData.txsMap[txHashHexEncoded] + tx, ok := lep.logsData.txsMap[txLog.TxHash] if ok { tx.HasLogs = true continue } - scr, ok := lep.logsData.scrsMap[txHashHexEncoded] + scr, ok := lep.logsData.scrsMap[txLog.TxHash] if ok { scr.HasLogs = true continue @@ -179,17 +180,17 @@ func (lep *logsAndEventsProcessor) processEvent(logHashHexEncoded string, logAdd // PrepareLogsForDB will prepare logs for database func (lep *logsAndEventsProcessor) PrepareLogsForDB( - logsAndEvents map[string]*transaction.Log, + logsAndEvents []*outport.LogData, timestamp uint64, ) []*data.Logs { logs := make([]*data.Logs, 0, len(logsAndEvents)) - for txHashHex, txLog := range logsAndEvents { + for _, txLog := range logsAndEvents { if txLog == nil { continue } - logs = append(logs, lep.prepareLogsForDB(txHashHex, txLog, timestamp)) + logs = append(logs, lep.prepareLogsForDB(txLog.TxHash, txLog.Log, timestamp)) } return logs diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go index f99a0c2f..99dca8e3 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/mock" @@ -57,66 +58,82 @@ func TestNewLogsAndEventsProcessor(t *testing.T) { func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) { t.Parallel() - logsAndEvents := map[string]*transaction.Log{ - hex.EncodeToString([]byte("wrong")): nil, - hex.EncodeToString([]byte("h3")): { - Events: []*transaction.Event{ - { - Address: []byte("addr"), - Identifier: []byte(core.SCDeployIdentifier), - Topics: [][]byte{[]byte("addr1"), []byte("addr2")}, + logsAndEvents := []*outport.LogData{ + nil, + { + TxHash: hex.EncodeToString([]byte("h3")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: []byte("addr"), + Identifier: []byte(core.SCDeployIdentifier), + Topics: [][]byte{[]byte("addr1"), []byte("addr2")}, + }, }, }, }, - - hex.EncodeToString([]byte("h1")): { - Address: []byte("address"), - Events: []*transaction.Event{ - { - Address: []byte("addr"), - Identifier: []byte(core.BuiltInFunctionESDTNFTTransfer), - Topics: [][]byte{[]byte("my-token"), big.NewInt(0).SetUint64(1).Bytes(), big.NewInt(100).Bytes(), []byte("receiver")}, + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Address: []byte("address"), + Events: []*transaction.Event{ + { + Address: []byte("addr"), + Identifier: []byte(core.BuiltInFunctionESDTNFTTransfer), + Topics: [][]byte{[]byte("my-token"), big.NewInt(0).SetUint64(1).Bytes(), big.NewInt(100).Bytes(), []byte("receiver")}, + }, }, }, }, - - hex.EncodeToString([]byte("h2")): { - Events: []*transaction.Event{ - { - Address: []byte("addr"), - Identifier: []byte(core.BuiltInFunctionESDTTransfer), - Topics: [][]byte{[]byte("esdt"), big.NewInt(0).Bytes(), big.NewInt(0).SetUint64(100).Bytes(), []byte("receiver")}, + { + TxHash: hex.EncodeToString([]byte("h2")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: []byte("addr"), + Identifier: []byte(core.BuiltInFunctionESDTTransfer), + Topics: [][]byte{[]byte("esdt"), big.NewInt(0).Bytes(), big.NewInt(0).SetUint64(100).Bytes(), []byte("receiver")}, + }, + nil, }, - nil, }, }, - hex.EncodeToString([]byte("h4")): { - Events: []*transaction.Event{ - { - Address: []byte("addr"), - Identifier: []byte(issueSemiFungibleESDTFunc), - Topics: [][]byte{[]byte("SEMI-abcd"), []byte("semi-token"), []byte("SEMI"), []byte(core.SemiFungibleESDT)}, + { + TxHash: hex.EncodeToString([]byte("h4")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: []byte("addr"), + Identifier: []byte(issueSemiFungibleESDTFunc), + Topics: [][]byte{[]byte("SEMI-abcd"), []byte("semi-token"), []byte("SEMI"), []byte(core.SemiFungibleESDT)}, + }, + nil, }, - nil, }, }, - hex.EncodeToString([]byte("h5")): { - Address: []byte("contract"), - Events: []*transaction.Event{ - { - Address: []byte("addr"), - Identifier: []byte(delegateFunc), - Topics: [][]byte{big.NewInt(1000).Bytes(), big.NewInt(1000000000).Bytes(), big.NewInt(10).Bytes(), big.NewInt(1000000000).Bytes()}, + { + TxHash: hex.EncodeToString([]byte("h5")), + Log: &transaction.Log{ + Address: []byte("contract"), + Events: []*transaction.Event{ + { + Address: []byte("addr"), + Identifier: []byte(delegateFunc), + Topics: [][]byte{big.NewInt(1000).Bytes(), big.NewInt(1000000000).Bytes(), big.NewInt(10).Bytes(), big.NewInt(1000000000).Bytes()}, + }, }, }, }, - hex.EncodeToString([]byte("h6")): { - Address: []byte("contract-second"), - Events: []*transaction.Event{ - { - Address: []byte("addr"), - Identifier: []byte(delegateFunc), - Topics: [][]byte{big.NewInt(1000).Bytes(), big.NewInt(1000000000).Bytes(), big.NewInt(10).Bytes(), big.NewInt(1000000000).Bytes()}, + { + TxHash: hex.EncodeToString([]byte("h6")), + Log: &transaction.Log{ + Address: []byte("contract-second"), + Events: []*transaction.Event{ + { + Address: []byte("addr"), + Identifier: []byte(delegateFunc), + Topics: [][]byte{big.NewInt(1000).Bytes(), big.NewInt(1000000000).Bytes(), big.NewInt(10).Bytes(), big.NewInt(1000000000).Bytes()}, + }, }, }, }, @@ -189,16 +206,18 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) func TestLogsAndEventsProcessor_PrepareLogsForDB(t *testing.T) { t.Parallel() - logsAndEvents := map[string]*transaction.Log{ - hex.EncodeToString([]byte("wrong")): nil, - - hex.EncodeToString([]byte("txHash")): { - Address: []byte("address"), - Events: []*transaction.Event{ - { - Address: []byte("addr"), - Identifier: []byte(core.BuiltInFunctionESDTNFTTransfer), - Topics: [][]byte{[]byte("my-token"), big.NewInt(0).SetUint64(1).Bytes(), []byte("receiver")}, + logsAndEvents := []*outport.LogData{ + nil, + { + TxHash: hex.EncodeToString([]byte("txHash")), + Log: &transaction.Log{ + Address: []byte("address"), + Events: []*transaction.Event{ + { + Address: []byte("addr"), + Identifier: []byte(core.BuiltInFunctionESDTNFTTransfer), + Topics: [][]byte{[]byte("my-token"), big.NewInt(0).SetUint64(1).Bytes(), []byte("receiver")}, + }, }, }, }, @@ -233,14 +252,17 @@ func TestLogsAndEventsProcessor_PrepareLogsForDB(t *testing.T) { func TestLogsAndEventsProcessor_ExtractDataFromLogsNFTBurn(t *testing.T) { t.Parallel() - logsAndEventsSlice := make(map[string]*transaction.Log, 1) - logsAndEventsSlice["h1"] = &transaction.Log{ - Address: []byte("address"), - Events: []*transaction.Event{ - { - Address: []byte("addr"), - Identifier: []byte(core.BuiltInFunctionESDTNFTBurn), - Topics: [][]byte{[]byte("MY-NFT"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes()}, + logsAndEventsSlice := make([]*outport.LogData, 1) + logsAndEventsSlice[0] = &outport.LogData{ + TxHash: "h1", + Log: &transaction.Log{ + Address: []byte("address"), + Events: []*transaction.Event{ + { + Address: []byte("addr"), + Identifier: []byte(core.BuiltInFunctionESDTNFTBurn), + Topics: [][]byte{[]byte("MY-NFT"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes()}, + }, }, }, } From a82089993c0d1d27792de40a003a1d1fe3c99d46 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 22 Mar 2023 11:57:13 +0200 Subject: [PATCH 033/189] fix integration tests --- .../accountsBalanceNftTransfer_test.go | 38 ++--- .../accountsBalanceWithLowerTimestamp_test.go | 38 ++--- integrationtests/accountsESDTRollback_test.go | 19 +-- .../accountsESDTWithTokenType_test.go | 76 +++++----- integrationtests/claimRewards_test.go | 17 ++- integrationtests/createNFTWithTags_test.go | 57 ++++---- integrationtests/delegators_test.go | 84 ++++++----- .../issueTokenAndSetRoles_test.go | 132 +++++++++-------- integrationtests/issueToken_test.go | 76 +++++----- integrationtests/logsCrossShard_test.go | 75 +++++----- ...sferWithScCallAndErrorSignaledBySC_test.go | 25 ++-- integrationtests/nftIssueCreateBurn_test.go | 59 ++++---- integrationtests/updateNFT_test.go | 133 ++++++++++-------- 13 files changed, 476 insertions(+), 353 deletions(-) diff --git a/integrationtests/accountsBalanceNftTransfer_test.go b/integrationtests/accountsBalanceNftTransfer_test.go index 23b0b342..752d3746 100644 --- a/integrationtests/accountsBalanceNftTransfer_test.go +++ b/integrationtests/accountsBalanceNftTransfer_test.go @@ -61,15 +61,18 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { } pool := &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(addr), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte("NFT-abcdef"), big.NewInt(7440483).Bytes(), big.NewInt(1).Bytes()}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(addr), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte("NFT-abcdef"), big.NewInt(7440483).Bytes(), big.NewInt(1).Bytes()}, + }, + nil, }, - nil, }, }, }, @@ -107,15 +110,18 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { } pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: []byte("test-address-balance-1"), - Identifier: []byte(core.BuiltInFunctionESDTNFTTransfer), - Topics: [][]byte{[]byte("NFT-abcdef"), big.NewInt(7440483).Bytes(), big.NewInt(1).Bytes(), decodeAddress(addrReceiver)}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: []byte("test-address-balance-1"), + Identifier: []byte(core.BuiltInFunctionESDTNFTTransfer), + Topics: [][]byte{[]byte("NFT-abcdef"), big.NewInt(7440483).Bytes(), big.NewInt(1).Bytes(), decodeAddress(addrReceiver)}, + }, + nil, }, - nil, }, }, }, diff --git a/integrationtests/accountsBalanceWithLowerTimestamp_test.go b/integrationtests/accountsBalanceWithLowerTimestamp_test.go index baef6736..1e984b3a 100644 --- a/integrationtests/accountsBalanceWithLowerTimestamp_test.go +++ b/integrationtests/accountsBalanceWithLowerTimestamp_test.go @@ -61,15 +61,18 @@ func TestIndexAccountsBalance(t *testing.T) { } pool := &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: []byte("eeeebbbb"), - Identifier: []byte(core.BuiltInFunctionESDTTransfer), - Topics: [][]byte{[]byte("TTTT-abcd"), nil, big.NewInt(1).Bytes()}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: []byte("eeeebbbb"), + Identifier: []byte(core.BuiltInFunctionESDTTransfer), + Topics: [][]byte{[]byte("TTTT-abcd"), nil, big.NewInt(1).Bytes()}, + }, + nil, }, - nil, }, }, }, @@ -134,15 +137,18 @@ func TestIndexAccountsBalance(t *testing.T) { FeeInfo: &outport.FeeInfo{}, }, }, - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(addr2), - Identifier: []byte(core.BuiltInFunctionESDTTransfer), - Topics: [][]byte{[]byte("TTTT-abcd"), nil, big.NewInt(1).Bytes()}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(addr2), + Identifier: []byte(core.BuiltInFunctionESDTTransfer), + Topics: [][]byte{[]byte("TTTT-abcd"), nil, big.NewInt(1).Bytes()}, + }, + nil, }, - nil, }, }, }, diff --git a/integrationtests/accountsESDTRollback_test.go b/integrationtests/accountsESDTRollback_test.go index 0717dec2..37d418cb 100644 --- a/integrationtests/accountsESDTRollback_test.go +++ b/integrationtests/accountsESDTRollback_test.go @@ -56,15 +56,18 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { // CREATE SEMI-FUNGIBLE TOKEN esdtDataBytes, _ := json.Marshal(esdtToken) pool := &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(addr), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte("TOKEN-eeee"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(addr), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte("TOKEN-eeee"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, + }, + nil, }, - nil, }, }, }, diff --git a/integrationtests/accountsESDTWithTokenType_test.go b/integrationtests/accountsESDTWithTokenType_test.go index b8f51136..25bf75ee 100644 --- a/integrationtests/accountsESDTWithTokenType_test.go +++ b/integrationtests/accountsESDTWithTokenType_test.go @@ -39,15 +39,18 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { address := "erd1sqy2ywvswp09ef7qwjhv8zwr9kzz3xas6y2ye5nuryaz0wcnfzzsnq0am3" pool := &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte("issueSemiFungible"), - Topics: [][]byte{[]byte("SEMI-abcd"), []byte("SEMI-token"), []byte("SEM"), []byte(core.SemiFungibleESDT)}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte("issueSemiFungible"), + Topics: [][]byte{[]byte("SEMI-abcd"), []byte("SEMI-token"), []byte("SEM"), []byte(core.SemiFungibleESDT)}, + }, + nil, }, - nil, }, }, }, @@ -97,15 +100,18 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { esdtDataBytes, _ := json.Marshal(esdtData) pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte("SEMI-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte("SEMI-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, + }, + nil, }, - nil, }, }, }, @@ -166,15 +172,18 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) esdtDataBytes, _ := json.Marshal(esdtData) pool := &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte("TTTT-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte("TTTT-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, + }, + nil, }, - nil, }, }, }, @@ -202,15 +211,18 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) require.Nil(t, err) pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte("issueSemiFungible"), - Topics: [][]byte{[]byte("TTTT-abcd"), []byte("TTTT-token"), []byte("SEM"), []byte(core.SemiFungibleESDT)}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte("issueSemiFungible"), + Topics: [][]byte{[]byte("TTTT-abcd"), []byte("TTTT-token"), []byte("SEM"), []byte(core.SemiFungibleESDT)}, + }, + nil, }, - nil, }, }, }, diff --git a/integrationtests/claimRewards_test.go b/integrationtests/claimRewards_test.go index bc27113c..853060aa 100644 --- a/integrationtests/claimRewards_test.go +++ b/integrationtests/claimRewards_test.go @@ -105,13 +105,16 @@ func TestTransactionWithClaimRewardsGasRefund(t *testing.T) { hex.EncodeToString(scrHash2): {SmartContractResult: scr2, FeeInfo: &outport.FeeInfo{}}, hex.EncodeToString(scrHash1): {SmartContractResult: scr1, FeeInfo: &outport.FeeInfo{}}, }, - Logs: map[string]*transaction.Log{ - hex.EncodeToString(txHash): { - Events: []*transaction.Event{ - { - Address: decodeAddress(addressSender), - Identifier: []byte("writeLog"), - Topics: [][]byte{[]byte("something")}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString(txHash), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(addressSender), + Identifier: []byte("writeLog"), + Topics: [][]byte{[]byte("something")}, + }, }, }, }, diff --git a/integrationtests/createNFTWithTags_test.go b/integrationtests/createNFTWithTags_test.go index 2b71508e..c3b7d579 100644 --- a/integrationtests/createNFTWithTags_test.go +++ b/integrationtests/createNFTWithTags_test.go @@ -48,15 +48,18 @@ func TestCreateNFTWithTags(t *testing.T) { // CREATE A FIRST NFT WITH THE TAGS address1 := "erd1v7e552pz9py4hv6raan0c4jflez3e6csdmzcgrncg0qrnk4tywvsqx0h5j" pool := &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte("DESK-abcd"), big.NewInt(1).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte("DESK-abcd"), big.NewInt(1).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, + }, + nil, }, - nil, }, }, }, @@ -113,15 +116,18 @@ func TestCreateNFTWithTags(t *testing.T) { // CREATE A SECOND NFT WITH THE SAME TAGS pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte("DESK-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte("DESK-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, + }, + nil, }, - nil, }, }, }, @@ -159,15 +165,18 @@ func TestCreateNFTWithTags(t *testing.T) { require.Nil(t, err) pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte("DESK-abcd"), big.NewInt(3).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte("DESK-abcd"), big.NewInt(3).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, + }, + nil, }, - nil, }, }, }, diff --git a/integrationtests/delegators_test.go b/integrationtests/delegators_test.go index c26883d9..d6c13e6e 100644 --- a/integrationtests/delegators_test.go +++ b/integrationtests/delegators_test.go @@ -41,16 +41,19 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { // delegate delegatedValue, _ := big.NewInt(0).SetString("200000000000000000000", 10) pool := &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Address: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqhllllsajxzat"), - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("delegate"), - Topics: [][]byte{delegatedValue.Bytes(), delegatedValue.Bytes(), big.NewInt(10).Bytes(), delegatedValue.Bytes()}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Address: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqhllllsajxzat"), + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("delegate"), + Topics: [][]byte{delegatedValue.Bytes(), delegatedValue.Bytes(), big.NewInt(10).Bytes(), delegatedValue.Bytes()}, + }, + nil, }, - nil, }, }, }, @@ -69,16 +72,19 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { unDelegatedValue, _ := big.NewInt(0).SetString("50000000000000000000", 10) totalDelegation, _ := big.NewInt(0).SetString("150000000000000000000", 10) pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h2")): { - Address: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqhllllsajxzat"), - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("unDelegate"), - Topics: [][]byte{unDelegatedValue.Bytes(), totalDelegation.Bytes(), big.NewInt(10).Bytes(), totalDelegation.Bytes(), []byte("1")}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h2")), + Log: &transaction.Log{ + Address: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqhllllsajxzat"), + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("unDelegate"), + Topics: [][]byte{unDelegatedValue.Bytes(), totalDelegation.Bytes(), big.NewInt(10).Bytes(), totalDelegation.Bytes(), []byte("1")}, + }, + nil, }, - nil, }, }, }, @@ -96,16 +102,19 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { unDelegatedValue, _ = big.NewInt(0).SetString("25500000000000000000", 10) totalDelegation, _ = big.NewInt(0).SetString("124500000000000000000", 10) pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h3")): { - Address: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqhllllsajxzat"), - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("unDelegate"), - Topics: [][]byte{unDelegatedValue.Bytes(), totalDelegation.Bytes(), big.NewInt(10).Bytes(), totalDelegation.Bytes(), []byte("2")}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h3")), + Log: &transaction.Log{ + Address: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqhllllsajxzat"), + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("unDelegate"), + Topics: [][]byte{unDelegatedValue.Bytes(), totalDelegation.Bytes(), big.NewInt(10).Bytes(), totalDelegation.Bytes(), []byte("2")}, + }, + nil, }, - nil, }, }, }, @@ -133,16 +142,19 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { // withdraw withdrawValue, _ := big.NewInt(0).SetString("725500000000000000000", 10) pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h4")): { - Address: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqhllllsajxzat"), - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("withdraw"), - Topics: [][]byte{withdrawValue.Bytes(), totalDelegation.Bytes(), big.NewInt(10).Bytes(), totalDelegation.Bytes(), []byte("false"), []byte("1"), []byte("2")}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h4")), + Log: &transaction.Log{ + Address: decodeAddress("erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqhllllsajxzat"), + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("withdraw"), + Topics: [][]byte{withdrawValue.Bytes(), totalDelegation.Bytes(), big.NewInt(10).Bytes(), totalDelegation.Bytes(), []byte("false"), []byte("1"), []byte("2")}, + }, + nil, }, - nil, }, }, }, diff --git a/integrationtests/issueTokenAndSetRoles_test.go b/integrationtests/issueTokenAndSetRoles_test.go index a598f9f4..09c6c6d7 100644 --- a/integrationtests/issueTokenAndSetRoles_test.go +++ b/integrationtests/issueTokenAndSetRoles_test.go @@ -35,20 +35,23 @@ func TestIssueTokenAndSetRole(t *testing.T) { address1 := "erd1k04pxr6c0gvlcx4rd5fje0a4uy33axqxwz0fpcrgtfdy3nrqauqqgvxprv" address2 := "erd1suhxyflu4w4pqdxmushpxzc6a3qszr89m8uswzqcvyh0mh9mzxwqdwkm0x" pool := &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("issueSemiFungible"), - Topics: [][]byte{[]byte("TOK-abcd"), []byte("semi-token"), []byte("SEMI"), []byte(core.SemiFungibleESDT)}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("issueSemiFungible"), + Topics: [][]byte{[]byte("TOK-abcd"), []byte("semi-token"), []byte("SEMI"), []byte(core.SemiFungibleESDT)}, + }, + { + Address: decodeAddress(address1), + Identifier: []byte("upgradeProperties"), + Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), []byte("canUpgrade"), []byte("true")}, + }, + nil, }, - { - Address: decodeAddress(address1), - Identifier: []byte("upgradeProperties"), - Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), []byte("canUpgrade"), []byte("true")}, - }, - nil, }, }, }, @@ -65,15 +68,18 @@ func TestIssueTokenAndSetRole(t *testing.T) { // SET ROLES pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionSetESDTRole), - Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte(core.ESDTRoleNFTCreate), []byte(core.ESDTRoleNFTBurn)}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionSetESDTRole), + Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte(core.ESDTRoleNFTCreate), []byte(core.ESDTRoleNFTBurn)}, + }, + nil, }, - nil, }, }, }, @@ -91,18 +97,21 @@ func TestIssueTokenAndSetRole(t *testing.T) { // TRANSFER ROLE pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreateRoleTransfer), - Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte("false")}, - }, - { - Address: decodeAddress(address2), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreateRoleTransfer), - Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte("true")}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreateRoleTransfer), + Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte("false")}, + }, + { + Address: decodeAddress(address2), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreateRoleTransfer), + Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte("true")}, + }, }, }, }, @@ -121,15 +130,18 @@ func TestIssueTokenAndSetRole(t *testing.T) { // UNSET ROLES pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionUnSetESDTRole), - Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte(core.ESDTRoleNFTBurn)}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionUnSetESDTRole), + Topics: [][]byte{[]byte("TOK-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte(core.ESDTRoleNFTBurn)}, + }, + nil, }, - nil, }, }, }, @@ -165,15 +177,18 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { address1 := "erd1k04pxr6c0gvlcx4rd5fje0a4uy33axqxwz0fpcrgtfdy3nrqauqqgvxprv" // SET ROLES pool := &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionSetESDTRole), - Topics: [][]byte{[]byte("TTT-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte(core.ESDTRoleNFTCreate), []byte(core.ESDTRoleNFTBurn)}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionSetESDTRole), + Topics: [][]byte{[]byte("TTT-abcd"), big.NewInt(0).Bytes(), big.NewInt(0).Bytes(), []byte(core.ESDTRoleNFTCreate), []byte(core.ESDTRoleNFTBurn)}, + }, + nil, }, - nil, }, }, }, @@ -191,15 +206,18 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { // ISSUE pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("issueSemiFungible"), - Topics: [][]byte{[]byte("TTT-abcd"), []byte("semi-token"), []byte("SEMI"), []byte(core.SemiFungibleESDT)}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("issueSemiFungible"), + Topics: [][]byte{[]byte("TTT-abcd"), []byte("semi-token"), []byte("SEMI"), []byte(core.SemiFungibleESDT)}, + }, + nil, }, - nil, }, }, }, diff --git a/integrationtests/issueToken_test.go b/integrationtests/issueToken_test.go index 2c321dde..6a453311 100644 --- a/integrationtests/issueToken_test.go +++ b/integrationtests/issueToken_test.go @@ -34,15 +34,18 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { address1 := "erd1v7e552pz9py4hv6raan0c4jflez3e6csdmzcgrncg0qrnk4tywvsqx0h5j" address2 := "erd1acjlnuhkd8773sqhmw85r0ur4lcyuqgm0n69h9ttxh0gwxtuuzxq4lckh6" pool := &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("issueSemiFungible"), - Topics: [][]byte{[]byte("SSSS-abcd"), []byte("semi-token"), []byte("SSSS"), []byte(core.SemiFungibleESDT), big.NewInt(18).Bytes()}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("issueSemiFungible"), + Topics: [][]byte{[]byte("SSSS-abcd"), []byte("semi-token"), []byte("SSSS"), []byte(core.SemiFungibleESDT), big.NewInt(18).Bytes()}, + }, + nil, }, - nil, }, }, }, @@ -63,15 +66,18 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { // transfer ownership pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("transferOwnership"), - Topics: [][]byte{[]byte("SSSS-abcd"), []byte("semi-token"), []byte("SSSS"), []byte(core.SemiFungibleESDT), decodeAddress(address2)}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("transferOwnership"), + Topics: [][]byte{[]byte("SSSS-abcd"), []byte("semi-token"), []byte("SSSS"), []byte(core.SemiFungibleESDT), decodeAddress(address2)}, + }, + nil, }, - nil, }, }, }, @@ -91,15 +97,18 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { // do pause pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("ESDTPause"), - Topics: [][]byte{[]byte("SSSS-abcd")}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("ESDTPause"), + Topics: [][]byte{[]byte("SSSS-abcd")}, + }, + nil, }, - nil, }, }, }, @@ -115,15 +124,18 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { // do unPause pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("ESDTUnPause"), - Topics: [][]byte{[]byte("SSSS-abcd")}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("ESDTUnPause"), + Topics: [][]byte{[]byte("SSSS-abcd")}, + }, + nil, }, - nil, }, }, }, diff --git a/integrationtests/logsCrossShard_test.go b/integrationtests/logsCrossShard_test.go index 496239d2..34f56e84 100644 --- a/integrationtests/logsCrossShard_test.go +++ b/integrationtests/logsCrossShard_test.go @@ -38,16 +38,19 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { // index on source pool := &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - logID: { - Address: decodeAddress(address1), - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionESDTTransfer), - Topics: [][]byte{[]byte("ESDT-abcd"), big.NewInt(0).Bytes(), big.NewInt(1).Bytes()}, + Logs: []*outport.LogData{ + { + TxHash: logID, + Log: &transaction.Log{ + Address: decodeAddress(address1), + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTTransfer), + Topics: [][]byte{[]byte("ESDT-abcd"), big.NewInt(0).Bytes(), big.NewInt(1).Bytes()}, + }, + nil, }, - nil, }, }, }, @@ -70,22 +73,25 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { TimeStamp: 6040, } pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - logID: { - Address: decodeAddress(address1), - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionESDTTransfer), - Topics: [][]byte{[]byte("ESDT-abcd"), big.NewInt(0).Bytes(), big.NewInt(1).Bytes()}, - }, - { - - Address: decodeAddress(address2), - Identifier: []byte("do-something"), - Topics: [][]byte{[]byte("topic1"), []byte("topic2")}, + Logs: []*outport.LogData{ + { + TxHash: logID, + Log: &transaction.Log{ + Address: decodeAddress(address1), + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTTransfer), + Topics: [][]byte{[]byte("ESDT-abcd"), big.NewInt(0).Bytes(), big.NewInt(1).Bytes()}, + }, + { + + Address: decodeAddress(address2), + Identifier: []byte("do-something"), + Topics: [][]byte{[]byte("topic1"), []byte("topic2")}, + }, + nil, }, - nil, }, }, }, @@ -106,16 +112,19 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { TimeStamp: 5000, } pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - logID: { - Address: decodeAddress(address1), - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionESDTTransfer), - Topics: [][]byte{[]byte("ESDT-abcd"), big.NewInt(0).Bytes(), big.NewInt(1).Bytes()}, + Logs: []*outport.LogData{ + { + TxHash: logID, + Log: &transaction.Log{ + Address: decodeAddress(address1), + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTTransfer), + Topics: [][]byte{[]byte("ESDT-abcd"), big.NewInt(0).Bytes(), big.NewInt(1).Bytes()}, + }, + nil, }, - nil, }, }, }, diff --git a/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go b/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go index 6fb7392f..5205533c 100644 --- a/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go +++ b/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go @@ -139,17 +139,20 @@ func TestMultiTransferCrossShardAndScCallErrorSignaledBySC(t *testing.T) { hex.EncodeToString(scrHash1): {SmartContractResult: scr1, FeeInfo: &outport.FeeInfo{}}, hex.EncodeToString(scrHash2): {SmartContractResult: scr2, FeeInfo: &outport.FeeInfo{}}, }, - Logs: map[string]*transaction.Log{ - hex.EncodeToString(scrHash1): { - Address: decodeAddress(address2), - Events: []*transaction.Event{ - { - Address: decodeAddress(address2), - Identifier: []byte("signalError"), - }, - { - Address: decodeAddress(address1), - Identifier: []byte("internalVMErrors"), + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString(scrHash1), + Log: &transaction.Log{ + Address: decodeAddress(address2), + Events: []*transaction.Event{ + { + Address: decodeAddress(address2), + Identifier: []byte("signalError"), + }, + { + Address: decodeAddress(address1), + Identifier: []byte("internalVMErrors"), + }, }, }, }, diff --git a/integrationtests/nftIssueCreateBurn_test.go b/integrationtests/nftIssueCreateBurn_test.go index 55ce802e..13ee8839 100644 --- a/integrationtests/nftIssueCreateBurn_test.go +++ b/integrationtests/nftIssueCreateBurn_test.go @@ -37,15 +37,18 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { address1 := "erd1ju8pkvg57cwdmjsjx58jlmnuf4l9yspstrhr9tgsrt98n9edpm2qtlgy99" pool := &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte("issueNonFungible"), - Topics: [][]byte{[]byte("NON-abcd"), []byte("NON-token"), []byte("NON"), []byte(core.NonFungibleESDT)}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte("issueNonFungible"), + Topics: [][]byte{[]byte("NON-abcd"), []byte("NON-token"), []byte("NON"), []byte(core.NonFungibleESDT)}, + }, + nil, }, - nil, }, }, }, @@ -78,16 +81,19 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { esdtDataBytes, _ := json.Marshal(esdtData) pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Address: decodeAddress(address1), - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte("NON-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Address: decodeAddress(address1), + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte("NON-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), esdtDataBytes}, + }, + nil, }, - nil, }, }, }, @@ -111,15 +117,18 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { } pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address1), - Identifier: []byte(core.BuiltInFunctionESDTNFTBurn), - Topics: [][]byte{[]byte("NON-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), decodeAddress(address1)}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.BuiltInFunctionESDTNFTBurn), + Topics: [][]byte{[]byte("NON-abcd"), big.NewInt(2).Bytes(), big.NewInt(1).Bytes(), decodeAddress(address1)}, + }, + nil, }, - nil, }, }, }, diff --git a/integrationtests/updateNFT_test.go b/integrationtests/updateNFT_test.go index 47cfc6e5..e6bfb226 100644 --- a/integrationtests/updateNFT_test.go +++ b/integrationtests/updateNFT_test.go @@ -43,15 +43,18 @@ func TestNFTUpdateMetadata(t *testing.T) { // CREATE NFT data address := "erd1w7jyzuj6cv4ngw8luhlkakatjpmjh3ql95lmxphd3vssc4vpymks6k5th7" pool := &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), - Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(1).Bytes(), marshalizedCreate}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte(core.BuiltInFunctionESDTNFTCreate), + Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(1).Bytes(), marshalizedCreate}, + }, + nil, }, - nil, }, }, }, @@ -67,15 +70,18 @@ func TestNFTUpdateMetadata(t *testing.T) { // Add URIS 1 pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte(core.BuiltInFunctionESDTNFTAddURI), - Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("uri1"), []byte("uri2")}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte(core.BuiltInFunctionESDTNFTAddURI), + Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("uri1"), []byte("uri2")}, + }, + nil, }, - nil, }, }, }, @@ -85,15 +91,18 @@ func TestNFTUpdateMetadata(t *testing.T) { // Add URIS 2 --- results should be the same pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte(core.BuiltInFunctionESDTNFTAddURI), - Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("uri1"), []byte("uri2")}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte(core.BuiltInFunctionESDTNFTAddURI), + Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("uri1"), []byte("uri2")}, + }, + nil, }, - nil, }, }, }, @@ -109,15 +118,18 @@ func TestNFTUpdateMetadata(t *testing.T) { require.JSONEq(t, readExpectedResult("./testdata/updateNFT/token-after-add-uris.json"), string(genericResponse.Docs[0].Source)) pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte(core.BuiltInFunctionESDTNFTUpdateAttributes), - Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("tags:test,free,fun;description:This is a test description for an awesome nft;metadata:metadata-test")}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte(core.BuiltInFunctionESDTNFTUpdateAttributes), + Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("tags:test,free,fun;description:This is a test description for an awesome nft;metadata:metadata-test")}, + }, + nil, }, - nil, }, }, }, @@ -134,15 +146,18 @@ func TestNFTUpdateMetadata(t *testing.T) { // Update attributes 2 pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte(core.BuiltInFunctionESDTNFTUpdateAttributes), - Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("something")}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte(core.BuiltInFunctionESDTNFTUpdateAttributes), + Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("something")}, + }, + nil, }, - nil, }, }, }, @@ -158,15 +173,18 @@ func TestNFTUpdateMetadata(t *testing.T) { // Freeze nft pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte(core.BuiltInFunctionESDTFreeze), - Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("something")}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte(core.BuiltInFunctionESDTFreeze), + Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("something")}, + }, + nil, }, - nil, }, }, }, @@ -181,15 +199,18 @@ func TestNFTUpdateMetadata(t *testing.T) { // UnFreeze nft pool = &outport.TransactionPool{ - Logs: map[string]*transaction.Log{ - hex.EncodeToString([]byte("h1")): { - Events: []*transaction.Event{ - { - Address: decodeAddress(address), - Identifier: []byte(core.BuiltInFunctionESDTUnFreeze), - Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("something")}, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Events: []*transaction.Event{ + { + Address: decodeAddress(address), + Identifier: []byte(core.BuiltInFunctionESDTUnFreeze), + Topics: [][]byte{[]byte("NFT-abcd"), big.NewInt(14).Bytes(), big.NewInt(0).Bytes(), []byte("something")}, + }, + nil, }, - nil, }, }, }, From 1822146c82d57eef31b9f89327ca0b856d70a0e9 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 28 Mar 2023 11:30:10 +0300 Subject: [PATCH 034/189] fixes after re review --- process/elasticproc/block/blockProcessor.go | 8 +- .../elasticproc/block/blockProcessor_test.go | 142 +++++++++--------- .../smartContractResultsProcessor.go | 7 +- 3 files changed, 77 insertions(+), 80 deletions(-) diff --git a/process/elasticproc/block/blockProcessor.go b/process/elasticproc/block/blockProcessor.go index cc91a8c7..26b60dd0 100644 --- a/process/elasticproc/block/blockProcessor.go +++ b/process/elasticproc/block/blockProcessor.go @@ -67,7 +67,7 @@ func (bp *blockProcessor) PrepareBlockForDB(obh *outport.OutportBlockWithHeader) return nil, errNilHeaderGasConsumed } - blockSizeInBytes, err := bp.computeBlockSize(obh.Header, obh.BlockData.Body) + blockSizeInBytes, err := bp.computeBlockSize(obh.BlockData.HeaderBytes, obh.BlockData.Body) if err != nil { return nil, err } @@ -311,11 +311,7 @@ func getExecutionOrderForTx(txHash []byte, mbType int32, pool *outport.Transacti return tx.GetExecutionOrder(), true } -func (bp *blockProcessor) computeBlockSize(header coreData.HeaderHandler, body *block.Body) (int, error) { - headerBytes, err := bp.marshalizer.Marshal(header) - if err != nil { - return 0, err - } +func (bp *blockProcessor) computeBlockSize(headerBytes []byte, body *block.Body) (int, error) { bodyBytes, err := bp.marshalizer.Marshal(body) if err != nil { return 0, err diff --git a/process/elasticproc/block/blockProcessor_test.go b/process/elasticproc/block/blockProcessor_test.go index efa3cf8e..48ac8b34 100644 --- a/process/elasticproc/block/blockProcessor_test.go +++ b/process/elasticproc/block/blockProcessor_test.go @@ -90,7 +90,7 @@ func TestBlockProcessor_PrepareBlockForDBShouldWork(t *testing.T) { SearchOrder: 0x3fc, MiniBlocksHashes: []string{"0796d34e8d443fd31bf4d9ec4051421b4d5d0e8c1db9ff942d6f4dc3a9ca2803", "4cc379ab1f0aef6602e85a0a7ffabb5bc9a2ba646dc0fd720028e06527bf873f"}, NotarizedBlocksHashes: []string(nil), - Size: 230, + Size: 114, AccumulatedFees: "0", DeveloperFees: "0", } @@ -154,17 +154,10 @@ func TestBlockProcessor_PrepareBlockForDBMarshalFailHeader(t *testing.T) { func TestBlockProcessor_PrepareBlockForDBMarshalFailBlock(t *testing.T) { t.Parallel() - count := 0 expectedErr := errors.New("local error") bp, _ := NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{ MarshalCalled: func(obj interface{}) ([]byte, error) { - defer func() { - count++ - }() - if count > 0 { - return nil, expectedErr - } - return nil, nil + return nil, expectedErr }, }) @@ -201,53 +194,57 @@ func TestBlockProcessor_PrepareBlockForDBEpochStartMeta(t *testing.T) { bp, _ := NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) - outportBlockWithHeader := &outport.OutportBlockWithHeader{ - Header: &dataBlock.MetaBlock{ - TxCount: 1000, - EpochStart: dataBlock.EpochStart{ - LastFinalizedHeaders: []dataBlock.EpochStartShardData{{ - ShardID: 1, - Nonce: 1234, - Round: 1500, - Epoch: 10, - HeaderHash: []byte("hh"), - RootHash: []byte("rh"), - ScheduledRootHash: []byte("sch"), - FirstPendingMetaBlock: []byte("fpmb"), - LastFinishedMetaBlock: []byte("lfmb"), - PendingMiniBlockHeaders: []dataBlock.MiniBlockHeader{ - { - Hash: []byte("mbh"), - SenderShardID: 0, - ReceiverShardID: 1, - Type: dataBlock.TxBlock, - Reserved: []byte("rrr"), - }, + header := &dataBlock.MetaBlock{ + TxCount: 1000, + EpochStart: dataBlock.EpochStart{ + LastFinalizedHeaders: []dataBlock.EpochStartShardData{{ + ShardID: 1, + Nonce: 1234, + Round: 1500, + Epoch: 10, + HeaderHash: []byte("hh"), + RootHash: []byte("rh"), + ScheduledRootHash: []byte("sch"), + FirstPendingMetaBlock: []byte("fpmb"), + LastFinishedMetaBlock: []byte("lfmb"), + PendingMiniBlockHeaders: []dataBlock.MiniBlockHeader{ + { + Hash: []byte("mbh"), + SenderShardID: 0, + ReceiverShardID: 1, + Type: dataBlock.TxBlock, + Reserved: []byte("rrr"), }, - }}, - Economics: dataBlock.Economics{ - TotalSupply: big.NewInt(100), - TotalToDistribute: big.NewInt(55), - TotalNewlyMinted: big.NewInt(20), - RewardsPerBlock: big.NewInt(15), - RewardsForProtocolSustainability: big.NewInt(2), - NodePrice: big.NewInt(10), - PrevEpochStartRound: 222, - PrevEpochStartHash: []byte("prevEpoch"), }, + }}, + Economics: dataBlock.Economics{ + TotalSupply: big.NewInt(100), + TotalToDistribute: big.NewInt(55), + TotalNewlyMinted: big.NewInt(20), + RewardsPerBlock: big.NewInt(15), + RewardsForProtocolSustainability: big.NewInt(2), + NodePrice: big.NewInt(10), + PrevEpochStartRound: 222, + PrevEpochStartHash: []byte("prevEpoch"), }, - MiniBlockHeaders: []dataBlock.MiniBlockHeader{ - { - TxCount: 50, - }, - { - TxCount: 120, - }, + }, + MiniBlockHeaders: []dataBlock.MiniBlockHeader{ + { + TxCount: 50, + }, + { + TxCount: 120, }, }, + } + + headerBytes, _ := bp.marshalizer.Marshal(header) + outportBlockWithHeader := &outport.OutportBlockWithHeader{ + Header: header, OutportBlock: &outport.OutportBlock{ BlockData: &outport.BlockData{ - HeaderHash: []byte("hash"), + HeaderBytes: headerBytes, + HeaderHash: []byte("hash"), Body: &dataBlock.Body{ MiniBlocks: []*dataBlock.MiniBlock{ {}, @@ -353,32 +350,35 @@ func TestBlockProcessor_PrepareBlockForDBMiniBlocksDetails(t *testing.T) { txHash, notExecutedTxHash, notFoundTxHash, invalidTxHash, rewardsTxHash, scrHash := "tx", "notExecuted", "notFound", "invalid", "reward", "scr" - outportBlockWithHeader := &outport.OutportBlockWithHeader{ - Header: &dataBlock.Header{ - TxCount: 5, - MiniBlockHeaders: []dataBlock.MiniBlockHeader{ - { - TxCount: 1, - Type: dataBlock.TxBlock, - Reserved: mbhrBytes, - }, - { - TxCount: 1, - Type: dataBlock.RewardsBlock, - }, - { - TxCount: 1, - Type: dataBlock.InvalidBlock, - }, - { - TxCount: 1, - Type: dataBlock.SmartContractResultBlock, - }, + header := &dataBlock.Header{ + TxCount: 5, + MiniBlockHeaders: []dataBlock.MiniBlockHeader{ + { + TxCount: 1, + Type: dataBlock.TxBlock, + Reserved: mbhrBytes, + }, + { + TxCount: 1, + Type: dataBlock.RewardsBlock, + }, + { + TxCount: 1, + Type: dataBlock.InvalidBlock, + }, + { + TxCount: 1, + Type: dataBlock.SmartContractResultBlock, }, }, + } + headerBytes, _ := bp.marshalizer.Marshal(header) + outportBlockWithHeader := &outport.OutportBlockWithHeader{ + Header: header, OutportBlock: &outport.OutportBlock{ BlockData: &outport.BlockData{ - HeaderHash: []byte("hash"), + HeaderBytes: headerBytes, + HeaderHash: []byte("hash"), Body: &dataBlock.Body{ MiniBlocks: []*dataBlock.MiniBlock{ { diff --git a/process/elasticproc/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go index a7ea19c3..432c84e9 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -84,10 +84,11 @@ func (proc *smartContractResultsProcessor) processSCRsFromMiniblock( indexerSCRs := make([]*indexerData.ScResult, 0, len(mb.TxHashes)) for _, scrHash := range mb.TxHashes { - scrInfo, ok := scrs[hex.EncodeToString(scrHash)] + scrHashHex := hex.EncodeToString(scrHash) + scrInfo, ok := scrs[scrHashHex] if !ok { log.Warn("smartContractResultsProcessor.processSCRsFromMiniblock scr not found in map", - "scr hash", hex.EncodeToString(scrHash), + "scr hash", scrHashHex, ) continue } @@ -95,7 +96,7 @@ func (proc *smartContractResultsProcessor) processSCRsFromMiniblock( indexerSCR := proc.prepareSmartContractResult(scrHash, mbHash, scrInfo, header, mb.SenderShardID, mb.ReceiverShardID, numOfShards) indexerSCRs = append(indexerSCRs, indexerSCR) - delete(scrs, hex.EncodeToString(scrHash)) + delete(scrs, scrHashHex) } return indexerSCRs From 2030ed993c34cc7c1a10abfe18ec0260781a25c6 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 28 Mar 2023 11:37:47 +0300 Subject: [PATCH 035/189] remove one element from map because the result is not consistent --- process/elasticproc/validators/validatorsProcessor_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/process/elasticproc/validators/validatorsProcessor_test.go b/process/elasticproc/validators/validatorsProcessor_test.go index 0eb6dbeb..3ae17216 100644 --- a/process/elasticproc/validators/validatorsProcessor_test.go +++ b/process/elasticproc/validators/validatorsProcessor_test.go @@ -27,14 +27,11 @@ func TestValidatorsProcessor_PrepareAnSerializeValidatorsPubKeys(t *testing.T) { Epoch: 30, ShardValidatorsPubKeys: map[uint32]*outport.PubKeys{ 0: {Keys: [][]byte{[]byte("k1"), []byte("k2")}}, - 1: {Keys: [][]byte{[]byte("k3"), []byte("k4")}}, }, } res, err := vp.PrepareAnSerializeValidatorsPubKeys(validators) require.Nil(t, err) require.Equal(t, `{ "index" : { "_id" : "0_30" } } {"publicKeys":["6b31","6b32"]} -{ "index" : { "_id" : "1_30" } } -{"publicKeys":["6b33","6b34"]} `, res[0].String()) } From 7918429f4e899fbe35dc16a7aea9a8d747254081 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 28 Mar 2023 16:36:36 +0300 Subject: [PATCH 036/189] fix after re re review --- process/elasticproc/block/blockProcessor_test.go | 11 +++++++++-- process/elasticproc/logsevents/nftsProcessor.go | 5 ++++- process/elasticproc/transactions/interface.go | 9 ++++++++- .../transactions/smartContractResultsProcessor.go | 10 +--------- .../transactions/transactionDBBuilder.go | 10 +--------- .../transactions/transactionsProcessor.go | 13 +++++++++++++ .../validators/validatorsProcessor_test.go | 1 + 7 files changed, 37 insertions(+), 22 deletions(-) diff --git a/process/elasticproc/block/blockProcessor_test.go b/process/elasticproc/block/blockProcessor_test.go index 48ac8b34..b162aaf4 100644 --- a/process/elasticproc/block/blockProcessor_test.go +++ b/process/elasticproc/block/blockProcessor_test.go @@ -9,6 +9,9 @@ import ( "github.com/multiversx/mx-chain-core-go/core" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-core-go/data/rewardTx" + "github.com/multiversx/mx-chain-core-go/data/smartContractResult" + "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-es-indexer-go/data" @@ -403,6 +406,7 @@ func TestBlockProcessor_PrepareBlockForDBMiniBlocksDetails(t *testing.T) { TransactionPool: &outport.TransactionPool{ Transactions: map[string]*outport.TxInfo{ hex.EncodeToString([]byte(txHash)): { + Transaction: &transaction.Transaction{}, ExecutionOrder: 2, }, hex.EncodeToString([]byte(notExecutedTxHash)): { @@ -411,16 +415,19 @@ func TestBlockProcessor_PrepareBlockForDBMiniBlocksDetails(t *testing.T) { }, Rewards: map[string]*outport.RewardInfo{ hex.EncodeToString([]byte(rewardsTxHash)): { + Reward: &rewardTx.RewardTx{}, ExecutionOrder: 3, }, }, InvalidTxs: map[string]*outport.TxInfo{ hex.EncodeToString([]byte(invalidTxHash)): { + Transaction: &transaction.Transaction{}, ExecutionOrder: 1, }}, SmartContractResults: map[string]*outport.SCRInfo{ hex.EncodeToString([]byte(scrHash)): { - ExecutionOrder: 0, + SmartContractResult: &smartContractResult.SmartContractResult{}, + ExecutionOrder: 0, }, }, }, @@ -434,7 +441,7 @@ func TestBlockProcessor_PrepareBlockForDBMiniBlocksDetails(t *testing.T) { require.Equal(t, &data.Block{ Hash: "68617368", Size: int64(723), - SizeTxs: 0, + SizeTxs: 15, AccumulatedFees: "0", DeveloperFees: "0", TxCount: uint32(5), diff --git a/process/elasticproc/logsevents/nftsProcessor.go b/process/elasticproc/logsevents/nftsProcessor.go index 2a93ec7c..95716f15 100644 --- a/process/elasticproc/logsevents/nftsProcessor.go +++ b/process/elasticproc/logsevents/nftsProcessor.go @@ -158,8 +158,11 @@ func (np *nftsProcessor) convertMetaData(metaData *esdt.MetaData) *alteredAccoun if metaData == nil { return nil } + encodedCreatorAddr, err := np.pubKeyConverter.Encode(metaData.Creator) + if err != nil { + log.Warn("nftsProcessor.convertMetaData", "cannot encode creator address", "error", err, "address", metaData.Creator) + } - encodedCreatorAddr := np.pubKeyConverter.SilentEncode(metaData.Creator, log) return &alteredAccount.TokenMetaData{ Nonce: metaData.Nonce, Name: string(metaData.Name), diff --git a/process/elasticproc/transactions/interface.go b/process/elasticproc/transactions/interface.go index 9cf02bb7..4a201d14 100644 --- a/process/elasticproc/transactions/interface.go +++ b/process/elasticproc/transactions/interface.go @@ -1,8 +1,15 @@ package transactions -import datafield "github.com/multiversx/mx-chain-vm-common-go/parsers/dataField" +import ( + "github.com/multiversx/mx-chain-core-go/data/outport" + datafield "github.com/multiversx/mx-chain-vm-common-go/parsers/dataField" +) // DataFieldParser defines what a data field parser should be able to do type DataFieldParser interface { Parse(dataField []byte, sender, receiver []byte, numOfShards uint32) *datafield.ResponseParseData } + +type feeInfoHandler interface { + GetFeeInfo() *outport.FeeInfo +} diff --git a/process/elasticproc/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go index 432c84e9..bde9774b 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -2,7 +2,6 @@ package transactions import ( "encoding/hex" - "math/big" "strconv" "time" @@ -154,14 +153,7 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( esdtValues = res.ESDTValues } - feeInfo := &outport.FeeInfo{ - Fee: big.NewInt(0), - InitialPaidFee: big.NewInt(0), - } - if scrInfo.FeeInfo != nil { - feeInfo = scrInfo.FeeInfo - } - + feeInfo := getFeeInfo(scrInfo) return &indexerData.ScResult{ Hash: hex.EncodeToString(scrHash), MBHash: hexEncodedMBHash, diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index 09deea1e..7b903de7 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -3,7 +3,6 @@ package transactions import ( "encoding/hex" "fmt" - "math/big" "time" "github.com/multiversx/mx-chain-core-go/core" @@ -64,14 +63,7 @@ func (dtb *dbTransactionBuilder) prepareTransaction( "hash", txHash, "error", err) } - feeInfo := &outport.FeeInfo{ - Fee: big.NewInt(0), - InitialPaidFee: big.NewInt(0), - } - if txInfo.FeeInfo != nil { - feeInfo = txInfo.FeeInfo - } - + feeInfo := getFeeInfo(txInfo) feeNum, err := dtb.balanceConverter.ComputeESDTBalanceAsFloat(feeInfo.Fee) if err != nil { log.Warn("dbTransactionBuilder.prepareTransaction: cannot compute transaction fee as num", "fee", feeInfo.Fee, diff --git a/process/elasticproc/transactions/transactionsProcessor.go b/process/elasticproc/transactions/transactionsProcessor.go index 7b6386ea..d8f089e7 100644 --- a/process/elasticproc/transactions/transactionsProcessor.go +++ b/process/elasticproc/transactions/transactionsProcessor.go @@ -2,6 +2,7 @@ package transactions import ( "encoding/hex" + "math/big" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" @@ -208,3 +209,15 @@ func mergeTxsMaps(dst, src map[string]*data.Transaction) { dst[key] = value } } + +func getFeeInfo(txWithFeeInfo feeInfoHandler) *outport.FeeInfo { + feeInfo := txWithFeeInfo.GetFeeInfo() + if feeInfo != nil { + return feeInfo + } + + return &outport.FeeInfo{ + Fee: big.NewInt(0), + InitialPaidFee: big.NewInt(0), + } +} diff --git a/process/elasticproc/validators/validatorsProcessor_test.go b/process/elasticproc/validators/validatorsProcessor_test.go index 3ae17216..adb8154b 100644 --- a/process/elasticproc/validators/validatorsProcessor_test.go +++ b/process/elasticproc/validators/validatorsProcessor_test.go @@ -31,6 +31,7 @@ func TestValidatorsProcessor_PrepareAnSerializeValidatorsPubKeys(t *testing.T) { } res, err := vp.PrepareAnSerializeValidatorsPubKeys(validators) require.Nil(t, err) + require.Len(t, res, 1) require.Equal(t, `{ "index" : { "_id" : "0_30" } } {"publicKeys":["6b31","6b32"]} `, res[0].String()) From bd290ebb9bf1b7914362441be1de6e323ecf2c56 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 29 Mar 2023 11:09:02 +0300 Subject: [PATCH 037/189] integrate new header creator --- go.mod | 2 +- go.sum | 2 ++ process/dataindexer/dataIndexer.go | 41 +++++++++++++++++++++++++++--- process/dataindexer/interface.go | 5 ++++ 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 658897a9..0fec510d 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gorilla/websocket v1.5.0 - github.com/multiversx/mx-chain-core-go v1.2.1-0.20230322093158-35195fa155c0 + github.com/multiversx/mx-chain-core-go v1.2.1-0.20230328143807-34b385f92114 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.0 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index 216c3679..4d47e63d 100644 --- a/go.sum +++ b/go.sum @@ -73,6 +73,8 @@ github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZ github.com/multiversx/mx-chain-core-go v1.2.0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1-0.20230322093158-35195fa155c0 h1:Hr0bAMUJigh5xGFm2qoKKglEtpIXckxCLjCDBUYv1DM= github.com/multiversx/mx-chain-core-go v1.2.1-0.20230322093158-35195fa155c0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230328143807-34b385f92114 h1:aTZm+1TlgEGn6ba6yCI2hVspNZ/BhA+7rv5l7uuw8eY= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230328143807-34b385f92114/go.mod h1:/lovncjwo+pXQ7IAERwNzwCifeH7SAWk0DGqjorX2bc= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= github.com/multiversx/mx-chain-vm-common-go v1.4.0 h1:0i0cJZJOXGzqYzwtKFHSr2yGmnFAdizOuISK8HgsnYo= diff --git a/process/dataindexer/dataIndexer.go b/process/dataindexer/dataIndexer.go index b9de6d4b..5634dac4 100644 --- a/process/dataindexer/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -3,7 +3,8 @@ package dataindexer import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" - "github.com/multiversx/mx-chain-core-go/core/unmarshal" + "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" @@ -21,6 +22,7 @@ type dataIndexer struct { dispatcher DispatcherHandler elasticProcessor ElasticProcessor headerMarshaller marshal.Marshalizer + blockContainer blockContainerHandler } // NewDataIndexer will create a new data indexer @@ -30,16 +32,40 @@ func NewDataIndexer(arguments ArgDataIndexer) (*dataIndexer, error) { return nil, err } + blockContainer, err := createBlockCreatorsContainer() + if err != nil { + return nil, err + } + dataIndexerObj := &dataIndexer{ isNilIndexer: false, dispatcher: arguments.DataDispatcher, elasticProcessor: arguments.ElasticProcessor, headerMarshaller: arguments.HeaderMarshaller, + blockContainer: blockContainer, } return dataIndexerObj, nil } +func createBlockCreatorsContainer() (blockContainerHandler, error) { + container := block.NewEmptyBlockCreatorsContainer() + err := container.Add(core.ShardHeaderV1, block.NewEmptyHeaderCreator()) + if err != nil { + return nil, err + } + err = container.Add(core.ShardHeaderV2, block.NewEmptyHeaderV2Creator()) + if err != nil { + return nil, err + } + err = container.Add(core.MetaHeader, block.NewEmptyMetaBlockCreator()) + if err != nil { + return nil, err + } + + return container, nil +} + func checkIndexerArgs(arguments ArgDataIndexer) error { if check.IfNil(arguments.DataDispatcher) { return ErrNilDataDispatcher @@ -54,9 +80,18 @@ func checkIndexerArgs(arguments ArgDataIndexer) error { return nil } +func (di *dataIndexer) getHeaderFromBytes(headerType core.HeaderType, headerBytes []byte) (header data.HeaderHandler, err error) { + creator, err := di.blockContainer.Get(headerType) + if err != nil { + return nil, err + } + + return block.GetHeaderFromBytes(di.headerMarshaller, creator, headerBytes) +} + // SaveBlock saves the block info in the queue to be sent to elastic func (di *dataIndexer) SaveBlock(outportBlock *outport.OutportBlock) error { - header, err := unmarshal.GetHeaderFromBytes(di.headerMarshaller, core.HeaderType(outportBlock.BlockData.HeaderType), outportBlock.BlockData.HeaderBytes) + header, err := di.getHeaderFromBytes(core.HeaderType(outportBlock.BlockData.HeaderType), outportBlock.BlockData.HeaderBytes) if err != nil { return err } @@ -80,7 +115,7 @@ func (di *dataIndexer) Close() error { // RevertIndexedBlock will remove from database block and miniblocks func (di *dataIndexer) RevertIndexedBlock(blockData *outport.BlockData) error { - header, err := unmarshal.GetHeaderFromBytes(di.headerMarshaller, core.HeaderType(blockData.HeaderType), blockData.HeaderBytes) + header, err := di.getHeaderFromBytes(core.HeaderType(blockData.HeaderType), blockData.HeaderBytes) if err != nil { return err } diff --git a/process/dataindexer/interface.go b/process/dataindexer/interface.go index 2cfd3188..dc026034 100644 --- a/process/dataindexer/interface.go +++ b/process/dataindexer/interface.go @@ -3,6 +3,7 @@ package dataindexer import ( "math/big" + "github.com/multiversx/mx-chain-core-go/core" coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" @@ -74,3 +75,7 @@ type BalanceConverter interface { ComputeSliceOfStringsAsFloat(values []string) ([]float64, error) IsInterfaceNil() bool } + +type blockContainerHandler interface { + Get(headerType core.HeaderType) (block.EmptyBlockCreator, error) +} From 616c3efeb55435200350f05fd886473f56b42155 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 29 Mar 2023 12:00:49 +0300 Subject: [PATCH 038/189] integrate new ws client --- cmd/elasticindexer/config/prefs.toml | 2 + config/config.go | 2 + factory/wsIndexerFactory.go | 9 +- go.mod | 2 +- go.sum | 2 + process/wsclient/client.go | 174 --------------------------- process/wsindexer/indexer.go | 28 ++++- 7 files changed, 39 insertions(+), 180 deletions(-) delete mode 100644 process/wsclient/client.go diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index 3dda19cb..d605456c 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -3,6 +3,8 @@ [config.web-socket] server-url = "localhost:22111" data-marshaller-type = "json" + retry-duration-in-seconds = 5 + blocking-ack-on-error = true [config.elastic-cluster] use-kibana = false diff --git a/config/config.go b/config/config.go index 40dead91..d3b1a9e5 100644 --- a/config/config.go +++ b/config/config.go @@ -38,6 +38,8 @@ type ClusterConfig struct { WebSocket struct { ServerURL string `toml:"server-url"` DataMarshallerType string `toml:"data-marshaller-type"` + RetryDurationInSec uint32 `toml:"retry-duration-in-seconds"` + BlockingAckOnError bool `toml:"blocking-ack-on-error"` } `toml:"web-socket"` ElasticCluster struct { UseKibana bool `toml:"use-kibana"` diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 6cea6b94..e4cc95fa 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -5,9 +5,9 @@ import ( factoryHasher "github.com/multiversx/mx-chain-core-go/hashing/factory" "github.com/multiversx/mx-chain-core-go/marshal" factoryMarshaller "github.com/multiversx/mx-chain-core-go/marshal/factory" + "github.com/multiversx/mx-chain-core-go/websocketOutportDriver/client" "github.com/multiversx/mx-chain-es-indexer-go/config" "github.com/multiversx/mx-chain-es-indexer-go/process/factory" - "github.com/multiversx/mx-chain-es-indexer-go/process/wsclient" "github.com/multiversx/mx-chain-es-indexer-go/process/wsindexer" ) @@ -32,7 +32,12 @@ func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsinde return nil, err } - return wsclient.New(clusterCfg.Config.WebSocket.ServerURL, indexer) + return client.CreateWsClient(client.ArgsCreateWsClient{ + Url: clusterCfg.Config.WebSocket.ServerURL, + RetryDurationInSec: clusterCfg.Config.WebSocket.RetryDurationInSec, + BlockingAckOnError: clusterCfg.Config.WebSocket.BlockingAckOnError, + PayloadProcessor: indexer, + }) } func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig, wsMarshaller marshal.Marshalizer) (wsindexer.DataIndexer, error) { diff --git a/go.mod b/go.mod index 0fec510d..76e3bf14 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gorilla/websocket v1.5.0 - github.com/multiversx/mx-chain-core-go v1.2.1-0.20230328143807-34b385f92114 + github.com/multiversx/mx-chain-core-go v1.2.1-0.20230329082847-b78e96c3ad5a github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.0 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index 4d47e63d..ef150001 100644 --- a/go.sum +++ b/go.sum @@ -75,6 +75,8 @@ github.com/multiversx/mx-chain-core-go v1.2.1-0.20230322093158-35195fa155c0 h1:H github.com/multiversx/mx-chain-core-go v1.2.1-0.20230322093158-35195fa155c0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1-0.20230328143807-34b385f92114 h1:aTZm+1TlgEGn6ba6yCI2hVspNZ/BhA+7rv5l7uuw8eY= github.com/multiversx/mx-chain-core-go v1.2.1-0.20230328143807-34b385f92114/go.mod h1:/lovncjwo+pXQ7IAERwNzwCifeH7SAWk0DGqjorX2bc= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230329082847-b78e96c3ad5a h1:cjPv/NIb4c3oBuBxxc2ggyaMvpmGlo1RO8mmzrkWARM= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230329082847-b78e96c3ad5a/go.mod h1:/lovncjwo+pXQ7IAERwNzwCifeH7SAWk0DGqjorX2bc= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= github.com/multiversx/mx-chain-vm-common-go v1.4.0 h1:0i0cJZJOXGzqYzwtKFHSr2yGmnFAdizOuISK8HgsnYo= diff --git a/process/wsclient/client.go b/process/wsclient/client.go deleted file mode 100644 index c1c3a105..00000000 --- a/process/wsclient/client.go +++ /dev/null @@ -1,174 +0,0 @@ -package wsclient - -import ( - "fmt" - "io" - "net/url" - "strings" - "time" - - "github.com/gorilla/websocket" - "github.com/multiversx/mx-chain-core-go/core/check" - "github.com/multiversx/mx-chain-core-go/data/typeConverters/uint64ByteSlice" - "github.com/multiversx/mx-chain-core-go/websocketOutportDriver" - "github.com/multiversx/mx-chain-core-go/websocketOutportDriver/data" - logger "github.com/multiversx/mx-chain-logger-go" -) - -const closedConnection = "use of closed network connection" - -type operationsHandler interface { - GetOperationsMap() map[data.OperationType]func(marshalledData []byte) error - Close() error -} - -type wsConn interface { - io.Closer - ReadMessage() (messageType int, p []byte, err error) - WriteMessage(messageType int, data []byte) error -} - -var ( - log = logger.GetOrCreate("process/wsclient") - retryDuration = time.Second * 5 -) - -type client struct { - urlReceive string - closeActions func() error - actions map[data.OperationType]func(marshalledData []byte) error - uint64ByteSliceConverter websocketOutportDriver.Uint64ByteSliceConverter - wsConnection wsConn -} - -// New will create a new instance of websocket client -func New( - urlReceive string, - operationsHandler operationsHandler, -) (*client, error) { - urlReceiveData := url.URL{Scheme: "ws", Host: urlReceive, Path: data.WSRoute} - - return &client{ - actions: operationsHandler.GetOperationsMap(), - closeActions: operationsHandler.Close, - urlReceive: urlReceiveData.String(), - uint64ByteSliceConverter: uint64ByteSlice.NewBigEndianConverter(), - }, nil -} - -// Start will initialize the connection to the server and start to listen for messages -func (c *client) Start() { - log.Info("connecting to", "url", c.urlReceive) - - for { - err := c.openConnection() - if err != nil { - log.Warn(fmt.Sprintf("c.openConnection(), retrying in %v...", retryDuration), "error", err.Error()) - time.Sleep(retryDuration) - continue - } - - closed := c.listeningOnWebSocket() - if closed { - return - } - } -} - -func (c *client) openConnection() error { - var err error - c.wsConnection, _, err = websocket.DefaultDialer.Dial(c.urlReceive, nil) - if err != nil { - return err - } - - return nil -} - -func (c *client) listeningOnWebSocket() (closed bool) { - for { - _, message, err := c.wsConnection.ReadMessage() - if err == nil { - c.verifyPayloadAndSendAckIfNeeded(message) - continue - } - - _, isConnectionClosed := err.(*websocket.CloseError) - if !isConnectionClosed { - if strings.Contains(err.Error(), closedConnection) { - return true - } - log.Warn("c.listeningOnWebSocket()-> connection problem, retrying", "error", err.Error()) - } else { - log.Warn(fmt.Sprintf("websocket terminated by the server side, retrying in %v...", retryDuration), "error", err.Error()) - } - return - } - -} - -func (c *client) verifyPayloadAndSendAckIfNeeded(payload []byte) { - if len(payload) == 0 { - log.Error("empty payload") - return - } - - payloadParser, _ := websocketOutportDriver.NewWebSocketPayloadParser(c.uint64ByteSliceConverter) - payloadData, err := payloadParser.ExtractPayloadData(payload) - if err != nil { - log.Error("error while extracting payload data: " + err.Error()) - return - } - - log.Info("processing payload", - "counter", payloadData.Counter, - "operation type", payloadData.OperationType, - "message length", len(payloadData.Payload), - ) - - function, ok := c.actions[payloadData.OperationType] - if !ok { - log.Warn("invalid operation", "operation type", payloadData.OperationType.String()) - } - - err = function(payloadData.Payload) - if err != nil { - log.Error("something went wrong", "error", err.Error()) - } - - if payloadData.WithAcknowledge { - counterBytes := c.uint64ByteSliceConverter.ToByteSlice(payloadData.Counter) - err = c.wsConnection.WriteMessage(websocket.BinaryMessage, counterBytes) - if err != nil { - log.Error("write acknowledge message", "error", err.Error()) - } - } -} - -func (c *client) closeWsConnection() { - log.Debug("closing ws connection...") - if check.IfNilReflect(c.wsConnection) { - return - } - - //Cleanly close the connection by sending a close message and then - //waiting (with timeout) for the server to close the connection. - err := c.wsConnection.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) - if err != nil { - log.Error("cannot send close message", "error", err) - } - err = c.wsConnection.Close() - if err != nil { - log.Error("cannot close ws connection", "error", err) - } -} - -func (c *client) Close() { - log.Info("closing all components...") - c.closeWsConnection() - - err := c.closeActions() - if err != nil { - log.Error("cannot close the operations handler", "error", err) - } -} diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index 82c0b47f..49b83b4d 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -19,6 +19,7 @@ var ( type indexer struct { marshaller marshal.Marshalizer di DataIndexer + actions map[data.OperationType]func(marshalledData []byte) error } // NewIndexer will create a new instance of *indexer @@ -30,14 +31,17 @@ func NewIndexer(marshaller marshal.Marshalizer, dataIndexer DataIndexer) (*index return nil, errNilDataIndexer } - return &indexer{ + i := &indexer{ marshaller: marshaller, di: dataIndexer, - }, nil + } + i.initActionsMap() + + return i, nil } // GetOperationsMap returns the map with all the operations that will index data -func (i *indexer) GetOperationsMap() map[data.OperationType]func(d []byte) error { +func (i *indexer) initActionsMap() map[data.OperationType]func(d []byte) error { return map[data.OperationType]func(d []byte) error{ data.OperationSaveBlock: i.saveBlock, data.OperationRevertIndexedBlock: i.revertIndexedBlock, @@ -49,6 +53,19 @@ func (i *indexer) GetOperationsMap() map[data.OperationType]func(d []byte) error } } +func (i *indexer) ProcessPayload(payload *data.PayloadData) error { + function, ok := i.actions[payload.OperationType] + if !ok { + log.Warn("invalid operation", "operation type", payload.OperationType.String()) + } + + err := function(payload.Payload) + if err != nil { + log.Error("something went wrong", "error", err.Error()) + } + return nil +} + func (i *indexer) saveBlock(marshalledData []byte) error { outportBlock := &outport.OutportBlock{} err := i.marshaller.Unmarshal(outportBlock, marshalledData) @@ -117,3 +134,8 @@ func (i *indexer) finalizedBlock(_ []byte) error { func (i *indexer) Close() error { return i.di.Close() } + +// IsInterfaceNil returns true if underlying object is nil +func (i *indexer) IsInterfaceNil() bool { + return i == nil +} From 6947d3953f82885ccad1727caa1dff6f1873441d Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 29 Mar 2023 12:50:34 +0300 Subject: [PATCH 039/189] bug fix --- process/wsindexer/indexer.go | 5 +++-- scripts/observers/.env | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index 49b83b4d..705820cc 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -41,8 +41,8 @@ func NewIndexer(marshaller marshal.Marshalizer, dataIndexer DataIndexer) (*index } // GetOperationsMap returns the map with all the operations that will index data -func (i *indexer) initActionsMap() map[data.OperationType]func(d []byte) error { - return map[data.OperationType]func(d []byte) error{ +func (i *indexer) initActionsMap() { + i.actions = map[data.OperationType]func(d []byte) error{ data.OperationSaveBlock: i.saveBlock, data.OperationRevertIndexedBlock: i.revertIndexedBlock, data.OperationSaveRoundsInfo: i.saveRounds, @@ -57,6 +57,7 @@ func (i *indexer) ProcessPayload(payload *data.PayloadData) error { function, ok := i.actions[payload.OperationType] if !ok { log.Warn("invalid operation", "operation type", payload.OperationType.String()) + return nil } err := function(payload.Payload) diff --git a/scripts/observers/.env b/scripts/observers/.env index 6d1d9bc5..f18a00df 100644 --- a/scripts/observers/.env +++ b/scripts/observers/.env @@ -1,7 +1,7 @@ MX_CHAIN_DEPLOY_GO_URL=https://github.com/multiversx/mx-chain-deploy-go NODE_GO_URL="https://github.com/multiversx/mx-chain-go" -NODE_GO_BRANCH="MX-13968-refactor-outport-interface" +NODE_GO_BRANCH="feat/outport-refactor" WORKING_DIRECTORY="IndexerObservers" OBSERVER_DIR_PREFIX="observer_shard_" From 935cce1699fafb8f864e77c54ca02307f95a81c3 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 29 Mar 2023 14:50:22 +0300 Subject: [PATCH 040/189] add proxy local testnet --- scripts/observers/.env | 1 + scripts/observers/config.py | 42 +++++++++++++++++++++++++++++++++++++ scripts/observers/start.py | 14 ++++++++++++- scripts/observers/stop.py | 1 + 4 files changed, 57 insertions(+), 1 deletion(-) diff --git a/scripts/observers/.env b/scripts/observers/.env index f18a00df..5da8ff2f 100644 --- a/scripts/observers/.env +++ b/scripts/observers/.env @@ -1,4 +1,5 @@ MX_CHAIN_DEPLOY_GO_URL=https://github.com/multiversx/mx-chain-deploy-go +MX_CHAIN_PROXY_URL=https://github.com/multiversx/mx-chain-proxy-go NODE_GO_URL="https://github.com/multiversx/mx-chain-go" NODE_GO_BRANCH="feat/outport-refactor" diff --git a/scripts/observers/config.py b/scripts/observers/config.py index b46da265..43983a3d 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -109,6 +109,10 @@ def clone_dependencies(working_dir): if not os.path.isdir(mx_chain_deploy_folder): Repo.clone_from(os.getenv('MX_CHAIN_DEPLOY_GO_URL'), mx_chain_deploy_folder) + mx_chain_proxy_folder = working_dir / "mx-chain-proxy-go" + if not os.path.isdir(mx_chain_proxy_folder): + Repo.clone_from(os.getenv('MX_CHAIN_PROXY_URL'), mx_chain_proxy_folder) + def prepare_seed_node(working_dir): print("preparing seed node") @@ -125,6 +129,42 @@ def prepare_seed_node(working_dir): os.chmod(working_dir/"seednode/seednode", st.st_mode | stat.S_IEXEC) +def prepare_proxy(working_dir): + print("preparing proxy") + mx_chain_proxy_go_folder = working_dir / "mx-chain-proxy-go" + subprocess.check_call(["go", "build"], cwd=mx_chain_proxy_go_folder / "cmd/proxy") + + mx_chain_proxy_go_binary_folder = mx_chain_proxy_go_folder / "cmd/proxy" + st = os.stat(mx_chain_proxy_go_binary_folder / "proxy") + os.chmod(mx_chain_proxy_go_binary_folder / "proxy", st.st_mode | stat.S_IEXEC) + + # prefs.toml + path_config = mx_chain_proxy_go_binary_folder / "config/config.toml" + config_data = toml.load(str(path_config)) + + config_data['GeneralSettings']['ServerPort'] = 7950 + + config_data['Observers'][0]['ShardId'] = 0 + config_data['Observers'][0]['Address'] = "http://127.0.0.1:9500" + + config_data['Observers'][1]['ShardId'] = 1 + config_data['Observers'][1]['Address'] = "http://127.0.0.1:9501" + + config_data['Observers'][2]['ShardId'] = 2 + config_data['Observers'][2]['Address'] = "http://127.0.0.1:9502" + + new_observer = { + 'ShardId': 4294967295, + 'Address': 'http://127.0.0.1:9503', + } + config_data['Observers'].append(new_observer) + del config_data['FullHistoryNodes'] + + f = open(path_config, 'w') + toml.dump(config_data, f) + f.close() + + def generate_config_for_local_testnet(working_dir): mx_chain_local_testnet_scripts = working_dir / "mx-chain-go/scripts/testnet" subprocess.check_call(["./clean.sh"], cwd=mx_chain_local_testnet_scripts) @@ -155,6 +195,8 @@ def main(): generate_config_for_local_testnet(working_dir) # prepare seednode prepare_seed_node(working_dir) + # prepare proxy + prepare_proxy(working_dir) # build binary mx-chain-go print("building node...") diff --git a/scripts/observers/start.py b/scripts/observers/start.py index 81f82941..6ef194b1 100644 --- a/scripts/observers/start.py +++ b/scripts/observers/start.py @@ -12,6 +12,17 @@ def start_seed_node(working_dir): os.chdir(current_directory) +def start_proxy(working_dir): + current_directory = os.getcwd() + + working_dir_proxy = working_dir/"mx-chain-proxy-go/cmd/proxy" + os.chdir(working_dir_proxy) + command = "./proxy" + os.system("screen -d -m -S proxy" + " " + command) + + os.chdir(current_directory) + + def start_observer(shard_id, working_dir, sk_index): current_observer = str(os.getenv('OBSERVER_DIR_PREFIX')) + str(shard_id) working_dir_observer = working_dir / current_observer @@ -19,7 +30,7 @@ def start_observer(shard_id, working_dir, sk_index): current_directory = os.getcwd() # start observer os.chdir(working_dir_observer / "node") - command = "./node" + " --log-level *:DEBUG --log-save --sk-index " + str(sk_index) + command = "./node" + " --log-level *:DEBUG --log-save --sk-index " + str(sk_index) + " --rest-api-interface :" + str(9500 + sk_index) os.system("screen -d -m -S obs" + str(shard_id) + " " + command) # start indexer @@ -40,6 +51,7 @@ def main(): print("staring observers and indexers....") start_seed_node(working_dir) + start_proxy(working_dir) start_observer(0, working_dir, 0) start_observer(1, working_dir, 1) start_observer(2, working_dir, 2) diff --git a/scripts/observers/stop.py b/scripts/observers/stop.py index 063f12ae..1bdf4bce 100644 --- a/scripts/observers/stop.py +++ b/scripts/observers/stop.py @@ -2,6 +2,7 @@ def main(): + os.system("screen -X -S proxy quit") os.system("screen -X -S seednode quit") os.system("screen -X -S obs0 quit") os.system("screen -X -S obs1 quit") From a20b4da96cbc2f857269eb06ddcfd1751f35d052 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 29 Mar 2023 16:00:10 +0300 Subject: [PATCH 041/189] fix config --- scripts/observers/config.py | 19 ++++++++++++++----- scripts/observers/start.py | 8 ++++---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/scripts/observers/config.py b/scripts/observers/config.py index 43983a3d..e842e3da 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -32,6 +32,15 @@ def update_toml_node(path, shard_id): toml.dump(prefs_data, f) f.close() + # config.toml + path_config = path / "config.toml" + config_data = toml.load(path_config) + config_data['DbLookupExtensions']['Enabled'] = True + config_data['EpochStartConfig']['RoundsPerEpoch'] = 20 + f = open(path_config, 'w') + toml.dump(config_data, f) + f.close() + # external.toml path_external = path / "external.toml" external_data = toml.load(str(path_external)) @@ -144,17 +153,17 @@ def prepare_proxy(working_dir): config_data['GeneralSettings']['ServerPort'] = 7950 - config_data['Observers'][0]['ShardId'] = 0 + config_data['Observers'][0]['ShardId'] = 4294967295 config_data['Observers'][0]['Address'] = "http://127.0.0.1:9500" - config_data['Observers'][1]['ShardId'] = 1 + config_data['Observers'][1]['ShardId'] = 0 config_data['Observers'][1]['Address'] = "http://127.0.0.1:9501" - config_data['Observers'][2]['ShardId'] = 2 + config_data['Observers'][2]['ShardId'] = 1 config_data['Observers'][2]['Address'] = "http://127.0.0.1:9502" new_observer = { - 'ShardId': 4294967295, + 'ShardId': 2, 'Address': 'http://127.0.0.1:9503', } config_data['Observers'].append(new_observer) @@ -210,10 +219,10 @@ def main(): # prepare observers config_folder = working_dir / "config" print("preparing config...") + prepare_observer(METACHAIN, working_dir, config_folder) prepare_observer(0, working_dir, config_folder) prepare_observer(1, working_dir, config_folder) prepare_observer(2, working_dir, config_folder) - prepare_observer(METACHAIN, working_dir, config_folder) if __name__ == "__main__": diff --git a/scripts/observers/start.py b/scripts/observers/start.py index 6ef194b1..ba8fb953 100644 --- a/scripts/observers/start.py +++ b/scripts/observers/start.py @@ -52,10 +52,10 @@ def main(): start_seed_node(working_dir) start_proxy(working_dir) - start_observer(0, working_dir, 0) - start_observer(1, working_dir, 1) - start_observer(2, working_dir, 2) - start_observer(METACHAIN, working_dir, 3) + start_observer(METACHAIN, working_dir, 0) + start_observer(0, working_dir, 1) + start_observer(1, working_dir, 2) + start_observer(2, working_dir, 3) print("done") From 2874013c5e244391d67ffdcbb7e5e4219cc6b5a9 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 29 Mar 2023 18:04:16 +0300 Subject: [PATCH 042/189] change log message --- process/dataindexer/workItems/workItemBlock.go | 10 +++++++--- scripts/observers/.env | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/process/dataindexer/workItems/workItemBlock.go b/process/dataindexer/workItems/workItemBlock.go index 64bda3da..b4491165 100644 --- a/process/dataindexer/workItems/workItemBlock.go +++ b/process/dataindexer/workItems/workItemBlock.go @@ -39,9 +39,13 @@ func (wib *itemBlock) Save() error { return nil } - defer func(startTime time.Time) { - log.Debug("wib.SaveBlockData duration", "time", time.Since(startTime)) - }(time.Now()) + defer func(startTime time.Time, headerHash []byte, headerNonce uint64) { + log.Debug("wib.SaveBlockData", + "duration", time.Since(startTime), + "block nonce", headerNonce, + "block hash", headerHash, + ) + }(time.Now(), wib.outportBlockWithHeader.BlockData.HeaderHash, wib.outportBlockWithHeader.Header.GetNonce()) log.Debug("indexer: starting indexing block", "hash", wib.outportBlockWithHeader.BlockData.HeaderHash, diff --git a/scripts/observers/.env b/scripts/observers/.env index 5da8ff2f..c215165f 100644 --- a/scripts/observers/.env +++ b/scripts/observers/.env @@ -2,7 +2,7 @@ MX_CHAIN_DEPLOY_GO_URL=https://github.com/multiversx/mx-chain-deploy-go MX_CHAIN_PROXY_URL=https://github.com/multiversx/mx-chain-proxy-go NODE_GO_URL="https://github.com/multiversx/mx-chain-go" -NODE_GO_BRANCH="feat/outport-refactor" +NODE_GO_BRANCH="fixes-outport-refactor" WORKING_DIRECTORY="IndexerObservers" OBSERVER_DIR_PREFIX="observer_shard_" From 2f7bdda868f60d44e0849ef0b92146fa95f12eb6 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 30 Mar 2023 12:51:59 +0300 Subject: [PATCH 043/189] configurable num of shards and proxy port --- scripts/observers/.env | 6 +++++ scripts/observers/config.py | 45 ++++++++++++++++++++++--------------- scripts/observers/start.py | 14 +++++++----- scripts/observers/stop.py | 16 ++++++++----- 4 files changed, 51 insertions(+), 30 deletions(-) diff --git a/scripts/observers/.env b/scripts/observers/.env index c215165f..5d1fa21b 100644 --- a/scripts/observers/.env +++ b/scripts/observers/.env @@ -9,3 +9,9 @@ OBSERVER_DIR_PREFIX="observer_shard_" # marshaller types: `json` or `gogo proto` WS_MARSHALLER_TYPE="gogo protobuf" + +NUM_OF_SHARDS=3 #MAX 3 + +OBSERVERS_START_PORT=9500 + +PROXY_PORT=7950 diff --git a/scripts/observers/config.py b/scripts/observers/config.py index e842e3da..b65d849e 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -33,10 +33,12 @@ def update_toml_node(path, shard_id): f.close() # config.toml + num_of_shards = int(os.getenv('NUM_OF_SHARDS')) path_config = path / "config.toml" config_data = toml.load(path_config) config_data['DbLookupExtensions']['Enabled'] = True config_data['EpochStartConfig']['RoundsPerEpoch'] = 20 + config_data['GeneralSettings']['GenesisMaxNumberOfShards'] = num_of_shards f = open(path_config, 'w') toml.dump(config_data, f) f.close() @@ -87,9 +89,10 @@ def prepare_observer(shard_id, working_dir, config_folder): def generate_new_config(working_dir): mx_chain_go_folder = working_dir / "mx-chain-go" / "scripts" / "testnet" + num_of_shards = str(os.getenv('NUM_OF_SHARDS')) with open(mx_chain_go_folder/"local.sh", "w") as file: - file.write("export SHARDCOUNT=3\n") + file.write(f'export SHARDCOUNT={num_of_shards}\n') file.write("export SHARD_VALIDATORCOUNT=1\n") file.write("export SHARD_OBSERVERCOUNT=0\n") file.write("export SHARD_CONSENSUS_SIZE=1\n") @@ -151,23 +154,28 @@ def prepare_proxy(working_dir): path_config = mx_chain_proxy_go_binary_folder / "config/config.toml" config_data = toml.load(str(path_config)) - config_data['GeneralSettings']['ServerPort'] = 7950 - - config_data['Observers'][0]['ShardId'] = 4294967295 - config_data['Observers'][0]['Address'] = "http://127.0.0.1:9500" - - config_data['Observers'][1]['ShardId'] = 0 - config_data['Observers'][1]['Address'] = "http://127.0.0.1:9501" + proxy_port = int(os.getenv('PROXY_PORT')) + config_data['GeneralSettings']['ServerPort'] = proxy_port + del config_data['Observers'] + del config_data['FullHistoryNodes'] - config_data['Observers'][2]['ShardId'] = 1 - config_data['Observers'][2]['Address'] = "http://127.0.0.1:9502" + config_data['Observers'] = [] - new_observer = { - 'ShardId': 2, - 'Address': 'http://127.0.0.1:9503', + observers_start_port = int(os.getenv('OBSERVERS_START_PORT')) + meta_observer = { + 'ShardId': 4294967295, + 'Address': f'http://127.0.0.1:{observers_start_port}', } - config_data['Observers'].append(new_observer) - del config_data['FullHistoryNodes'] + config_data['Observers'].append(meta_observer) + + num_of_shards = int(os.getenv('NUM_OF_SHARDS')) + for shardID in range(num_of_shards): + shard_observer_port = observers_start_port+shardID+1 + meta_observer = { + 'ShardId': shardID, + 'Address': f'http://127.0.0.1:{shard_observer_port}', + } + config_data['Observers'].append(meta_observer) f = open(path_config, 'w') toml.dump(config_data, f) @@ -220,9 +228,10 @@ def main(): config_folder = working_dir / "config" print("preparing config...") prepare_observer(METACHAIN, working_dir, config_folder) - prepare_observer(0, working_dir, config_folder) - prepare_observer(1, working_dir, config_folder) - prepare_observer(2, working_dir, config_folder) + + num_of_shards = int(os.getenv('NUM_OF_SHARDS')) + for shardID in range(num_of_shards): + prepare_observer(shardID, working_dir, config_folder) if __name__ == "__main__": diff --git a/scripts/observers/start.py b/scripts/observers/start.py index ba8fb953..049e3b45 100644 --- a/scripts/observers/start.py +++ b/scripts/observers/start.py @@ -23,14 +23,15 @@ def start_proxy(working_dir): os.chdir(current_directory) -def start_observer(shard_id, working_dir, sk_index): +def start_observer_and_indexer(shard_id, working_dir, sk_index): current_observer = str(os.getenv('OBSERVER_DIR_PREFIX')) + str(shard_id) working_dir_observer = working_dir / current_observer current_directory = os.getcwd() # start observer os.chdir(working_dir_observer / "node") - command = "./node" + " --log-level *:DEBUG --log-save --sk-index " + str(sk_index) + " --rest-api-interface :" + str(9500 + sk_index) + observers_start_port = int(os.getenv('OBSERVERS_START_PORT')) + command = "./node" + " --log-level *:DEBUG --log-save --sk-index " + str(sk_index) + " --rest-api-interface :" + str(observers_start_port + sk_index) os.system("screen -d -m -S obs" + str(shard_id) + " " + command) # start indexer @@ -52,10 +53,11 @@ def main(): start_seed_node(working_dir) start_proxy(working_dir) - start_observer(METACHAIN, working_dir, 0) - start_observer(0, working_dir, 1) - start_observer(1, working_dir, 2) - start_observer(2, working_dir, 3) + start_observer_and_indexer(METACHAIN, working_dir, 0) + + num_of_shards = int(os.getenv('NUM_OF_SHARDS')) + for shard_id in range(num_of_shards): + start_observer_and_indexer(shard_id, working_dir, shard_id+1) print("done") diff --git a/scripts/observers/stop.py b/scripts/observers/stop.py index 1bdf4bce..776b8ee7 100644 --- a/scripts/observers/stop.py +++ b/scripts/observers/stop.py @@ -1,17 +1,21 @@ import os +from dotenv import load_dotenv + def main(): + load_dotenv() os.system("screen -X -S proxy quit") os.system("screen -X -S seednode quit") - os.system("screen -X -S obs0 quit") - os.system("screen -X -S obs1 quit") - os.system("screen -X -S obs2 quit") + os.system("screen -X -S obs4294967295 quit") - os.system("screen -X -S indexer0 quit") - os.system("screen -X -S indexer1 quit") - os.system("screen -X -S indexer2 quit") os.system("screen -X -S indexer4294967295 quit") + + num_of_shards = int(os.getenv('NUM_OF_SHARDS')) + for shard_id in range(num_of_shards): + os.system(f'screen -X -S obs{shard_id} quit') + os.system(f'screen -X -S indexer{shard_id} quit') + print("done") From 2932ee969d2b6d8f50c21b5c23c3a608c5b94eed Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 30 Mar 2023 13:09:28 +0300 Subject: [PATCH 044/189] comments --- cmd/elasticindexer/config/prefs.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index d605456c..d13b4ae1 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -2,8 +2,11 @@ disabled-indices = [] [config.web-socket] server-url = "localhost:22111" + # Possible values: json, gogo protobuf. Should be compatible with mx-chain-node outport driver config data-marshaller-type = "json" + # Retry duration (receive/ send ack signal) in seconds retry-duration-in-seconds = 5 + # Signals if in case of data payload processing error, we should send the ack signal or not blocking-ack-on-error = true [config.elastic-cluster] From 85bf03b861696715c7360418182a84d53067e72a Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 31 Mar 2023 10:30:39 +0300 Subject: [PATCH 045/189] fixes after first review --- cmd/elasticindexer/config/prefs.toml | 2 +- process/dataindexer/dataIndexer.go | 31 ++++++---------------------- process/dataindexer/errors.go | 3 +++ process/dataindexer/interface.go | 3 ++- process/factory/indexerFactory.go | 25 ++++++++++++++++++++++ process/wsindexer/indexer.go | 6 +----- 6 files changed, 38 insertions(+), 32 deletions(-) diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index d13b4ae1..76ec7ede 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -7,7 +7,7 @@ # Retry duration (receive/ send ack signal) in seconds retry-duration-in-seconds = 5 # Signals if in case of data payload processing error, we should send the ack signal or not - blocking-ack-on-error = true + blocking-ack-on-error = false [config.elastic-cluster] use-kibana = false diff --git a/process/dataindexer/dataIndexer.go b/process/dataindexer/dataIndexer.go index 5634dac4..a2eab46d 100644 --- a/process/dataindexer/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -15,6 +15,7 @@ type ArgDataIndexer struct { HeaderMarshaller marshal.Marshalizer DataDispatcher DispatcherHandler ElasticProcessor ElasticProcessor + BlockContainer BlockContainerHandler } type dataIndexer struct { @@ -22,7 +23,7 @@ type dataIndexer struct { dispatcher DispatcherHandler elasticProcessor ElasticProcessor headerMarshaller marshal.Marshalizer - blockContainer blockContainerHandler + blockContainer BlockContainerHandler } // NewDataIndexer will create a new data indexer @@ -32,40 +33,17 @@ func NewDataIndexer(arguments ArgDataIndexer) (*dataIndexer, error) { return nil, err } - blockContainer, err := createBlockCreatorsContainer() - if err != nil { - return nil, err - } - dataIndexerObj := &dataIndexer{ isNilIndexer: false, dispatcher: arguments.DataDispatcher, elasticProcessor: arguments.ElasticProcessor, headerMarshaller: arguments.HeaderMarshaller, - blockContainer: blockContainer, + blockContainer: arguments.BlockContainer, } return dataIndexerObj, nil } -func createBlockCreatorsContainer() (blockContainerHandler, error) { - container := block.NewEmptyBlockCreatorsContainer() - err := container.Add(core.ShardHeaderV1, block.NewEmptyHeaderCreator()) - if err != nil { - return nil, err - } - err = container.Add(core.ShardHeaderV2, block.NewEmptyHeaderV2Creator()) - if err != nil { - return nil, err - } - err = container.Add(core.MetaHeader, block.NewEmptyMetaBlockCreator()) - if err != nil { - return nil, err - } - - return container, nil -} - func checkIndexerArgs(arguments ArgDataIndexer) error { if check.IfNil(arguments.DataDispatcher) { return ErrNilDataDispatcher @@ -76,6 +54,9 @@ func checkIndexerArgs(arguments ArgDataIndexer) error { if check.IfNil(arguments.HeaderMarshaller) { return ErrNilMarshalizer } + if check.IfNilReflect(arguments.BlockContainer) { + return ErrNilBlockContainerHandler + } return nil } diff --git a/process/dataindexer/errors.go b/process/dataindexer/errors.go index 0e2a33a7..6c33926c 100644 --- a/process/dataindexer/errors.go +++ b/process/dataindexer/errors.go @@ -91,3 +91,6 @@ var ErrNilBalanceConverter = errors.New("nil balance converter") // ErrNilOperationsHandler signals that a nil operations handler has been provided var ErrNilOperationsHandler = errors.New("nil operations handler") + +// ErrNilBlockContainerHandler signals that a nil block container handler has been provided +var ErrNilBlockContainerHandler = errors.New("nil bock container handler") diff --git a/process/dataindexer/interface.go b/process/dataindexer/interface.go index dc026034..31a87cc7 100644 --- a/process/dataindexer/interface.go +++ b/process/dataindexer/interface.go @@ -76,6 +76,7 @@ type BalanceConverter interface { IsInterfaceNil() bool } -type blockContainerHandler interface { +// BlockContainerHandler defines what a block container should be able to do +type BlockContainerHandler interface { Get(headerType core.HeaderType) (block.EmptyBlockCreator, error) } diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index 02771456..eb7c11d5 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -9,6 +9,7 @@ import ( "github.com/elastic/go-elasticsearch/v7" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-es-indexer-go/client" @@ -59,10 +60,16 @@ func NewIndexer(args ArgsIndexerFactory) (dataindexer.Indexer, error) { dispatcher.StartIndexData() + blockContainer, err := createBlockCreatorsContainer() + if err != nil { + return nil, err + } + arguments := dataindexer.ArgDataIndexer{ HeaderMarshaller: args.HeaderMarshaller, ElasticProcessor: elasticProcessor, DataDispatcher: dispatcher, + BlockContainer: blockContainer, } return dataindexer.NewDataIndexer(arguments) @@ -128,3 +135,21 @@ func checkDataIndexerParams(arguments ArgsIndexerFactory) error { return nil } + +func createBlockCreatorsContainer() (dataindexer.BlockContainerHandler, error) { + container := block.NewEmptyBlockCreatorsContainer() + err := container.Add(core.ShardHeaderV1, block.NewEmptyHeaderCreator()) + if err != nil { + return nil, err + } + err = container.Add(core.ShardHeaderV2, block.NewEmptyHeaderV2Creator()) + if err != nil { + return nil, err + } + err = container.Add(core.MetaHeader, block.NewEmptyMetaBlockCreator()) + if err != nil { + return nil, err + } + + return container, nil +} diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index 705820cc..e372f7ae 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -60,11 +60,7 @@ func (i *indexer) ProcessPayload(payload *data.PayloadData) error { return nil } - err := function(payload.Payload) - if err != nil { - log.Error("something went wrong", "error", err.Error()) - } - return nil + return function(payload.Payload) } func (i *indexer) saveBlock(marshalledData []byte) error { From f6b2e49b504481a9840ed657348356eb4d2754ed Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 31 Mar 2023 11:09:55 +0300 Subject: [PATCH 046/189] fixes second first review --- cmd/elasticindexer/config/prefs.toml | 2 +- mock/blockContainerStub.go | 19 +++++++++++++++++++ process/dataindexer/dataIndexer_test.go | 10 ++++++++++ process/wsindexer/indexer.go | 10 +++++----- scripts/observers/config.py | 7 +++++-- scripts/observers/start.py | 4 +++- scripts/observers/utils.py | 8 ++++++++ 7 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 mock/blockContainerStub.go diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index 76ec7ede..4b6d350f 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -4,7 +4,7 @@ server-url = "localhost:22111" # Possible values: json, gogo protobuf. Should be compatible with mx-chain-node outport driver config data-marshaller-type = "json" - # Retry duration (receive/ send ack signal) in seconds + # Retry duration (receive/send ack signal) in seconds retry-duration-in-seconds = 5 # Signals if in case of data payload processing error, we should send the ack signal or not blocking-ack-on-error = false diff --git a/mock/blockContainerStub.go b/mock/blockContainerStub.go new file mode 100644 index 00000000..4849af23 --- /dev/null +++ b/mock/blockContainerStub.go @@ -0,0 +1,19 @@ +package mock + +import ( + "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/data/block" +) + +type BlockContainerStub struct { + GetCalled func(headerType core.HeaderType) (block.EmptyBlockCreator, error) +} + +// Get - +func (bcs *BlockContainerStub) Get(headerType core.HeaderType) (block.EmptyBlockCreator, error) { + if bcs.GetCalled != nil { + return bcs.GetCalled(headerType) + } + + return nil, nil +} diff --git a/process/dataindexer/dataIndexer_test.go b/process/dataindexer/dataIndexer_test.go index 46b04ab4..ce07eab2 100644 --- a/process/dataindexer/dataIndexer_test.go +++ b/process/dataindexer/dataIndexer_test.go @@ -17,6 +17,7 @@ func NewDataIndexerArguments() ArgDataIndexer { DataDispatcher: &mock.DispatcherMock{}, ElasticProcessor: &mock.ElasticProcessorStub{}, HeaderMarshaller: &mock.MarshalizerMock{}, + BlockContainer: &mock.BlockContainerStub{}, } } @@ -65,6 +66,11 @@ func TestDataIndexer_SaveBlock(t *testing.T) { called = true }, } + arguments.BlockContainer = &mock.BlockContainerStub{ + GetCalled: func(headerType core.HeaderType) (dataBlock.EmptyBlockCreator, error) { + return dataBlock.NewEmptyHeaderV2Creator(), nil + }, + } ei, _ := NewDataIndexer(arguments) args := &outport.OutportBlock{ @@ -144,6 +150,10 @@ func TestDataIndexer_RevertIndexedBlock(t *testing.T) { called = true }, } + arguments.BlockContainer = &mock.BlockContainerStub{ + GetCalled: func(headerType core.HeaderType) (dataBlock.EmptyBlockCreator, error) { + return dataBlock.NewEmptyHeaderV2Creator(), nil + }} ei, _ := NewDataIndexer(arguments) err := ei.RevertIndexedBlock(&outport.BlockData{ diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index e372f7ae..42e335f0 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -31,13 +31,13 @@ func NewIndexer(marshaller marshal.Marshalizer, dataIndexer DataIndexer) (*index return nil, errNilDataIndexer } - i := &indexer{ + payloadIndexer := &indexer{ marshaller: marshaller, di: dataIndexer, } - i.initActionsMap() + payloadIndexer.initActionsMap() - return i, nil + return payloadIndexer, nil } // GetOperationsMap returns the map with all the operations that will index data @@ -54,13 +54,13 @@ func (i *indexer) initActionsMap() { } func (i *indexer) ProcessPayload(payload *data.PayloadData) error { - function, ok := i.actions[payload.OperationType] + operationAction, ok := i.actions[payload.OperationType] if !ok { log.Warn("invalid operation", "operation type", payload.OperationType.String()) return nil } - return function(payload.Payload) + return operationAction(payload.Payload) } func (i *indexer) saveBlock(marshalledData []byte) error { diff --git a/scripts/observers/config.py b/scripts/observers/config.py index b65d849e..07abb131 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -150,7 +150,7 @@ def prepare_proxy(working_dir): st = os.stat(mx_chain_proxy_go_binary_folder / "proxy") os.chmod(mx_chain_proxy_go_binary_folder / "proxy", st.st_mode | stat.S_IEXEC) - # prefs.toml + # config.toml path_config = mx_chain_proxy_go_binary_folder / "config/config.toml" config_data = toml.load(str(path_config)) @@ -203,6 +203,9 @@ def main(): print("use `python3 clean.py` command first") sys.exit() + num_of_shards = int(os.getenv('NUM_OF_SHARDS')) + check_num_of_shards(num_of_shards) + # clone mx-chain-go clone_mx_chain_go(working_dir) # clone dependencies @@ -229,7 +232,7 @@ def main(): print("preparing config...") prepare_observer(METACHAIN, working_dir, config_folder) - num_of_shards = int(os.getenv('NUM_OF_SHARDS')) + for shardID in range(num_of_shards): prepare_observer(shardID, working_dir, config_folder) diff --git a/scripts/observers/start.py b/scripts/observers/start.py index 049e3b45..2c237b58 100644 --- a/scripts/observers/start.py +++ b/scripts/observers/start.py @@ -49,13 +49,15 @@ def main(): print("working directory folder is missing...you should run first `python3 config.py` command") sys.exit() + num_of_shards = int(os.getenv('NUM_OF_SHARDS')) + check_num_of_shards(num_of_shards) + print("staring observers and indexers....") start_seed_node(working_dir) start_proxy(working_dir) start_observer_and_indexer(METACHAIN, working_dir, 0) - num_of_shards = int(os.getenv('NUM_OF_SHARDS')) for shard_id in range(num_of_shards): start_observer_and_indexer(shard_id, working_dir, shard_id+1) diff --git a/scripts/observers/utils.py b/scripts/observers/utils.py index 930a61dc..8de9661e 100644 --- a/scripts/observers/utils.py +++ b/scripts/observers/utils.py @@ -6,6 +6,7 @@ METACHAIN = 4294967295 WS_PORT_BASE = 22111 WS_METACHAIN_PORT = WS_PORT_BASE + 50 +MAX_NUM_OF_SHARDS = 3 def get_working_dir(): @@ -15,3 +16,10 @@ def get_working_dir(): sys.exit() return Path.home() / working_dir_var + + +def check_num_of_shards(num_of_shards): + if num_of_shards > MAX_NUM_OF_SHARDS: + print(f'the NUM_OF_SHARDS variable cannot be greater than {MAX_NUM_OF_SHARDS}') + sys.exit() + From 10c924fc5631ad42717323d58d7cad268730b4cb Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 31 Mar 2023 11:20:14 +0300 Subject: [PATCH 047/189] small fix --- mock/blockContainerStub.go | 1 + 1 file changed, 1 insertion(+) diff --git a/mock/blockContainerStub.go b/mock/blockContainerStub.go index 4849af23..e74643de 100644 --- a/mock/blockContainerStub.go +++ b/mock/blockContainerStub.go @@ -5,6 +5,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/block" ) +// BlockContainerStub - type BlockContainerStub struct { GetCalled func(headerType core.HeaderType) (block.EmptyBlockCreator, error) } From 2211a6137d8af6ee2cf405f1323ca73579c0aaf6 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 3 Apr 2023 11:54:17 +0300 Subject: [PATCH 048/189] latest version of core --- go.mod | 4 ++-- go.sum | 8 ++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 76e3bf14..d11e0669 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 - github.com/gorilla/websocket v1.5.0 - github.com/multiversx/mx-chain-core-go v1.2.1-0.20230329082847-b78e96c3ad5a + github.com/multiversx/mx-chain-core-go v1.2.1-0.20230330105824-932a718276f6 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.0 github.com/stretchr/testify v1.7.0 @@ -20,6 +19,7 @@ require ( github.com/denisbrodbeck/machineid v1.0.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/gorilla/websocket v1.5.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/pelletier/go-toml v1.9.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index ef150001..0276fe6e 100644 --- a/go.sum +++ b/go.sum @@ -71,12 +71,8 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230322093158-35195fa155c0 h1:Hr0bAMUJigh5xGFm2qoKKglEtpIXckxCLjCDBUYv1DM= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230322093158-35195fa155c0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230328143807-34b385f92114 h1:aTZm+1TlgEGn6ba6yCI2hVspNZ/BhA+7rv5l7uuw8eY= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230328143807-34b385f92114/go.mod h1:/lovncjwo+pXQ7IAERwNzwCifeH7SAWk0DGqjorX2bc= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230329082847-b78e96c3ad5a h1:cjPv/NIb4c3oBuBxxc2ggyaMvpmGlo1RO8mmzrkWARM= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230329082847-b78e96c3ad5a/go.mod h1:/lovncjwo+pXQ7IAERwNzwCifeH7SAWk0DGqjorX2bc= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230330105824-932a718276f6 h1:4Nv0uxJbfSZ1fqWcQEYyQ1SdAAluDEbHjTi0X8ZFXFs= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230330105824-932a718276f6/go.mod h1:/lovncjwo+pXQ7IAERwNzwCifeH7SAWk0DGqjorX2bc= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= github.com/multiversx/mx-chain-vm-common-go v1.4.0 h1:0i0cJZJOXGzqYzwtKFHSr2yGmnFAdizOuISK8HgsnYo= From 5cfb29a9156e3587daa6833bc5c8f0df1d188476 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 13 Apr 2023 11:09:24 +0300 Subject: [PATCH 049/189] integrate new ws client modify scripts --- cmd/elasticindexer/config/prefs.toml | 1 + config/config.go | 1 + factory/wsIndexerFactory.go | 9 ++++- go.mod | 4 +- go.sum | 7 +++- process/dataindexer/nilIndexer.go | 59 ---------------------------- process/wsindexer/indexer.go | 2 +- scripts/observers/.env | 6 ++- scripts/observers/config.py | 58 ++++++++++++++++++++------- scripts/observers/start.py | 19 +++++++++ scripts/observers/stop.py | 12 +++++- 11 files changed, 94 insertions(+), 84 deletions(-) delete mode 100644 process/dataindexer/nilIndexer.go diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index 4b6d350f..d5ca0dc1 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -2,6 +2,7 @@ disabled-indices = [] [config.web-socket] server-url = "localhost:22111" + is-server = true # Possible values: json, gogo protobuf. Should be compatible with mx-chain-node outport driver config data-marshaller-type = "json" # Retry duration (receive/send ack signal) in seconds diff --git a/config/config.go b/config/config.go index d3b1a9e5..8a0d7a63 100644 --- a/config/config.go +++ b/config/config.go @@ -37,6 +37,7 @@ type ClusterConfig struct { DisabledIndices []string `toml:"disabled-indices"` WebSocket struct { ServerURL string `toml:"server-url"` + IsServer bool `toml:"is-server"` DataMarshallerType string `toml:"data-marshaller-type"` RetryDurationInSec uint32 `toml:"retry-duration-in-seconds"` BlockingAckOnError bool `toml:"blocking-ack-on-error"` diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index e4cc95fa..76e80be1 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -5,16 +5,19 @@ import ( factoryHasher "github.com/multiversx/mx-chain-core-go/hashing/factory" "github.com/multiversx/mx-chain-core-go/marshal" factoryMarshaller "github.com/multiversx/mx-chain-core-go/marshal/factory" - "github.com/multiversx/mx-chain-core-go/websocketOutportDriver/client" + "github.com/multiversx/mx-chain-core-go/webSockets/clientServerReceiver" "github.com/multiversx/mx-chain-es-indexer-go/config" "github.com/multiversx/mx-chain-es-indexer-go/process/factory" "github.com/multiversx/mx-chain-es-indexer-go/process/wsindexer" + logger "github.com/multiversx/mx-chain-logger-go" ) const ( indexerCacheSize = 1 ) +var log = logger.GetOrCreate("elasticindexer") + // CreateWsIndexer will create a new instance of wsindexer.WSClient func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsindexer.WSClient, error) { wsMarshaller, err := factoryMarshaller.NewMarshalizer(clusterCfg.Config.WebSocket.DataMarshallerType) @@ -32,11 +35,13 @@ func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsinde return nil, err } - return client.CreateWsClient(client.ArgsCreateWsClient{ + return clientServerReceiver.NewClientServerReceiver(clientServerReceiver.ArgsWsClientServerReceiver{ Url: clusterCfg.Config.WebSocket.ServerURL, RetryDurationInSec: clusterCfg.Config.WebSocket.RetryDurationInSec, BlockingAckOnError: clusterCfg.Config.WebSocket.BlockingAckOnError, PayloadProcessor: indexer, + IsServer: clusterCfg.Config.WebSocket.IsServer, + Log: log, }) } diff --git a/go.mod b/go.mod index d11e0669..6f0fbcaa 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 - github.com/multiversx/mx-chain-core-go v1.2.1-0.20230330105824-932a718276f6 + github.com/multiversx/mx-chain-core-go v1.2.1-0.20230412135601-c9a2ae778109 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.0 github.com/stretchr/testify v1.7.0 @@ -19,6 +19,8 @@ require ( github.com/denisbrodbeck/machineid v1.0.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/pelletier/go-toml v1.9.3 // indirect diff --git a/go.sum b/go.sum index 0276fe6e..18db3d36 100644 --- a/go.sum +++ b/go.sum @@ -54,6 +54,9 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= @@ -71,8 +74,8 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230330105824-932a718276f6 h1:4Nv0uxJbfSZ1fqWcQEYyQ1SdAAluDEbHjTi0X8ZFXFs= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230330105824-932a718276f6/go.mod h1:/lovncjwo+pXQ7IAERwNzwCifeH7SAWk0DGqjorX2bc= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230412135601-c9a2ae778109 h1:iD7EuLFHvJVaM7U37/Lugc2JhUitahVZCFneFys6MdE= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230412135601-c9a2ae778109/go.mod h1:EMXipgB7JzH9ozDhGZwjY1t6UQBtaVgxb1aMo/gzfEA= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= github.com/multiversx/mx-chain-vm-common-go v1.4.0 h1:0i0cJZJOXGzqYzwtKFHSr2yGmnFAdizOuISK8HgsnYo= diff --git a/process/dataindexer/nilIndexer.go b/process/dataindexer/nilIndexer.go deleted file mode 100644 index 318eeb39..00000000 --- a/process/dataindexer/nilIndexer.go +++ /dev/null @@ -1,59 +0,0 @@ -package dataindexer - -import ( - "github.com/multiversx/mx-chain-core-go/data/outport" -) - -// NilIndexer will be used when an Indexer is required, but another one isn't necessary or available -type NilIndexer struct { -} - -// SaveBlock returns nil -func (ni *NilIndexer) SaveBlock(_ *outport.OutportBlock) error { - return nil -} - -// RevertIndexedBlock returns nil -func (ni *NilIndexer) RevertIndexedBlock(_ *outport.BlockData) error { - return nil -} - -// SaveRoundsInfo returns nil -func (ni *NilIndexer) SaveRoundsInfo(_ *outport.RoundsInfo) error { - return nil -} - -// SaveValidatorsRating returns nil -func (ni *NilIndexer) SaveValidatorsRating(_ *outport.ValidatorsRating) error { - return nil -} - -// SaveValidatorsPubKeys returns nil -func (ni *NilIndexer) SaveValidatorsPubKeys(_ *outport.ValidatorsPubKeys) error { - return nil -} - -// SaveAccounts returns nil -func (ni *NilIndexer) SaveAccounts(_ *outport.Accounts) error { - return nil -} - -// Close will do nothing -func (ni *NilIndexer) Close() error { - return nil -} - -// FinalizedBlock returns nil -func (ni *NilIndexer) FinalizedBlock(_ []byte) error { - return nil -} - -// IsInterfaceNil returns true if there is no value under the interface -func (ni *NilIndexer) IsInterfaceNil() bool { - return ni == nil -} - -// IsNilIndexer will return a bool value that signals if the indexer's implementation is a NilIndexer -func (ni *NilIndexer) IsNilIndexer() bool { - return true -} diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index 42e335f0..76f43c1a 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -6,7 +6,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/marshal" - "github.com/multiversx/mx-chain-core-go/websocketOutportDriver/data" + "github.com/multiversx/mx-chain-core-go/webSockets/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" logger "github.com/multiversx/mx-chain-logger-go" ) diff --git a/scripts/observers/.env b/scripts/observers/.env index 5d1fa21b..6f2481e2 100644 --- a/scripts/observers/.env +++ b/scripts/observers/.env @@ -2,16 +2,18 @@ MX_CHAIN_DEPLOY_GO_URL=https://github.com/multiversx/mx-chain-deploy-go MX_CHAIN_PROXY_URL=https://github.com/multiversx/mx-chain-proxy-go NODE_GO_URL="https://github.com/multiversx/mx-chain-go" -NODE_GO_BRANCH="fixes-outport-refactor" +NODE_GO_BRANCH="integrate-new-ws-client-server" WORKING_DIRECTORY="IndexerObservers" OBSERVER_DIR_PREFIX="observer_shard_" # marshaller types: `json` or `gogo proto` -WS_MARSHALLER_TYPE="gogo protobuf" +WS_MARSHALLER_TYPE="json" NUM_OF_SHARDS=3 #MAX 3 OBSERVERS_START_PORT=9500 PROXY_PORT=7950 + +INDEXER_BINARY_SERVER=true diff --git a/scripts/observers/config.py b/scripts/observers/config.py index 07abb131..5255633e 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -9,13 +9,21 @@ def update_toml_indexer(path, shard_id): # prefs.toml + is_indexer_server = os.getenv('INDEXER_BINARY_SERVER') path_prefs = path / "prefs.toml" prefs_data = toml.load(str(path_prefs)) - prefs_data['config']['web-socket']['server-url'] = str(shard_id) + + port = WS_PORT_BASE + shard_id + meta_port = WS_METACHAIN_PORT + if is_indexer_server: + port = WS_PORT_BASE + meta_port = WS_PORT_BASE + prefs_data['config']['web-socket']['is-server'] = True + if shard_id != METACHAIN: - prefs_data['config']['web-socket']['server-url'] = "localhost:" + str(WS_PORT_BASE + shard_id) + prefs_data['config']['web-socket']['server-url'] = "localhost:" + str(port) else: - prefs_data['config']['web-socket']['server-url'] = "localhost:" + str(WS_METACHAIN_PORT) + prefs_data['config']['web-socket']['server-url'] = "localhost:" + str(meta_port) prefs_data['config']['web-socket']['data-marshaller-type'] = str(os.getenv('WS_MARSHALLER_TYPE')) f = open(path_prefs, 'w') @@ -46,13 +54,23 @@ def update_toml_node(path, shard_id): # external.toml path_external = path / "external.toml" external_data = toml.load(str(path_external)) - external_data['WebSocketConnector']['Enabled'] = True + external_data['WebSocketsConnector']['Enabled'] = True + + port = WS_PORT_BASE + shard_id + meta_port = WS_METACHAIN_PORT + + is_indexer_server = os.getenv('INDEXER_BINARY_SERVER') + if is_indexer_server: + external_data['WebSocketsConnector']['IsServer'] = False + port = WS_PORT_BASE + meta_port = WS_PORT_BASE + if shard_id != METACHAIN: - external_data['WebSocketConnector']['URL'] = "localhost:" + str(WS_PORT_BASE + shard_id) + external_data['WebSocketsConnector']['URL'] = "localhost:" + str(port) else: - external_data['WebSocketConnector']['URL'] = "localhost:" + str(WS_METACHAIN_PORT) + external_data['WebSocketsConnector']['URL'] = "localhost:" + str(meta_port) - external_data['WebSocketConnector']['MarshallerType'] = str(os.getenv('WS_MARSHALLER_TYPE')) + external_data['WebSocketsConnector']['MarshallerType'] = str(os.getenv('WS_MARSHALLER_TYPE')) f = open(path_external, 'w') toml.dump(external_data, f) f.close() @@ -87,11 +105,21 @@ def prepare_observer(shard_id, working_dir, config_folder): update_toml_indexer(indexer_config, shard_id) +def prepare_indexer_server(meta_id, working_dir): + is_indexer_server = os.getenv('INDEXER_BINARY_SERVER') + if not is_indexer_server: + return + + current_observer = str(os.getenv('OBSERVER_DIR_PREFIX')) + str(meta_id) + working_dir_observer = working_dir / current_observer + shutil.copytree(working_dir_observer / "indexer", working_dir / "indexer") + + def generate_new_config(working_dir): mx_chain_go_folder = working_dir / "mx-chain-go" / "scripts" / "testnet" num_of_shards = str(os.getenv('NUM_OF_SHARDS')) - with open(mx_chain_go_folder/"local.sh", "w") as file: + with open(mx_chain_go_folder / "local.sh", "w") as file: file.write(f'export SHARDCOUNT={num_of_shards}\n') file.write("export SHARD_VALIDATORCOUNT=1\n") file.write("export SHARD_OBSERVERCOUNT=0\n") @@ -129,16 +157,16 @@ def clone_dependencies(working_dir): def prepare_seed_node(working_dir): print("preparing seed node") seed_node = Path.home() / "MultiversX/testnet/seednode" - shutil.copytree(seed_node, working_dir/"seednode") + shutil.copytree(seed_node, working_dir / "seednode") mx_chain_go_folder = working_dir / "mx-chain-go" subprocess.check_call(["go", "build"], cwd=mx_chain_go_folder / "cmd/seednode") seed_node_exec = mx_chain_go_folder / "cmd/seednode/seednode" - shutil.copyfile(seed_node_exec, working_dir/"seednode/seednode") + shutil.copyfile(seed_node_exec, working_dir / "seednode/seednode") - st = os.stat(working_dir/"seednode/seednode") - os.chmod(working_dir/"seednode/seednode", st.st_mode | stat.S_IEXEC) + st = os.stat(working_dir / "seednode/seednode") + os.chmod(working_dir / "seednode/seednode", st.st_mode | stat.S_IEXEC) def prepare_proxy(working_dir): @@ -170,7 +198,7 @@ def prepare_proxy(working_dir): num_of_shards = int(os.getenv('NUM_OF_SHARDS')) for shardID in range(num_of_shards): - shard_observer_port = observers_start_port+shardID+1 + shard_observer_port = observers_start_port + shardID + 1 meta_observer = { 'ShardId': shardID, 'Address': f'http://127.0.0.1:{shard_observer_port}', @@ -189,7 +217,7 @@ def generate_config_for_local_testnet(working_dir): config_folder = Path.home() / "MultiversX/testnet/node/config" os.rename(config_folder / "config_validator.toml", config_folder / "config.toml") - shutil.copytree(config_folder, working_dir/"config") + shutil.copytree(config_folder, working_dir / "config") def main(): @@ -231,7 +259,7 @@ def main(): config_folder = working_dir / "config" print("preparing config...") prepare_observer(METACHAIN, working_dir, config_folder) - + prepare_indexer_server(METACHAIN, working_dir) for shardID in range(num_of_shards): prepare_observer(shardID, working_dir, config_folder) diff --git a/scripts/observers/start.py b/scripts/observers/start.py index 2c237b58..4e6a865a 100644 --- a/scripts/observers/start.py +++ b/scripts/observers/start.py @@ -1,3 +1,5 @@ +import shutil + from dotenv import load_dotenv from utils import * @@ -35,6 +37,10 @@ def start_observer_and_indexer(shard_id, working_dir, sk_index): os.system("screen -d -m -S obs" + str(shard_id) + " " + command) # start indexer + is_indexer_server = os.getenv('INDEXER_BINARY_SERVER') + if is_indexer_server: + return + os.chdir(working_dir_observer / "indexer") command = "./elasticindexer" + " --log-level *:DEBUG --log-save" os.system("screen -d -m -S indexer" + str(shard_id) + " " + command) @@ -42,6 +48,15 @@ def start_observer_and_indexer(shard_id, working_dir, sk_index): os.chdir(current_directory) +def start_indexer_server(working_dir): + current_directory = os.getcwd() + os.chdir(working_dir / "indexer") + command = "./elasticindexer" + " --log-level *:DEBUG --log-save" + os.system("screen -d -m -S indexer" + "server" + " " + command) + + os.chdir(current_directory) + + def main(): load_dotenv() working_dir = get_working_dir() @@ -61,6 +76,10 @@ def main(): for shard_id in range(num_of_shards): start_observer_and_indexer(shard_id, working_dir, shard_id+1) + is_indexer_server = os.getenv('INDEXER_BINARY_SERVER') + if is_indexer_server: + start_indexer_server(working_dir) + print("done") diff --git a/scripts/observers/stop.py b/scripts/observers/stop.py index 776b8ee7..f4410e31 100644 --- a/scripts/observers/stop.py +++ b/scripts/observers/stop.py @@ -5,16 +5,24 @@ def main(): load_dotenv() + + is_indexer_server = os.getenv('INDEXER_BINARY_SERVER') + os.system("screen -X -S proxy quit") os.system("screen -X -S seednode quit") os.system("screen -X -S obs4294967295 quit") - os.system("screen -X -S indexer4294967295 quit") + if not is_indexer_server: + os.system("screen -X -S indexer4294967295 quit") num_of_shards = int(os.getenv('NUM_OF_SHARDS')) for shard_id in range(num_of_shards): os.system(f'screen -X -S obs{shard_id} quit') - os.system(f'screen -X -S indexer{shard_id} quit') + if not is_indexer_server: + os.system(f'screen -X -S indexer{shard_id} quit') + + if is_indexer_server: + os.system(f'screen -X -S indexerserver quit') print("done") From 7be2b714f608d4e5cd88bdd6725c0ec621827481 Mon Sep 17 00:00:00 2001 From: jules01 Date: Fri, 21 Apr 2023 13:23:49 +0300 Subject: [PATCH 050/189] - new releases --- go.mod | 4 ++-- go.sum | 8 ++++---- process/elasticproc/transactions/transactionDBBuilder.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index bd7b19af..59c9c2b2 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,9 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gorilla/websocket v1.5.0 - github.com/multiversx/mx-chain-core-go v1.2.0 + github.com/multiversx/mx-chain-core-go v1.2.1-0.20230421093838-792f0ca57194 github.com/multiversx/mx-chain-logger-go v1.0.11 - github.com/multiversx/mx-chain-vm-common-go v1.4.0 + github.com/multiversx/mx-chain-vm-common-go v1.4.1-0.20230421102013-5b1684df1757 github.com/stretchr/testify v1.7.0 github.com/tidwall/gjson v1.14.0 github.com/urfave/cli v1.22.10 diff --git a/go.sum b/go.sum index e1f7b242..feada152 100644 --- a/go.sum +++ b/go.sum @@ -70,12 +70,12 @@ github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.0 h1:K539hKZKcHjBiFQpowFbA3BUd95Fe5+FLC+rKBOFZF0= -github.com/multiversx/mx-chain-core-go v1.2.0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230421093838-792f0ca57194 h1:q55UzSoDmsRj3La0nBsd3hNjNXOcrKI7ltjLyRJGHgc= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230421093838-792f0ca57194/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= -github.com/multiversx/mx-chain-vm-common-go v1.4.0 h1:0i0cJZJOXGzqYzwtKFHSr2yGmnFAdizOuISK8HgsnYo= -github.com/multiversx/mx-chain-vm-common-go v1.4.0/go.mod h1:odBJC92ANA8zLtPh/wwajUUGJOaS88F5QYGf0t8Wgzw= +github.com/multiversx/mx-chain-vm-common-go v1.4.1-0.20230421102013-5b1684df1757 h1:Wxg5UNDl2T+0oiTd+BBq8DESZ8rw1O8r1VN2KRMEEKI= +github.com/multiversx/mx-chain-vm-common-go v1.4.1-0.20230421102013-5b1684df1757/go.mod h1:yOpR74vIXNeh5jidNEvzH+aQjC/go1M3L9FaaaQG6qw= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index 575eafa2..668986e0 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -81,7 +81,7 @@ func (dtb *dbTransactionBuilder) prepareTransaction( } guardianAddress := "" if len(tx.GuardianAddr) > 0 { - guardianAddress = dtb.addressPubkeyConverter.Encode(tx.GuardianAddr) + guardianAddress = dtb.addressPubkeyConverter.SilentEncode(tx.GuardianAddr, log) } return &data.Transaction{ From ea27f41fc61cab073c4280207318f26896feae7a Mon Sep 17 00:00:00 2001 From: jules01 Date: Fri, 21 Apr 2023 15:42:28 +0300 Subject: [PATCH 051/189] - new releases --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 59c9c2b2..6c874cd9 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,9 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gorilla/websocket v1.5.0 - github.com/multiversx/mx-chain-core-go v1.2.1-0.20230421093838-792f0ca57194 + github.com/multiversx/mx-chain-core-go v1.2.1-0.20230421123655-43ef5c8bf7c6 github.com/multiversx/mx-chain-logger-go v1.0.11 - github.com/multiversx/mx-chain-vm-common-go v1.4.1-0.20230421102013-5b1684df1757 + github.com/multiversx/mx-chain-vm-common-go v1.4.1-0.20230421123923-6b8b9b23e272 github.com/stretchr/testify v1.7.0 github.com/tidwall/gjson v1.14.0 github.com/urfave/cli v1.22.10 diff --git a/go.sum b/go.sum index feada152..f6dda09e 100644 --- a/go.sum +++ b/go.sum @@ -70,12 +70,12 @@ github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230421093838-792f0ca57194 h1:q55UzSoDmsRj3La0nBsd3hNjNXOcrKI7ltjLyRJGHgc= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230421093838-792f0ca57194/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230421123655-43ef5c8bf7c6 h1:B9UF0Kj/xQZvMh5SAy4+q/rCCiXcnnF+g2dFSFMkmOM= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230421123655-43ef5c8bf7c6/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= -github.com/multiversx/mx-chain-vm-common-go v1.4.1-0.20230421102013-5b1684df1757 h1:Wxg5UNDl2T+0oiTd+BBq8DESZ8rw1O8r1VN2KRMEEKI= -github.com/multiversx/mx-chain-vm-common-go v1.4.1-0.20230421102013-5b1684df1757/go.mod h1:yOpR74vIXNeh5jidNEvzH+aQjC/go1M3L9FaaaQG6qw= +github.com/multiversx/mx-chain-vm-common-go v1.4.1-0.20230421123923-6b8b9b23e272 h1:0fycWHx3T5tl43D5MTfBvmm01nnPo0R6cQCiDV7+zno= +github.com/multiversx/mx-chain-vm-common-go v1.4.1-0.20230421123923-6b8b9b23e272/go.mod h1:xlPYEjbq25Vv5JLV7nW6DFG21tC5Z2d/+QqvAO9y/eY= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= From 4b102359f07b602f663ef5691cf9a9db920f6165 Mon Sep 17 00:00:00 2001 From: jules01 Date: Mon, 24 Apr 2023 13:45:56 +0300 Subject: [PATCH 052/189] - proper releases --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 6c874cd9..7b91306c 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,9 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gorilla/websocket v1.5.0 - github.com/multiversx/mx-chain-core-go v1.2.1-0.20230421123655-43ef5c8bf7c6 + github.com/multiversx/mx-chain-core-go v1.2.1 github.com/multiversx/mx-chain-logger-go v1.0.11 - github.com/multiversx/mx-chain-vm-common-go v1.4.1-0.20230421123923-6b8b9b23e272 + github.com/multiversx/mx-chain-vm-common-go v1.4.1 github.com/stretchr/testify v1.7.0 github.com/tidwall/gjson v1.14.0 github.com/urfave/cli v1.22.10 diff --git a/go.sum b/go.sum index f6dda09e..4f774995 100644 --- a/go.sum +++ b/go.sum @@ -70,12 +70,12 @@ github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230421123655-43ef5c8bf7c6 h1:B9UF0Kj/xQZvMh5SAy4+q/rCCiXcnnF+g2dFSFMkmOM= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230421123655-43ef5c8bf7c6/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= +github.com/multiversx/mx-chain-core-go v1.2.1 h1:kmDfK7Znl3S0IJlDEE4sFuBOmA2rZkBudxlGhI1bvQc= +github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= -github.com/multiversx/mx-chain-vm-common-go v1.4.1-0.20230421123923-6b8b9b23e272 h1:0fycWHx3T5tl43D5MTfBvmm01nnPo0R6cQCiDV7+zno= -github.com/multiversx/mx-chain-vm-common-go v1.4.1-0.20230421123923-6b8b9b23e272/go.mod h1:xlPYEjbq25Vv5JLV7nW6DFG21tC5Z2d/+QqvAO9y/eY= +github.com/multiversx/mx-chain-vm-common-go v1.4.1 h1:HHZF9zU4WsMbfLrCarx3ESM95caWUrPBleGHKdsbzgc= +github.com/multiversx/mx-chain-vm-common-go v1.4.1/go.mod h1:K6yCdro8VohzYI6GwjGzTO+fJiPgO5coo2sgQb+zA24= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= From b59fa1401339d3ea4e066a9e0013b01818b2d56a Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 5 May 2023 14:15:17 +0300 Subject: [PATCH 053/189] integrate new client --- cmd/elasticindexer/config/prefs.toml | 1 + cmd/elasticindexer/main.go | 10 ++++-- config/config.go | 1 + factory/wsIndexerFactory.go | 52 +++++++++++++++++++++++----- go.mod | 2 +- go.sum | 4 +++ process/wsindexer/indexer.go | 2 +- process/wsindexer/interface.go | 2 +- 8 files changed, 59 insertions(+), 15 deletions(-) diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index d5ca0dc1..d51cf6ad 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -9,6 +9,7 @@ retry-duration-in-seconds = 5 # Signals if in case of data payload processing error, we should send the ack signal or not blocking-ack-on-error = false + with-acknowledge = true [config.elastic-cluster] use-kibana = false diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index 518644ea..8ac503e5 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -80,7 +80,7 @@ func startIndexer(ctx *cli.Context) error { return err } - wsClient, err := factory.CreateWsIndexer(cfg, clusterCfg) + wsHost, err := factory.CreateWsIndexer(cfg, clusterCfg) if err != nil { log.Error("cannot create ws indexer", "error", err) } @@ -88,11 +88,15 @@ func startIndexer(ctx *cli.Context) error { interrupt := make(chan os.Signal, 1) signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM) - go wsClient.Start() + wsHost.Start() <-interrupt log.Info("closing app at user's signal") - wsClient.Close() + err = wsHost.Close() + if err != nil { + log.Error("cannot close ws indexer", "error", err) + } + if !check.IfNilReflect(fileLogging) { err = fileLogging.Close() log.LogIfError(err) diff --git a/config/config.go b/config/config.go index 8a0d7a63..84023c54 100644 --- a/config/config.go +++ b/config/config.go @@ -41,6 +41,7 @@ type ClusterConfig struct { DataMarshallerType string `toml:"data-marshaller-type"` RetryDurationInSec uint32 `toml:"retry-duration-in-seconds"` BlockingAckOnError bool `toml:"blocking-ack-on-error"` + WithAcknowledge bool `toml:"with-acknowledge"` } `toml:"web-socket"` ElasticCluster struct { UseKibana bool `toml:"use-kibana"` diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 76e80be1..28f8cd33 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -2,10 +2,13 @@ package factory import ( "github.com/multiversx/mx-chain-core-go/core/pubkeyConverter" + "github.com/multiversx/mx-chain-core-go/data/typeConverters/uint64ByteSlice" factoryHasher "github.com/multiversx/mx-chain-core-go/hashing/factory" "github.com/multiversx/mx-chain-core-go/marshal" factoryMarshaller "github.com/multiversx/mx-chain-core-go/marshal/factory" - "github.com/multiversx/mx-chain-core-go/webSockets/clientServerReceiver" + "github.com/multiversx/mx-chain-core-go/webSocket" + "github.com/multiversx/mx-chain-core-go/webSocket/client" + "github.com/multiversx/mx-chain-core-go/webSocket/server" "github.com/multiversx/mx-chain-es-indexer-go/config" "github.com/multiversx/mx-chain-es-indexer-go/process/factory" "github.com/multiversx/mx-chain-es-indexer-go/process/wsindexer" @@ -35,14 +38,17 @@ func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsinde return nil, err } - return clientServerReceiver.NewClientServerReceiver(clientServerReceiver.ArgsWsClientServerReceiver{ - Url: clusterCfg.Config.WebSocket.ServerURL, - RetryDurationInSec: clusterCfg.Config.WebSocket.RetryDurationInSec, - BlockingAckOnError: clusterCfg.Config.WebSocket.BlockingAckOnError, - PayloadProcessor: indexer, - IsServer: clusterCfg.Config.WebSocket.IsServer, - Log: log, - }) + host, err := createWsHost(clusterCfg) + if err != nil { + return nil, err + } + + err = host.SetPayloadHandler(indexer) + if err != nil { + return nil, err + } + + return host, nil } func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig, wsMarshaller marshal.Marshalizer) (wsindexer.DataIndexer, error) { @@ -98,3 +104,31 @@ func prepareIndices(availableIndices, disabledIndices []string) []string { return indices } + +func createWsHost(clusterCfg config.ClusterConfig) (webSocket.HostWebSocket, error) { + uint64Converter := uint64ByteSlice.NewBigEndianConverter() + payloadConverter, err := webSocket.NewWebSocketPayloadConverter(uint64Converter) + if err != nil { + return nil, err + } + + if clusterCfg.Config.WebSocket.IsServer { + return server.NewWebSocketServer(server.ArgsWebSocketServer{ + RetryDurationInSeconds: int(clusterCfg.Config.WebSocket.RetryDurationInSec), + BlockingAckOnError: clusterCfg.Config.WebSocket.BlockingAckOnError, + WithAcknowledge: clusterCfg.Config.WebSocket.WithAcknowledge, + URL: clusterCfg.Config.WebSocket.ServerURL, + PayloadConverter: payloadConverter, + Log: log, + }) + } + + return client.NewWebSocketClient(client.ArgsWebSocketClient{ + RetryDurationInSeconds: int(clusterCfg.Config.WebSocket.RetryDurationInSec), + BlockingAckOnError: clusterCfg.Config.WebSocket.BlockingAckOnError, + WithAcknowledge: clusterCfg.Config.WebSocket.WithAcknowledge, + URL: clusterCfg.Config.WebSocket.ServerURL, + PayloadConverter: payloadConverter, + Log: log, + }) +} diff --git a/go.mod b/go.mod index 6f0fbcaa..75b2e19a 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 - github.com/multiversx/mx-chain-core-go v1.2.1-0.20230412135601-c9a2ae778109 + github.com/multiversx/mx-chain-core-go v1.2.1-0.20230505111209-1e30ee6bb1ca github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.0 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index 18db3d36..b3ce455e 100644 --- a/go.sum +++ b/go.sum @@ -76,6 +76,10 @@ github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZ github.com/multiversx/mx-chain-core-go v1.2.0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1-0.20230412135601-c9a2ae778109 h1:iD7EuLFHvJVaM7U37/Lugc2JhUitahVZCFneFys6MdE= github.com/multiversx/mx-chain-core-go v1.2.1-0.20230412135601-c9a2ae778109/go.mod h1:EMXipgB7JzH9ozDhGZwjY1t6UQBtaVgxb1aMo/gzfEA= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230505104029-ec390285a02f h1:yZI/JFVQtTGb+jN6UwlPchEabEwWEfUenKHxdt3p4QY= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230505104029-ec390285a02f/go.mod h1:/lovncjwo+pXQ7IAERwNzwCifeH7SAWk0DGqjorX2bc= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230505111209-1e30ee6bb1ca h1:tNkZDGd0AIfx7NPsFwG2fjaW0bQAPCCsPn1buONpLtE= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230505111209-1e30ee6bb1ca/go.mod h1:/lovncjwo+pXQ7IAERwNzwCifeH7SAWk0DGqjorX2bc= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= github.com/multiversx/mx-chain-vm-common-go v1.4.0 h1:0i0cJZJOXGzqYzwtKFHSr2yGmnFAdizOuISK8HgsnYo= diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index 76f43c1a..ff5c3a8b 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -6,7 +6,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/marshal" - "github.com/multiversx/mx-chain-core-go/webSockets/data" + "github.com/multiversx/mx-chain-core-go/webSocket/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" logger "github.com/multiversx/mx-chain-logger-go" ) diff --git a/process/wsindexer/interface.go b/process/wsindexer/interface.go index 91caee7d..e43d1474 100644 --- a/process/wsindexer/interface.go +++ b/process/wsindexer/interface.go @@ -7,7 +7,7 @@ import ( // WSClient defines what a websocket client should do type WSClient interface { Start() - Close() + Close() error } // DataIndexer dines what a data indexer should do From 0ef2ad798e3ce0d9315d887b6fae1f34d9c3e08f Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 5 May 2023 14:15:53 +0300 Subject: [PATCH 054/189] go mod tidy --- go.mod | 1 - go.sum | 6 ------ 2 files changed, 7 deletions(-) diff --git a/go.mod b/go.mod index 75b2e19a..4a0024d2 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,6 @@ require ( github.com/denisbrodbeck/machineid v1.0.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect diff --git a/go.sum b/go.sum index b3ce455e..504e95ec 100644 --- a/go.sum +++ b/go.sum @@ -54,8 +54,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -74,10 +72,6 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230412135601-c9a2ae778109 h1:iD7EuLFHvJVaM7U37/Lugc2JhUitahVZCFneFys6MdE= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230412135601-c9a2ae778109/go.mod h1:EMXipgB7JzH9ozDhGZwjY1t6UQBtaVgxb1aMo/gzfEA= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230505104029-ec390285a02f h1:yZI/JFVQtTGb+jN6UwlPchEabEwWEfUenKHxdt3p4QY= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230505104029-ec390285a02f/go.mod h1:/lovncjwo+pXQ7IAERwNzwCifeH7SAWk0DGqjorX2bc= github.com/multiversx/mx-chain-core-go v1.2.1-0.20230505111209-1e30ee6bb1ca h1:tNkZDGd0AIfx7NPsFwG2fjaW0bQAPCCsPn1buONpLtE= github.com/multiversx/mx-chain-core-go v1.2.1-0.20230505111209-1e30ee6bb1ca/go.mod h1:/lovncjwo+pXQ7IAERwNzwCifeH7SAWk0DGqjorX2bc= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= From 0dad67329d10c82be9c09b91e933a4c1d6b9ed71 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 8 May 2023 15:04:11 +0300 Subject: [PATCH 055/189] latest mx-chain-core-go --- factory/wsIndexerFactory.go | 8 +++----- go.mod | 2 +- go.sum | 4 ++-- process/wsindexer/indexer.go | 5 +++-- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 28f8cd33..74dfae75 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -2,7 +2,6 @@ package factory import ( "github.com/multiversx/mx-chain-core-go/core/pubkeyConverter" - "github.com/multiversx/mx-chain-core-go/data/typeConverters/uint64ByteSlice" factoryHasher "github.com/multiversx/mx-chain-core-go/hashing/factory" "github.com/multiversx/mx-chain-core-go/marshal" factoryMarshaller "github.com/multiversx/mx-chain-core-go/marshal/factory" @@ -38,7 +37,7 @@ func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsinde return nil, err } - host, err := createWsHost(clusterCfg) + host, err := createWsHost(clusterCfg, wsMarshaller) if err != nil { return nil, err } @@ -105,9 +104,8 @@ func prepareIndices(availableIndices, disabledIndices []string) []string { return indices } -func createWsHost(clusterCfg config.ClusterConfig) (webSocket.HostWebSocket, error) { - uint64Converter := uint64ByteSlice.NewBigEndianConverter() - payloadConverter, err := webSocket.NewWebSocketPayloadConverter(uint64Converter) +func createWsHost(clusterCfg config.ClusterConfig, wsMarshaller marshal.Marshalizer) (webSocket.HostWebSocket, error) { + payloadConverter, err := webSocket.NewWebSocketPayloadConverter(wsMarshaller) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index 4a0024d2..ec722373 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 - github.com/multiversx/mx-chain-core-go v1.2.1-0.20230505111209-1e30ee6bb1ca + github.com/multiversx/mx-chain-core-go v1.2.1-0.20230508115722-66623a0ad03e github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.0 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index 504e95ec..77f13d5e 100644 --- a/go.sum +++ b/go.sum @@ -72,8 +72,8 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230505111209-1e30ee6bb1ca h1:tNkZDGd0AIfx7NPsFwG2fjaW0bQAPCCsPn1buONpLtE= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230505111209-1e30ee6bb1ca/go.mod h1:/lovncjwo+pXQ7IAERwNzwCifeH7SAWk0DGqjorX2bc= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230508115722-66623a0ad03e h1:zXXc3zR1ejRaNNBHiWVOpRSo5SiGLmzXjWcXF+o6Xg0= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230508115722-66623a0ad03e/go.mod h1:/lovncjwo+pXQ7IAERwNzwCifeH7SAWk0DGqjorX2bc= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= github.com/multiversx/mx-chain-vm-common-go v1.4.0 h1:0i0cJZJOXGzqYzwtKFHSr2yGmnFAdizOuISK8HgsnYo= diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index ff5c3a8b..d4052341 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -54,9 +54,10 @@ func (i *indexer) initActionsMap() { } func (i *indexer) ProcessPayload(payload *data.PayloadData) error { - operationAction, ok := i.actions[payload.OperationType] + operationType := data.OperationType(payload.OperationType) + operationAction, ok := i.actions[operationType] if !ok { - log.Warn("invalid operation", "operation type", payload.OperationType.String()) + log.Warn("invalid operation", "operation type", operationType.String()) return nil } From 80c30aef1ffefe36cb16f353158d0c273dae3158 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 9 May 2023 12:35:38 +0300 Subject: [PATCH 056/189] fixes --- go.mod | 2 +- go.sum | 4 ++-- process/wsindexer/indexer.go | 27 +++++++++++++-------------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index ec722373..03c7d7ff 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 - github.com/multiversx/mx-chain-core-go v1.2.1-0.20230508115722-66623a0ad03e + github.com/multiversx/mx-chain-core-go v1.2.1-0.20230509093445-e9c9024c3bed github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.0 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index 77f13d5e..5e522d3c 100644 --- a/go.sum +++ b/go.sum @@ -72,8 +72,8 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230508115722-66623a0ad03e h1:zXXc3zR1ejRaNNBHiWVOpRSo5SiGLmzXjWcXF+o6Xg0= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230508115722-66623a0ad03e/go.mod h1:/lovncjwo+pXQ7IAERwNzwCifeH7SAWk0DGqjorX2bc= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230509093445-e9c9024c3bed h1:I3UnfSatLwxWR6vuTXB2A2chyMUv0KBodowjNe0LAX0= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230509093445-e9c9024c3bed/go.mod h1:/lovncjwo+pXQ7IAERwNzwCifeH7SAWk0DGqjorX2bc= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= github.com/multiversx/mx-chain-vm-common-go v1.4.0 h1:0i0cJZJOXGzqYzwtKFHSr2yGmnFAdizOuISK8HgsnYo= diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index d4052341..31173701 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -19,7 +19,7 @@ var ( type indexer struct { marshaller marshal.Marshalizer di DataIndexer - actions map[data.OperationType]func(marshalledData []byte) error + actions map[data.PayloadType]func(marshalledData []byte) error } // NewIndexer will create a new instance of *indexer @@ -42,26 +42,25 @@ func NewIndexer(marshaller marshal.Marshalizer, dataIndexer DataIndexer) (*index // GetOperationsMap returns the map with all the operations that will index data func (i *indexer) initActionsMap() { - i.actions = map[data.OperationType]func(d []byte) error{ - data.OperationSaveBlock: i.saveBlock, - data.OperationRevertIndexedBlock: i.revertIndexedBlock, - data.OperationSaveRoundsInfo: i.saveRounds, - data.OperationSaveValidatorsRating: i.saveValidatorsRating, - data.OperationSaveValidatorsPubKeys: i.saveValidatorsPubKeys, - data.OperationSaveAccounts: i.saveAccounts, - data.OperationFinalizedBlock: i.finalizedBlock, + i.actions = map[data.PayloadType]func(d []byte) error{ + data.PayloadSaveBlock: i.saveBlock, + data.PayloadRevertIndexedBlock: i.revertIndexedBlock, + data.PayloadSaveRoundsInfo: i.saveRounds, + data.PayloadSaveValidatorsRating: i.saveValidatorsRating, + data.PayloadSaveValidatorsPubKeys: i.saveValidatorsPubKeys, + data.PayloadSaveAccounts: i.saveAccounts, + data.PayloadFinalizedBlock: i.finalizedBlock, } } -func (i *indexer) ProcessPayload(payload *data.PayloadData) error { - operationType := data.OperationType(payload.OperationType) - operationAction, ok := i.actions[operationType] +func (i *indexer) ProcessPayload(payload []byte, payloadType data.PayloadType) error { + payloadTypeAction, ok := i.actions[payloadType] if !ok { - log.Warn("invalid operation", "operation type", operationType.String()) + log.Warn("invalid payload type", "payloadType type", payloadType.String()) return nil } - return operationAction(payload.Payload) + return payloadTypeAction(payload) } func (i *indexer) saveBlock(marshalledData []byte) error { From c63eae2132409a40e89fe89b8b90b348320b0cb2 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 11 May 2023 10:49:05 +0300 Subject: [PATCH 057/189] integrate new host --- cmd/elasticindexer/main.go | 2 -- factory/wsIndexerFactory.go | 40 +++++++++++----------------------- go.mod | 7 +++--- go.sum | 17 +++++++++++---- process/wsindexer/indexer.go | 25 ++++++++++----------- process/wsindexer/interface.go | 1 - 6 files changed, 42 insertions(+), 50 deletions(-) diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index 8ac503e5..d70d80e8 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -88,8 +88,6 @@ func startIndexer(ctx *cli.Context) error { interrupt := make(chan os.Signal, 1) signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM) - wsHost.Start() - <-interrupt log.Info("closing app at user's signal") err = wsHost.Close() diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 74dfae75..e10acf4d 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -1,13 +1,12 @@ package factory import ( + "github.com/multiversx/mx-chain-communication-go/websocket/data" + factoryHost "github.com/multiversx/mx-chain-communication-go/websocket/factory" "github.com/multiversx/mx-chain-core-go/core/pubkeyConverter" factoryHasher "github.com/multiversx/mx-chain-core-go/hashing/factory" "github.com/multiversx/mx-chain-core-go/marshal" factoryMarshaller "github.com/multiversx/mx-chain-core-go/marshal/factory" - "github.com/multiversx/mx-chain-core-go/webSocket" - "github.com/multiversx/mx-chain-core-go/webSocket/client" - "github.com/multiversx/mx-chain-core-go/webSocket/server" "github.com/multiversx/mx-chain-es-indexer-go/config" "github.com/multiversx/mx-chain-es-indexer-go/process/factory" "github.com/multiversx/mx-chain-es-indexer-go/process/wsindexer" @@ -104,29 +103,16 @@ func prepareIndices(availableIndices, disabledIndices []string) []string { return indices } -func createWsHost(clusterCfg config.ClusterConfig, wsMarshaller marshal.Marshalizer) (webSocket.HostWebSocket, error) { - payloadConverter, err := webSocket.NewWebSocketPayloadConverter(wsMarshaller) - if err != nil { - return nil, err - } - - if clusterCfg.Config.WebSocket.IsServer { - return server.NewWebSocketServer(server.ArgsWebSocketServer{ - RetryDurationInSeconds: int(clusterCfg.Config.WebSocket.RetryDurationInSec), - BlockingAckOnError: clusterCfg.Config.WebSocket.BlockingAckOnError, - WithAcknowledge: clusterCfg.Config.WebSocket.WithAcknowledge, - URL: clusterCfg.Config.WebSocket.ServerURL, - PayloadConverter: payloadConverter, - Log: log, - }) - } - - return client.NewWebSocketClient(client.ArgsWebSocketClient{ - RetryDurationInSeconds: int(clusterCfg.Config.WebSocket.RetryDurationInSec), - BlockingAckOnError: clusterCfg.Config.WebSocket.BlockingAckOnError, - WithAcknowledge: clusterCfg.Config.WebSocket.WithAcknowledge, - URL: clusterCfg.Config.WebSocket.ServerURL, - PayloadConverter: payloadConverter, - Log: log, +func createWsHost(clusterCfg config.ClusterConfig, wsMarshaller marshal.Marshalizer) (factoryHost.FullDuplexHost, error) { + return factoryHost.CreateWebSocketHost(factoryHost.ArgsWebSocketHost{ + WebSocketConfig: data.WebSocketConfig{ + URL: clusterCfg.Config.WebSocket.ServerURL, + WithAcknowledge: clusterCfg.Config.WebSocket.WithAcknowledge, + IsServer: clusterCfg.Config.WebSocket.IsServer, + RetryDurationInSec: int(clusterCfg.Config.WebSocket.RetryDurationInSec), + BlockingAckOnError: clusterCfg.Config.WebSocket.BlockingAckOnError, + }, + Marshaller: wsMarshaller, + Log: log, }) } diff --git a/go.mod b/go.mod index 03c7d7ff..cbadab23 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,11 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 - github.com/multiversx/mx-chain-core-go v1.2.1-0.20230509093445-e9c9024c3bed + github.com/multiversx/mx-chain-communication-go v0.0.0-20230510151427-d0bd41659967 + github.com/multiversx/mx-chain-core-go v1.2.1-0.20230511072341-83cfe4713dac github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.0 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.8.2 github.com/tidwall/gjson v1.14.0 github.com/urfave/cli v1.22.10 ) @@ -31,5 +32,5 @@ require ( golang.org/x/crypto v0.3.0 // indirect golang.org/x/sys v0.2.0 // indirect google.golang.org/protobuf v1.26.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 5e522d3c..59c13450 100644 --- a/go.sum +++ b/go.sum @@ -70,10 +70,13 @@ github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/multiversx/mx-chain-communication-go v0.0.0-20230510151427-d0bd41659967 h1:/Zg28BlD8Xs5Gl8V4ltfJ2n/UkP5Jw4anvTOOMkb4Bg= +github.com/multiversx/mx-chain-communication-go v0.0.0-20230510151427-d0bd41659967/go.mod h1:GPHOm4HSXbvC0IotMziWXQmhtsUe69ScBPYsb+mF9bk= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230509093445-e9c9024c3bed h1:I3UnfSatLwxWR6vuTXB2A2chyMUv0KBodowjNe0LAX0= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230509093445-e9c9024c3bed/go.mod h1:/lovncjwo+pXQ7IAERwNzwCifeH7SAWk0DGqjorX2bc= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230510143029-ab37792342df/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230511072341-83cfe4713dac h1:cueVlh2q3oO6reSZndgBEv/VL9xkaCgvPedo1ADqohg= +github.com/multiversx/mx-chain-core-go v1.2.1-0.20230511072341-83cfe4713dac/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= github.com/multiversx/mx-chain-vm-common-go v1.4.0 h1:0i0cJZJOXGzqYzwtKFHSr2yGmnFAdizOuISK8HgsnYo= @@ -97,8 +100,13 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -188,5 +196,6 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index 31173701..4b715c4d 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -6,7 +6,6 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/marshal" - "github.com/multiversx/mx-chain-core-go/webSocket/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" logger "github.com/multiversx/mx-chain-logger-go" ) @@ -19,7 +18,7 @@ var ( type indexer struct { marshaller marshal.Marshalizer di DataIndexer - actions map[data.PayloadType]func(marshalledData []byte) error + actions map[string]func(marshalledData []byte) error } // NewIndexer will create a new instance of *indexer @@ -42,21 +41,21 @@ func NewIndexer(marshaller marshal.Marshalizer, dataIndexer DataIndexer) (*index // GetOperationsMap returns the map with all the operations that will index data func (i *indexer) initActionsMap() { - i.actions = map[data.PayloadType]func(d []byte) error{ - data.PayloadSaveBlock: i.saveBlock, - data.PayloadRevertIndexedBlock: i.revertIndexedBlock, - data.PayloadSaveRoundsInfo: i.saveRounds, - data.PayloadSaveValidatorsRating: i.saveValidatorsRating, - data.PayloadSaveValidatorsPubKeys: i.saveValidatorsPubKeys, - data.PayloadSaveAccounts: i.saveAccounts, - data.PayloadFinalizedBlock: i.finalizedBlock, + i.actions = map[string]func(d []byte) error{ + outport.TopicSaveBlock: i.saveBlock, + outport.TopicRevertIndexedBlock: i.revertIndexedBlock, + outport.TopicSaveRoundsInfo: i.saveRounds, + outport.TopicSaveValidatorsRating: i.saveValidatorsRating, + outport.TopicSaveValidatorsPubKeys: i.saveValidatorsPubKeys, + outport.TopicSaveAccounts: i.saveAccounts, + outport.TopicFinalizedBlock: i.finalizedBlock, } } -func (i *indexer) ProcessPayload(payload []byte, payloadType data.PayloadType) error { - payloadTypeAction, ok := i.actions[payloadType] +func (i *indexer) ProcessPayload(payload []byte, topic string) error { + payloadTypeAction, ok := i.actions[topic] if !ok { - log.Warn("invalid payload type", "payloadType type", payloadType.String()) + log.Warn("invalid payload type", "topic", topic) return nil } diff --git a/process/wsindexer/interface.go b/process/wsindexer/interface.go index e43d1474..22069859 100644 --- a/process/wsindexer/interface.go +++ b/process/wsindexer/interface.go @@ -6,7 +6,6 @@ import ( // WSClient defines what a websocket client should do type WSClient interface { - Start() Close() error } From 87909d92f17666f7169057f7a94c6a661b0a2d7f Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 11 May 2023 11:20:31 +0300 Subject: [PATCH 058/189] update scripts --- scripts/observers/config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/observers/config.py b/scripts/observers/config.py index 5255633e..3922e8e9 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -54,23 +54,23 @@ def update_toml_node(path, shard_id): # external.toml path_external = path / "external.toml" external_data = toml.load(str(path_external)) - external_data['WebSocketsConnector']['Enabled'] = True + external_data['HostDriverConfig']['Enabled'] = True port = WS_PORT_BASE + shard_id meta_port = WS_METACHAIN_PORT is_indexer_server = os.getenv('INDEXER_BINARY_SERVER') if is_indexer_server: - external_data['WebSocketsConnector']['IsServer'] = False + external_data['HostDriverConfig']['IsServer'] = False port = WS_PORT_BASE meta_port = WS_PORT_BASE if shard_id != METACHAIN: - external_data['WebSocketsConnector']['URL'] = "localhost:" + str(port) + external_data['HostDriverConfig']['URL'] = "localhost:" + str(port) else: - external_data['WebSocketsConnector']['URL'] = "localhost:" + str(meta_port) + external_data['HostDriverConfig']['URL'] = "localhost:" + str(meta_port) - external_data['WebSocketsConnector']['MarshallerType'] = str(os.getenv('WS_MARSHALLER_TYPE')) + external_data['HostDriverConfig']['MarshallerType'] = str(os.getenv('WS_MARSHALLER_TYPE')) f = open(path_external, 'w') toml.dump(external_data, f) f.close() From 7a424dd3b3d0de494620eb0e78012129a5af79a4 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 12 May 2023 13:23:09 +0300 Subject: [PATCH 059/189] fixes after review --- cmd/elasticindexer/config/prefs.toml | 8 ++++++-- config/config.go | 4 ++-- factory/wsIndexerFactory.go | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- scripts/observers/config.py | 2 +- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index d51cf6ad..08ad3f68 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -1,14 +1,18 @@ [config] disabled-indices = [] [config.web-socket] - server-url = "localhost:22111" - is-server = true + # URL for the WebSocket client/server connection + # This value represents the IP address and port number that the WebSocket client or server will use to establish a connection. + url = "localhost:22111" + # This flag will start the WebSocket connector as server or client( can be "client" or "server") + mode = "server" # Possible values: json, gogo protobuf. Should be compatible with mx-chain-node outport driver config data-marshaller-type = "json" # Retry duration (receive/send ack signal) in seconds retry-duration-in-seconds = 5 # Signals if in case of data payload processing error, we should send the ack signal or not blocking-ack-on-error = false + # After a message will be sent it will wait for an ack message if this flag is enabled with-acknowledge = true [config.elastic-cluster] diff --git a/config/config.go b/config/config.go index 84023c54..301d41fb 100644 --- a/config/config.go +++ b/config/config.go @@ -36,8 +36,8 @@ type ClusterConfig struct { Config struct { DisabledIndices []string `toml:"disabled-indices"` WebSocket struct { - ServerURL string `toml:"server-url"` - IsServer bool `toml:"is-server"` + URL string `toml:"url"` + Mode string `toml:"mode"` DataMarshallerType string `toml:"data-marshaller-type"` RetryDurationInSec uint32 `toml:"retry-duration-in-seconds"` BlockingAckOnError bool `toml:"blocking-ack-on-error"` diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index e10acf4d..4f544115 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -106,9 +106,9 @@ func prepareIndices(availableIndices, disabledIndices []string) []string { func createWsHost(clusterCfg config.ClusterConfig, wsMarshaller marshal.Marshalizer) (factoryHost.FullDuplexHost, error) { return factoryHost.CreateWebSocketHost(factoryHost.ArgsWebSocketHost{ WebSocketConfig: data.WebSocketConfig{ - URL: clusterCfg.Config.WebSocket.ServerURL, + URL: clusterCfg.Config.WebSocket.URL, WithAcknowledge: clusterCfg.Config.WebSocket.WithAcknowledge, - IsServer: clusterCfg.Config.WebSocket.IsServer, + Mode: clusterCfg.Config.WebSocket.Mode, RetryDurationInSec: int(clusterCfg.Config.WebSocket.RetryDurationInSec), BlockingAckOnError: clusterCfg.Config.WebSocket.BlockingAckOnError, }, diff --git a/go.mod b/go.mod index cbadab23..08a96dad 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 - github.com/multiversx/mx-chain-communication-go v0.0.0-20230510151427-d0bd41659967 + github.com/multiversx/mx-chain-communication-go v0.0.0-20230512095548-5bc637293104 github.com/multiversx/mx-chain-core-go v1.2.1-0.20230511072341-83cfe4713dac github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.0 diff --git a/go.sum b/go.sum index 59c13450..1a952b3d 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiversx/mx-chain-communication-go v0.0.0-20230510151427-d0bd41659967 h1:/Zg28BlD8Xs5Gl8V4ltfJ2n/UkP5Jw4anvTOOMkb4Bg= -github.com/multiversx/mx-chain-communication-go v0.0.0-20230510151427-d0bd41659967/go.mod h1:GPHOm4HSXbvC0IotMziWXQmhtsUe69ScBPYsb+mF9bk= +github.com/multiversx/mx-chain-communication-go v0.0.0-20230512095548-5bc637293104 h1:oFsYNkebv7TQygdEjN4aGgQ8ICLPmS9bDJmzlOHtU2Y= +github.com/multiversx/mx-chain-communication-go v0.0.0-20230512095548-5bc637293104/go.mod h1:GPHOm4HSXbvC0IotMziWXQmhtsUe69ScBPYsb+mF9bk= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1-0.20230510143029-ab37792342df/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= diff --git a/scripts/observers/config.py b/scripts/observers/config.py index 3922e8e9..e8f84bd4 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -18,7 +18,7 @@ def update_toml_indexer(path, shard_id): if is_indexer_server: port = WS_PORT_BASE meta_port = WS_PORT_BASE - prefs_data['config']['web-socket']['is-server'] = True + prefs_data['config']['web-socket']['mode'] = "server" if shard_id != METACHAIN: prefs_data['config']['web-socket']['server-url'] = "localhost:" + str(port) From c0cb76b274fb88776271c7333b363310c57d29b2 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 12 May 2023 13:23:48 +0300 Subject: [PATCH 060/189] fix --- scripts/observers/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/observers/config.py b/scripts/observers/config.py index e8f84bd4..037cbce7 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -21,9 +21,9 @@ def update_toml_indexer(path, shard_id): prefs_data['config']['web-socket']['mode'] = "server" if shard_id != METACHAIN: - prefs_data['config']['web-socket']['server-url'] = "localhost:" + str(port) + prefs_data['config']['web-socket']['url'] = "localhost:" + str(port) else: - prefs_data['config']['web-socket']['server-url'] = "localhost:" + str(meta_port) + prefs_data['config']['web-socket']['url'] = "localhost:" + str(meta_port) prefs_data['config']['web-socket']['data-marshaller-type'] = str(os.getenv('WS_MARSHALLER_TYPE')) f = open(path_prefs, 'w') From ee6f43ebdd2247876df0802014faa93701059536 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 12 May 2023 13:51:20 +0300 Subject: [PATCH 061/189] small fix --- cmd/elasticindexer/config/prefs.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index 08ad3f68..141b7a0a 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -4,7 +4,7 @@ # URL for the WebSocket client/server connection # This value represents the IP address and port number that the WebSocket client or server will use to establish a connection. url = "localhost:22111" - # This flag will start the WebSocket connector as server or client( can be "client" or "server") + # This flag describes the mode to start the WebSocket connector. Can be "client" or "server" mode = "server" # Possible values: json, gogo protobuf. Should be compatible with mx-chain-node outport driver config data-marshaller-type = "json" From 4ddd6b7d1dc3a5e33c040c7171580ed4ffed240a Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 15 May 2023 10:26:17 +0300 Subject: [PATCH 062/189] added --- process/wsindexer/indexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index 4b715c4d..e98e65b4 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -52,6 +52,7 @@ func (i *indexer) initActionsMap() { } } +// ProcessPayload will proces the provided payload based on the topic func (i *indexer) ProcessPayload(payload []byte, topic string) error { payloadTypeAction, ok := i.actions[topic] if !ok { From edcb91661b08c747d58b0bc2c4d22133ca1a4454 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 15 May 2023 13:13:10 +0300 Subject: [PATCH 063/189] fixes --- cmd/elasticindexer/main.go | 8 ++++---- scripts/observers/.env | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index d70d80e8..3dca13ae 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -67,22 +67,22 @@ func main() { func startIndexer(ctx *cli.Context) error { cfg, err := loadMainConfig(ctx.GlobalString(configurationFile.Name)) if err != nil { - return err + return fmt.Errorf("%w while loading the config file", err) } clusterCfg, err := loadClusterConfig(ctx.GlobalString(configurationPreferencesFile.Name)) if err != nil { - return err + return fmt.Errorf("%w while loading the preferences config file", err) } fileLogging, err := initializeLogger(ctx, cfg) if err != nil { - return err + return fmt.Errorf("%w while initializing the logger", err) } wsHost, err := factory.CreateWsIndexer(cfg, clusterCfg) if err != nil { - log.Error("cannot create ws indexer", "error", err) + return fmt.Errorf("%w while creating the indexer", err) } interrupt := make(chan os.Signal, 1) diff --git a/scripts/observers/.env b/scripts/observers/.env index 6f2481e2..5418c53d 100644 --- a/scripts/observers/.env +++ b/scripts/observers/.env @@ -7,8 +7,8 @@ NODE_GO_BRANCH="integrate-new-ws-client-server" WORKING_DIRECTORY="IndexerObservers" OBSERVER_DIR_PREFIX="observer_shard_" -# marshaller types: `json` or `gogo proto` -WS_MARSHALLER_TYPE="json" +# marshaller types: `json` or `gogo protobuf` +WS_MARSHALLER_TYPE="gogo protobuf" NUM_OF_SHARDS=3 #MAX 3 From 8ad121726fafda67f3468901174c87c29ca37fc5 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 15 May 2023 13:16:42 +0300 Subject: [PATCH 064/189] small refactoring --- process/dataindexer/workItems/workItemBlock.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/process/dataindexer/workItems/workItemBlock.go b/process/dataindexer/workItems/workItemBlock.go index b4491165..c551900f 100644 --- a/process/dataindexer/workItems/workItemBlock.go +++ b/process/dataindexer/workItems/workItemBlock.go @@ -39,13 +39,17 @@ func (wib *itemBlock) Save() error { return nil } - defer func(startTime time.Time, headerHash []byte, headerNonce uint64) { + headerNonce := wib.outportBlockWithHeader.Header.GetNonce() + headerHash := wib.outportBlockWithHeader.BlockData.HeaderHash + shardID := wib.outportBlockWithHeader.Header.GetShardID() + defer func(startTime time.Time, headerHash []byte, headerNonce uint64, shardID uint32) { log.Debug("wib.SaveBlockData", "duration", time.Since(startTime), - "block nonce", headerNonce, - "block hash", headerHash, + "shardID", shardID, + "nonce", headerNonce, + "hash", headerHash, ) - }(time.Now(), wib.outportBlockWithHeader.BlockData.HeaderHash, wib.outportBlockWithHeader.Header.GetNonce()) + }(time.Now(), headerHash, headerNonce, shardID) log.Debug("indexer: starting indexing block", "hash", wib.outportBlockWithHeader.BlockData.HeaderHash, @@ -58,7 +62,7 @@ func (wib *itemBlock) Save() error { err := wib.indexer.SaveHeader(wib.outportBlockWithHeader) if err != nil { return fmt.Errorf("%w when saving header block, hash %s, nonce %d", - err, hex.EncodeToString(wib.outportBlockWithHeader.BlockData.HeaderHash), wib.outportBlockWithHeader.Header.GetNonce()) + err, hex.EncodeToString(headerHash), headerNonce) } if len(wib.outportBlockWithHeader.BlockData.Body.MiniBlocks) == 0 { @@ -68,13 +72,13 @@ func (wib *itemBlock) Save() error { err = wib.indexer.SaveMiniblocks(wib.outportBlockWithHeader.Header, wib.outportBlockWithHeader.BlockData.Body) if err != nil { return fmt.Errorf("%w when saving miniblocks, block hash %s, nonce %d", - err, hex.EncodeToString(wib.outportBlockWithHeader.BlockData.HeaderHash), wib.outportBlockWithHeader.Header.GetNonce()) + err, hex.EncodeToString(headerHash), headerNonce) } err = wib.indexer.SaveTransactions(wib.outportBlockWithHeader) if err != nil { return fmt.Errorf("%w when saving transactions, block hash %s, nonce %d", - err, hex.EncodeToString(wib.outportBlockWithHeader.BlockData.HeaderHash), wib.outportBlockWithHeader.Header.GetNonce()) + err, hex.EncodeToString(headerHash), headerNonce) } return nil From a185f25d76da0f0a2f09eed8e6c877e2c11962b6 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 15 May 2023 14:33:33 +0300 Subject: [PATCH 065/189] fixed --- process/dataindexer/workItems/workItemBlock.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/process/dataindexer/workItems/workItemBlock.go b/process/dataindexer/workItems/workItemBlock.go index c551900f..346e7f3f 100644 --- a/process/dataindexer/workItems/workItemBlock.go +++ b/process/dataindexer/workItems/workItemBlock.go @@ -42,14 +42,13 @@ func (wib *itemBlock) Save() error { headerNonce := wib.outportBlockWithHeader.Header.GetNonce() headerHash := wib.outportBlockWithHeader.BlockData.HeaderHash shardID := wib.outportBlockWithHeader.Header.GetShardID() - defer func(startTime time.Time, headerHash []byte, headerNonce uint64, shardID uint32) { - log.Debug("wib.SaveBlockData", - "duration", time.Since(startTime), - "shardID", shardID, - "nonce", headerNonce, - "hash", headerHash, - ) - }(time.Now(), headerHash, headerNonce, shardID) + startTime := time.Now() + defer log.Debug("wib.SaveBlockData", + "duration", time.Since(startTime), + "shardID", shardID, + "nonce", headerNonce, + "hash", headerHash, + ) log.Debug("indexer: starting indexing block", "hash", wib.outportBlockWithHeader.BlockData.HeaderHash, From 5f3764a8f1f5dd4d374e195847e49e898d76e39b Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 15 May 2023 14:48:49 +0300 Subject: [PATCH 066/189] revert --- process/dataindexer/workItems/workItemBlock.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/process/dataindexer/workItems/workItemBlock.go b/process/dataindexer/workItems/workItemBlock.go index 346e7f3f..c551900f 100644 --- a/process/dataindexer/workItems/workItemBlock.go +++ b/process/dataindexer/workItems/workItemBlock.go @@ -42,13 +42,14 @@ func (wib *itemBlock) Save() error { headerNonce := wib.outportBlockWithHeader.Header.GetNonce() headerHash := wib.outportBlockWithHeader.BlockData.HeaderHash shardID := wib.outportBlockWithHeader.Header.GetShardID() - startTime := time.Now() - defer log.Debug("wib.SaveBlockData", - "duration", time.Since(startTime), - "shardID", shardID, - "nonce", headerNonce, - "hash", headerHash, - ) + defer func(startTime time.Time, headerHash []byte, headerNonce uint64, shardID uint32) { + log.Debug("wib.SaveBlockData", + "duration", time.Since(startTime), + "shardID", shardID, + "nonce", headerNonce, + "hash", headerHash, + ) + }(time.Now(), headerHash, headerNonce, shardID) log.Debug("indexer: starting indexing block", "hash", wib.outportBlockWithHeader.BlockData.HeaderHash, From c7c22e214123686a080820e76a49488ec22dbd5a Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 18 May 2023 13:29:16 +0300 Subject: [PATCH 067/189] update mx-chain-core-go add a new flag with the import db mode --- cmd/elasticindexer/flags.go | 6 ++++++ cmd/elasticindexer/main.go | 4 +++- factory/wsIndexerFactory.go | 7 ++++--- go.mod | 2 +- go.sum | 4 ++-- process/elasticproc/elasticProcessor.go | 6 ++++-- process/elasticproc/factory/elasticProcessorFactory.go | 2 ++ process/factory/indexerFactory.go | 2 ++ 8 files changed, 24 insertions(+), 9 deletions(-) diff --git a/cmd/elasticindexer/flags.go b/cmd/elasticindexer/flags.go index 4a82dd5e..c8e2421d 100644 --- a/cmd/elasticindexer/flags.go +++ b/cmd/elasticindexer/flags.go @@ -35,4 +35,10 @@ var ( Name: "disable-ansi-color", Usage: "Boolean option for disabling ANSI colors in the logging system.", } + importDB = cli.BoolFlag{ + Name: "import-db", + Usage: "This flag, when enabled, triggers the indexer to operate in import database mode. In this mode," + + " the indexer excludes the indexing of cross shard transactions received from the source shard. " + + "This flag must be enabled when the observers are in import database mode.", + } ) diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index 3dca13ae..67cc107d 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -47,6 +47,7 @@ func main() { logLevel, logSaveFile, disableAnsiColor, + importDB, } app.Authors = []cli.Author{ { @@ -80,7 +81,8 @@ func startIndexer(ctx *cli.Context) error { return fmt.Errorf("%w while initializing the logger", err) } - wsHost, err := factory.CreateWsIndexer(cfg, clusterCfg) + importDBMode := ctx.GlobalBool(importDB.Name) + wsHost, err := factory.CreateWsIndexer(cfg, clusterCfg, importDBMode) if err != nil { return fmt.Errorf("%w while creating the indexer", err) } diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 4f544115..3b2a9f60 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -20,13 +20,13 @@ const ( var log = logger.GetOrCreate("elasticindexer") // CreateWsIndexer will create a new instance of wsindexer.WSClient -func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsindexer.WSClient, error) { +func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig, importDB bool) (wsindexer.WSClient, error) { wsMarshaller, err := factoryMarshaller.NewMarshalizer(clusterCfg.Config.WebSocket.DataMarshallerType) if err != nil { return nil, err } - dataIndexer, err := createDataIndexer(cfg, clusterCfg, wsMarshaller) + dataIndexer, err := createDataIndexer(cfg, clusterCfg, wsMarshaller, importDB) if err != nil { return nil, err } @@ -49,7 +49,7 @@ func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig) (wsinde return host, nil } -func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig, wsMarshaller marshal.Marshalizer) (wsindexer.DataIndexer, error) { +func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig, wsMarshaller marshal.Marshalizer, importDB bool) (wsindexer.DataIndexer, error) { marshaller, err := factoryMarshaller.NewMarshalizer(cfg.Config.Marshaller.Type) if err != nil { return nil, err @@ -81,6 +81,7 @@ func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig, wsMar AddressPubkeyConverter: addressPubkeyConverter, ValidatorPubkeyConverter: validatorPubkeyConverter, HeaderMarshaller: wsMarshaller, + ImportDB: importDB, }) } diff --git a/go.mod b/go.mod index 08a96dad..3ac40a05 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/multiversx/mx-chain-communication-go v0.0.0-20230512095548-5bc637293104 - github.com/multiversx/mx-chain-core-go v1.2.1-0.20230511072341-83cfe4713dac + github.com/multiversx/mx-chain-core-go v1.2.4-0.20230517135533-2e54a17cd912 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.0 github.com/stretchr/testify v1.8.2 diff --git a/go.sum b/go.sum index 1a952b3d..502a0d50 100644 --- a/go.sum +++ b/go.sum @@ -75,8 +75,8 @@ github.com/multiversx/mx-chain-communication-go v0.0.0-20230512095548-5bc6372931 github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1-0.20230510143029-ab37792342df/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230511072341-83cfe4713dac h1:cueVlh2q3oO6reSZndgBEv/VL9xkaCgvPedo1ADqohg= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230511072341-83cfe4713dac/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= +github.com/multiversx/mx-chain-core-go v1.2.4-0.20230517135533-2e54a17cd912 h1:7dgFuxG2sUbQEFA4y36NAoRjuN+Z3PlY0znmmgr7ZSo= +github.com/multiversx/mx-chain-core-go v1.2.4-0.20230517135533-2e54a17cd912/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= github.com/multiversx/mx-chain-vm-common-go v1.4.0 h1:0i0cJZJOXGzqYzwtKFHSr2yGmnFAdizOuISK8HgsnYo= diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index 283b4cd9..295c14c2 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -44,6 +44,7 @@ type objectsMap = map[string]interface{} type ArgElasticProcessor struct { BulkRequestMaxSize int UseKibana bool + ImportDB bool IndexTemplates map[string]*bytes.Buffer IndexPolicies map[string]*bytes.Buffer EnabledIndexes map[string]struct{} @@ -60,6 +61,7 @@ type ArgElasticProcessor struct { type elasticProcessor struct { bulkRequestMaxSize int + importDB bool enabledIndexes map[string]struct{} elasticClient DatabaseClientHandler accountsProc DBAccountHandler @@ -406,7 +408,7 @@ func (ei *elasticProcessor) miniblocksInDBMap(mbs []*data.Miniblock) (map[string func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader) error { headerTimestamp := obh.Header.GetTimeStamp() - preparedResults := ei.transactionsProc.PrepareTransactionsForDatabase(obh.BlockData.Body, obh.Header, obh.TransactionPool, obh.IsImportDB, obh.NumberOfShards) + preparedResults := ei.transactionsProc.PrepareTransactionsForDatabase(obh.BlockData.Body, obh.Header, obh.TransactionPool, ei.importDB, obh.NumberOfShards) logsData := ei.logsAndEventsProc.ExtractDataFromLogs(obh.TransactionPool.Logs, preparedResults, headerTimestamp, obh.Header.GetShardID(), obh.NumberOfShards) buffers := data.NewBufferSlice(ei.bulkRequestMaxSize) @@ -415,7 +417,7 @@ func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader return err } - err = ei.prepareAndIndexOperations(preparedResults.Transactions, preparedResults.TxHashStatus, obh.Header, preparedResults.ScResults, buffers, obh.IsImportDB) + err = ei.prepareAndIndexOperations(preparedResults.Transactions, preparedResults.TxHashStatus, obh.Header, preparedResults.ScResults, buffers, ei.importDB) if err != nil { return err } diff --git a/process/elasticproc/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go index dc5033bc..f10ed48f 100644 --- a/process/elasticproc/factory/elasticProcessorFactory.go +++ b/process/elasticproc/factory/elasticProcessorFactory.go @@ -29,6 +29,7 @@ type ArgElasticProcessorFactory struct { Denomination int BulkRequestMaxSize int UseKibana bool + ImportDB bool } // CreateElasticProcessor will create a new instance of ElasticProcessor @@ -118,6 +119,7 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E IndexTemplates: indexTemplates, IndexPolicies: indexPolicies, OperationsProc: operationsProc, + ImportDB: arguments.ImportDB, } return elasticproc.NewElasticProcessor(args) diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index eb7c11d5..76a59ae0 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -26,6 +26,7 @@ var log = logger.GetOrCreate("indexer/factory") type ArgsIndexerFactory struct { Enabled bool UseKibana bool + ImportDB bool IndexerCacheSize int Denomination int BulkRequestMaxSize int @@ -105,6 +106,7 @@ func createElasticProcessor(args ArgsIndexerFactory) (dataindexer.ElasticProcess Denomination: args.Denomination, EnabledIndexes: args.EnabledIndexes, BulkRequestMaxSize: args.BulkRequestMaxSize, + ImportDB: args.ImportDB, } return factory.CreateElasticProcessor(argsElasticProcFac) From 87b0947855ef96e0c519744b791e471ddae28901 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 18 May 2023 13:43:14 +0300 Subject: [PATCH 068/189] fix integration tests --- .../accountsBalanceNftTransfer_test.go | 6 ++---- .../accountsBalanceWithLowerTimestamp_test.go | 8 ++++---- integrationtests/accountsESDTRollback_test.go | 2 +- integrationtests/accountsESDTWithTokenType_test.go | 8 ++++---- integrationtests/claimRewards_test.go | 2 +- integrationtests/createNFTWithTags_test.go | 6 +++--- integrationtests/delegators_test.go | 8 ++++---- integrationtests/esdtTransfer_test.go | 2 +- integrationtests/issueTokenAndSetRoles_test.go | 12 ++++++------ integrationtests/issueToken_test.go | 8 ++++---- integrationtests/logsCrossShard_test.go | 6 +++--- ...iTransferWithScCallAndErrorSignaledBySC_test.go | 4 ++-- integrationtests/nftIssueCreateBurn_test.go | 6 +++--- integrationtests/nftTransferCrossShard_test.go | 12 ++++++------ .../nftTransferCrossWithScCall_test.go | 2 +- integrationtests/relayedTx_test.go | 6 +++--- integrationtests/scCallIntraShard_test.go | 4 ++-- integrationtests/transactions_test.go | 2 +- integrationtests/updateNFT_test.go | 14 +++++++------- 19 files changed, 58 insertions(+), 60 deletions(-) diff --git a/integrationtests/accountsBalanceNftTransfer_test.go b/integrationtests/accountsBalanceNftTransfer_test.go index 752d3746..6d586e89 100644 --- a/integrationtests/accountsBalanceNftTransfer_test.go +++ b/integrationtests/accountsBalanceNftTransfer_test.go @@ -23,7 +23,6 @@ func createOutportBlockWithHeader( header coreData.HeaderHandler, pool *outport.TransactionPool, coreAlteredAccounts map[string]*alteredAccount.AlteredAccount, - importDD bool, numOfShards uint32, ) *outport.OutportBlockWithHeader { return &outport.OutportBlockWithHeader{ @@ -34,7 +33,6 @@ func createOutportBlockWithHeader( TransactionPool: pool, AlteredAccounts: coreAlteredAccounts, NumberOfShards: numOfShards, - IsImportDB: importDD, }, Header: header, } @@ -91,7 +89,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, testNumOfShards)) require.Nil(t, err) ids := []string{fmt.Sprintf("%s-NFT-abcdef-718863", addr)} @@ -152,7 +150,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { }, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, testNumOfShards)) require.Nil(t, err) ids = []string{fmt.Sprintf("%s-NFT-abcdef-718863", addr)} diff --git a/integrationtests/accountsBalanceWithLowerTimestamp_test.go b/integrationtests/accountsBalanceWithLowerTimestamp_test.go index 1e984b3a..4aa250af 100644 --- a/integrationtests/accountsBalanceWithLowerTimestamp_test.go +++ b/integrationtests/accountsBalanceWithLowerTimestamp_test.go @@ -78,7 +78,7 @@ func TestIndexAccountsBalance(t *testing.T) { }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, testNumOfShards)) require.Nil(t, err) ids := []string{addr} @@ -101,7 +101,7 @@ func TestIndexAccountsBalance(t *testing.T) { ShardID: 2, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, testNumOfShards)) require.Nil(t, err) ids = []string{addr} @@ -163,7 +163,7 @@ func TestIndexAccountsBalance(t *testing.T) { }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, testNumOfShards)) require.Nil(t, err) ids = []string{addr} @@ -198,7 +198,7 @@ func TestIndexAccountsBalance(t *testing.T) { } pool.Transactions = make(map[string]*outport.TxInfo) - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, testNumOfShards)) require.Nil(t, err) ids = []string{addr} diff --git a/integrationtests/accountsESDTRollback_test.go b/integrationtests/accountsESDTRollback_test.go index 37d418cb..9f6cdda5 100644 --- a/integrationtests/accountsESDTRollback_test.go +++ b/integrationtests/accountsESDTRollback_test.go @@ -80,7 +80,7 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { ShardID: 2, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, testNumOfShards)) require.Nil(t, err) ids := []string{fmt.Sprintf("%s-TOKEN-eeee-02", addr)} diff --git a/integrationtests/accountsESDTWithTokenType_test.go b/integrationtests/accountsESDTWithTokenType_test.go index 25bf75ee..9d5833aa 100644 --- a/integrationtests/accountsESDTWithTokenType_test.go +++ b/integrationtests/accountsESDTWithTokenType_test.go @@ -56,7 +56,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, testNumOfShards)) require.Nil(t, err) ids := []string{"SEMI-abcd"} @@ -117,7 +117,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, testNumOfShards)) require.Nil(t, err) ids = []string{fmt.Sprintf("%s-SEMI-abcd-02", address)} @@ -189,7 +189,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, testNumOfShards)) require.Nil(t, err) ids := []string{fmt.Sprintf("%s-TTTT-abcd-02", address)} @@ -228,7 +228,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, testNumOfShards)) require.Nil(t, err) ids = []string{"TTTT-abcd"} diff --git a/integrationtests/claimRewards_test.go b/integrationtests/claimRewards_test.go index 853060aa..e3c237bf 100644 --- a/integrationtests/claimRewards_test.go +++ b/integrationtests/claimRewards_test.go @@ -121,7 +121,7 @@ func TestTransactionWithClaimRewardsGasRefund(t *testing.T) { }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/createNFTWithTags_test.go b/integrationtests/createNFTWithTags_test.go index c3b7d579..fa572fe2 100644 --- a/integrationtests/createNFTWithTags_test.go +++ b/integrationtests/createNFTWithTags_test.go @@ -88,7 +88,7 @@ func TestCreateNFTWithTags(t *testing.T) { } body := &dataBlock.Body{} - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, testNumOfShards)) require.Nil(t, err) ids := []string{fmt.Sprintf("%s-DESK-abcd-01", address1)} @@ -135,7 +135,7 @@ func TestCreateNFTWithTags(t *testing.T) { coreAlteredAccounts[address1].Tokens[0].Nonce = 2 body = &dataBlock.Body{} - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, testNumOfShards)) require.Nil(t, err) genericResponse = &GenericResponse{} @@ -183,7 +183,7 @@ func TestCreateNFTWithTags(t *testing.T) { } body = &dataBlock.Body{} - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, coreAlteredAccounts, testNumOfShards)) require.Nil(t, err) ids = append(ids, "XFxcXFxcXFxcXFxcXFxcXFxcXA==", "JycnJw==", "PDw8Pj4+JiYmJiYmJiYmJiYmJiYm") diff --git a/integrationtests/delegators_test.go b/integrationtests/delegators_test.go index d6c13e6e..85ca507f 100644 --- a/integrationtests/delegators_test.go +++ b/integrationtests/delegators_test.go @@ -59,7 +59,7 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids := []string{"9v/pLAXxUZJ4Oy1U+x5al/Xg5sebh1dYCRTeZwg/u68="} @@ -91,7 +91,7 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { } header.TimeStamp = 5050 - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse) @@ -121,7 +121,7 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { } header.TimeStamp = 5060 - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse) @@ -161,7 +161,7 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { } header.TimeStamp = 5070 - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse) diff --git a/integrationtests/esdtTransfer_test.go b/integrationtests/esdtTransfer_test.go index 13a6e5e8..132b743e 100644 --- a/integrationtests/esdtTransfer_test.go +++ b/integrationtests/esdtTransfer_test.go @@ -102,7 +102,7 @@ func TestESDTTransferTooMuchGasProvided(t *testing.T) { hex.EncodeToString(scrHash1): {SmartContractResult: scr1, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/issueTokenAndSetRoles_test.go b/integrationtests/issueTokenAndSetRoles_test.go index 09c6c6d7..25920c50 100644 --- a/integrationtests/issueTokenAndSetRoles_test.go +++ b/integrationtests/issueTokenAndSetRoles_test.go @@ -57,7 +57,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, testNumOfShards)) require.Nil(t, err) ids := []string{"TOK-abcd"} @@ -86,7 +86,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -119,7 +119,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -148,7 +148,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, testNumOfShards)) require.Nil(t, err) ids = []string{"TOK-abcd"} @@ -195,7 +195,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, testNumOfShards)) require.Nil(t, err) ids := []string{"TTT-abcd"} @@ -223,7 +223,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, testNumOfShards)) require.Nil(t, err) ids = []string{"TTT-abcd"} diff --git a/integrationtests/issueToken_test.go b/integrationtests/issueToken_test.go index 6a453311..d501c7d2 100644 --- a/integrationtests/issueToken_test.go +++ b/integrationtests/issueToken_test.go @@ -51,7 +51,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids := []string{"SSSS-abcd"} @@ -84,7 +84,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) @@ -115,7 +115,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) @@ -142,7 +142,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { } header.TimeStamp = 10000 - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) diff --git a/integrationtests/logsCrossShard_test.go b/integrationtests/logsCrossShard_test.go index 34f56e84..52869697 100644 --- a/integrationtests/logsCrossShard_test.go +++ b/integrationtests/logsCrossShard_test.go @@ -55,7 +55,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { }, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, testNumOfShards)) require.Nil(t, err) ids := []string{logID} @@ -96,7 +96,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { }, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse) @@ -129,7 +129,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { }, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse) diff --git a/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go b/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go index 5205533c..9b08e3c3 100644 --- a/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go +++ b/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go @@ -91,7 +91,7 @@ func TestMultiTransferCrossShardAndScCallErrorSignaledBySC(t *testing.T) { hex.EncodeToString(scrHash1): {SmartContractResult: scr1, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} @@ -159,7 +159,7 @@ func TestMultiTransferCrossShardAndScCallErrorSignaledBySC(t *testing.T) { }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids = []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/nftIssueCreateBurn_test.go b/integrationtests/nftIssueCreateBurn_test.go index 13ee8839..5bd8f628 100644 --- a/integrationtests/nftIssueCreateBurn_test.go +++ b/integrationtests/nftIssueCreateBurn_test.go @@ -54,7 +54,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids := []string{"NON-abcd"} @@ -99,7 +99,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids = []string{"NON-abcd-02"} @@ -134,7 +134,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids = []string{"NON-abcd-02"} diff --git a/integrationtests/nftTransferCrossShard_test.go b/integrationtests/nftTransferCrossShard_test.go index bf02fbfc..eea7c1d9 100644 --- a/integrationtests/nftTransferCrossShard_test.go +++ b/integrationtests/nftTransferCrossShard_test.go @@ -91,7 +91,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -131,7 +131,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -235,7 +235,7 @@ func TestNFTTransferCrossShard(t *testing.T) { hex.EncodeToString(scrHash2): {SmartContractResult: scr2, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -273,7 +273,7 @@ func TestNFTTransferCrossShard(t *testing.T) { }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -341,7 +341,7 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -410,7 +410,7 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t hex.EncodeToString(scrHash2): {SmartContractResult: scr2, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) diff --git a/integrationtests/nftTransferCrossWithScCall_test.go b/integrationtests/nftTransferCrossWithScCall_test.go index 5d83fdda..336f5da3 100644 --- a/integrationtests/nftTransferCrossWithScCall_test.go +++ b/integrationtests/nftTransferCrossWithScCall_test.go @@ -87,7 +87,7 @@ func TestNFTTransferCrossShardWithScCall(t *testing.T) { hex.EncodeToString(scrHash2): {SmartContractResult: scr2, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/relayedTx_test.go b/integrationtests/relayedTx_test.go index b1883d9c..08868c26 100644 --- a/integrationtests/relayedTx_test.go +++ b/integrationtests/relayedTx_test.go @@ -90,7 +90,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { hex.EncodeToString(scrHash1): {SmartContractResult: scr1, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -132,7 +132,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, testNumOfShards)) require.Nil(t, err) err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) @@ -231,7 +231,7 @@ func TestRelayedTransactionIntraShard(t *testing.T) { hex.EncodeToString(scrHash2): {SmartContractResult: scr2, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/scCallIntraShard_test.go b/integrationtests/scCallIntraShard_test.go index 30296ff7..7e7492dc 100644 --- a/integrationtests/scCallIntraShard_test.go +++ b/integrationtests/scCallIntraShard_test.go @@ -82,7 +82,7 @@ func TestTransactionWithSCCallFail(t *testing.T) { }, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} @@ -162,7 +162,7 @@ func TestTransactionWithScCallSuccess(t *testing.T) { }, FeeInfo: &outport.FeeInfo{}}, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/transactions_test.go b/integrationtests/transactions_test.go index 77b8b694..55ec3910 100644 --- a/integrationtests/transactions_test.go +++ b/integrationtests/transactions_test.go @@ -63,7 +63,7 @@ func TestElasticIndexerSaveTransactions(t *testing.T) { hex.EncodeToString(txHash): txInfo, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/updateNFT_test.go b/integrationtests/updateNFT_test.go index e6bfb226..17dff437 100644 --- a/integrationtests/updateNFT_test.go +++ b/integrationtests/updateNFT_test.go @@ -59,7 +59,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids := []string{"NFT-abcd-0e"} @@ -86,7 +86,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) // Add URIS 2 --- results should be the same @@ -107,7 +107,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) // Update attributes 1 @@ -134,7 +134,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids = []string{"NFT-abcd-0e"} @@ -162,7 +162,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids = []string{"NFT-abcd-0e"} @@ -189,7 +189,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids = []string{"NFT-abcd-0e"} genericResponse = &GenericResponse{} @@ -215,7 +215,7 @@ func TestNFTUpdateMetadata(t *testing.T) { }, }, } - err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, false, testNumOfShards)) + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) ids = []string{"NFT-abcd-0e"} genericResponse = &GenericResponse{} From d84c36c52dea1fc83bcc80f4cb5656ba7f04be6f Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 18 May 2023 15:49:40 +0300 Subject: [PATCH 069/189] fixes after merge --- go.sum | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go.sum b/go.sum index 502a0d50..750bf091 100644 --- a/go.sum +++ b/go.sum @@ -73,14 +73,14 @@ github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjW github.com/multiversx/mx-chain-communication-go v0.0.0-20230512095548-5bc637293104 h1:oFsYNkebv7TQygdEjN4aGgQ8ICLPmS9bDJmzlOHtU2Y= github.com/multiversx/mx-chain-communication-go v0.0.0-20230512095548-5bc637293104/go.mod h1:GPHOm4HSXbvC0IotMziWXQmhtsUe69ScBPYsb+mF9bk= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.0/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1-0.20230510143029-ab37792342df/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= +github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.4-0.20230517135533-2e54a17cd912 h1:7dgFuxG2sUbQEFA4y36NAoRjuN+Z3PlY0znmmgr7ZSo= github.com/multiversx/mx-chain-core-go v1.2.4-0.20230517135533-2e54a17cd912/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= -github.com/multiversx/mx-chain-vm-common-go v1.4.0 h1:0i0cJZJOXGzqYzwtKFHSr2yGmnFAdizOuISK8HgsnYo= -github.com/multiversx/mx-chain-vm-common-go v1.4.0/go.mod h1:odBJC92ANA8zLtPh/wwajUUGJOaS88F5QYGf0t8Wgzw= +github.com/multiversx/mx-chain-vm-common-go v1.4.1 h1:HHZF9zU4WsMbfLrCarx3ESM95caWUrPBleGHKdsbzgc= +github.com/multiversx/mx-chain-vm-common-go v1.4.1/go.mod h1:K6yCdro8VohzYI6GwjGzTO+fJiPgO5coo2sgQb+zA24= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= From b01cae831f38acfcefc0fa0f99ebdce9b31f99c4 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 19 May 2023 13:42:04 +0300 Subject: [PATCH 070/189] proper tags --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index a8573895..567c89a6 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 - github.com/multiversx/mx-chain-communication-go v0.0.0-20230512095548-5bc637293104 - github.com/multiversx/mx-chain-core-go v1.2.4-0.20230517135533-2e54a17cd912 + github.com/multiversx/mx-chain-communication-go v1.0.0 + github.com/multiversx/mx-chain-core-go v1.2.4 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.1 github.com/stretchr/testify v1.8.2 diff --git a/go.sum b/go.sum index 750bf091..94eb08da 100644 --- a/go.sum +++ b/go.sum @@ -70,13 +70,13 @@ github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiversx/mx-chain-communication-go v0.0.0-20230512095548-5bc637293104 h1:oFsYNkebv7TQygdEjN4aGgQ8ICLPmS9bDJmzlOHtU2Y= -github.com/multiversx/mx-chain-communication-go v0.0.0-20230512095548-5bc637293104/go.mod h1:GPHOm4HSXbvC0IotMziWXQmhtsUe69ScBPYsb+mF9bk= +github.com/multiversx/mx-chain-communication-go v1.0.0 h1:ZGIIrWIE7RqpF7gvMfshH+CJUehviXzkWlxnpZ02efE= +github.com/multiversx/mx-chain-communication-go v1.0.0/go.mod h1:GPHOm4HSXbvC0IotMziWXQmhtsUe69ScBPYsb+mF9bk= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1-0.20230510143029-ab37792342df/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.4-0.20230517135533-2e54a17cd912 h1:7dgFuxG2sUbQEFA4y36NAoRjuN+Z3PlY0znmmgr7ZSo= -github.com/multiversx/mx-chain-core-go v1.2.4-0.20230517135533-2e54a17cd912/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= +github.com/multiversx/mx-chain-core-go v1.2.4 h1:BRXyajUevLU6zHszR8jnp2+7C2bAQBor51YTc4dp3YQ= +github.com/multiversx/mx-chain-core-go v1.2.4/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= github.com/multiversx/mx-chain-vm-common-go v1.4.1 h1:HHZF9zU4WsMbfLrCarx3ESM95caWUrPBleGHKdsbzgc= From b11a612035cd50ba52bd6d86f6618aca73756865 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 25 May 2023 14:48:22 +0300 Subject: [PATCH 071/189] update go mod adn small changes in scripts --- go.mod | 8 +- go.sum | 890 +++++++++++++++++++++++++++++++++++- scripts/observers/.env | 2 +- scripts/observers/config.py | 5 +- 4 files changed, 892 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 567c89a6..0be3b866 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 - github.com/multiversx/mx-chain-communication-go v1.0.0 + github.com/multiversx/mx-chain-communication-go v1.0.2 github.com/multiversx/mx-chain-core-go v1.2.4 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.1 @@ -15,11 +15,12 @@ require ( require ( github.com/btcsuite/btcd/btcutil v1.1.3 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/denisbrodbeck/machineid v1.0.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.8 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect @@ -31,6 +32,7 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect golang.org/x/crypto v0.3.0 // indirect golang.org/x/sys v0.2.0 // indirect - google.golang.org/protobuf v1.26.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 94eb08da..e3abfc40 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,67 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= +dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= +dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= +filippo.io/edwards25519 v1.0.0/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= +git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= +github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M= +github.com/btcsuite/btcd v0.22.1/go.mod h1:wqgTSL29+50LRkmOVknEdmt8ZojIzhuWvgu/iptuN7Y= github.com/btcsuite/btcd v0.23.0 h1:V2/ZgjfDFIygAX3ZapeigkVBoVUtOJKSwrhZdlpSvaA= github.com/btcsuite/btcd v0.23.0/go.mod h1:0QJIIN1wwIXF/3G/m87gIwGniDMDQqjVn4SZgnFpsYY= github.com/btcsuite/btcd/btcec/v2 v2.1.0/go.mod h1:2VzYrv4Gm4apmbVVsSq5bqf1Ec8v56E48Vt0Y/umPgA= @@ -14,6 +74,7 @@ github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtyd github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= @@ -21,181 +82,996 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= +github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= +github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgraph-io/badger v1.6.1/go.mod h1:FRmFw3uxvcpa8zG3Rxs0th+hCLIuaQg8HlNV5bjgnuU= +github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elastic/go-elasticsearch/v7 v7.12.0 h1:j4tvcMrZJLp39L2NYvBb7f+lHKPqPHSL3nvB8+/DV+s= github.com/elastic/go-elasticsearch/v7 v7.12.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= +github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= +github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= +github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= +github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= +github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/herumi/bls-go-binary v1.0.0/go.mod h1:O4Vp1AfR4raRGwFeQpr9X/PQtncEicMoOe6BQt1oX0Y= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= +github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= +github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= +github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= +github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= +github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro= +github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= +github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= +github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= +github.com/ipfs/go-datastore v0.5.1/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= +github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= +github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= +github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk= +github.com/ipfs/go-ds-badger v0.3.0/go.mod h1:1ke6mXNqeV8K3y5Ak2bAA0osoTfmxUdupVCGm4QUIek= +github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= +github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= +github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= +github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= +github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= +github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= +github.com/ipfs/go-ipns v0.2.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24= +github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= +github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A= +github.com/ipfs/go-log v1.0.4/go.mod h1:oDCg2FkjogeFOhqqb+N39l2RpTNPL6F/StPkB3kPgcs= +github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= +github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= +github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= +github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= +github.com/ipfs/go-log/v2 v2.5.0/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= +github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= +github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= +github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= +github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= +github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= +github.com/koron/go-ssdp v0.0.3/go.mod h1:b2MxI6yh02pKrsyNoQUsk4+YNikaGhe4894J+Q5lDvA= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= +github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= +github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= +github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= +github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= +github.com/libp2p/go-flow-metrics v0.0.2/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= +github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= +github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= +github.com/libp2p/go-libp2p v0.22.0/go.mod h1:UDolmweypBSjQb2f7xutPnwZ/fxioLbMBxSjRksxxU4= +github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo= +github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= +github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= +github.com/libp2p/go-libp2p-core v0.2.5/go.mod h1:6+5zJmKhsf7yHn1RbmYDu08qDUpIUxGdqHuEZckmZOA= +github.com/libp2p/go-libp2p-core v0.3.0/go.mod h1:ACp3DmS3/N64c2jDzcV429ukDpicbL6+TrrxANBjPGw= +github.com/libp2p/go-libp2p-core v0.5.3/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y= +github.com/libp2p/go-libp2p-core v0.5.4/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y= +github.com/libp2p/go-libp2p-core v0.6.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= +github.com/libp2p/go-libp2p-core v0.19.0/go.mod h1:AkA+FUKQfYt1FLNef5fOPlo/naAWjKy/RCjkcPjqzYg= +github.com/libp2p/go-libp2p-core v0.20.0/go.mod h1:6zR8H7CvQWgYLsbG4on6oLNSGcyKaYFSEYyDt51+bIY= +github.com/libp2p/go-libp2p-kad-dht v0.18.0/go.mod h1:Gb92MYIPm3K2pJLGn8wl0m8wiKDvHrYpg+rOd0GzzPA= +github.com/libp2p/go-libp2p-kbucket v0.3.1/go.mod h1:oyjT5O7tS9CQurok++ERgc46YLwEpuGoFq9ubvoUOio= +github.com/libp2p/go-libp2p-kbucket v0.4.7/go.mod h1:XyVo99AfQH0foSf176k4jY1xUJ2+jUJIZCSDm7r2YKk= +github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs= +github.com/libp2p/go-libp2p-peerstore v0.2.6/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= +github.com/libp2p/go-libp2p-peerstore v0.8.0/go.mod h1:9geHWmNA3YDlQBjL/uPEJD6vpDK12aDNlUNHJ6kio/s= +github.com/libp2p/go-libp2p-pubsub v0.8.1/go.mod h1:e4kT+DYjzPUYGZeWk4I+oxCSYTXizzXii5LDRRhjKSw= +github.com/libp2p/go-libp2p-record v0.1.2/go.mod h1:pal0eNcT5nqZaTV7UGhqeGqxFgGdsU/9W//C8dqjQDk= +github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= +github.com/libp2p/go-libp2p-routing-helpers v0.2.3/go.mod h1:795bh+9YeoFl99rMASoiVgHdi5bjack0N1+AFAdbvBw= +github.com/libp2p/go-libp2p-testing v0.11.0/go.mod h1:qG4sF27dfKFoK9KlVzK2y52LQKhp0VEmLjV5aDqr1Hg= +github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= +github.com/libp2p/go-libp2p-xor v0.1.0/go.mod h1:LSTM5yRnjGZbWNTA/hRwq2gGFrvRIbQJscoIL/u6InY= +github.com/libp2p/go-maddr-filter v0.1.0/go.mod h1:VzZhTXkMucEGGEOSKddrwGiOv0tUhgnKqNEmIAz/bPU= +github.com/libp2p/go-mplex v0.7.0/go.mod h1:rW8ThnRcYWft/Jb2jeORBmPd6xuG3dGxWN/W168L9EU= +github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= +github.com/libp2p/go-msgio v0.0.6/go.mod h1:4ecVB6d9f4BDSL5fqvPiC4A3KivjWn+Venn/1ALLMWA= +github.com/libp2p/go-msgio v0.2.0/go.mod h1:dBVM1gW3Jk9XqHkU4eKdGvVHdLa51hoGfll6jMJMSlY= +github.com/libp2p/go-nat v0.1.0/go.mod h1:X7teVkwRHNInVNWQiO/tAiAVRwSr5zoRz4YSTC3uRBM= +github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= +github.com/libp2p/go-netroute v0.2.0/go.mod h1:Vio7LTzZ+6hoT4CMZi5/6CpY3Snzh2vgZhWgxMNwlQI= +github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= +github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= +github.com/libp2p/go-openssl v0.0.7/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= +github.com/libp2p/go-openssl v0.1.0/go.mod h1:OiOxwPpL3n4xlenjx2h7AwSGaFSC/KZvf6gNdOBQMtc= +github.com/libp2p/go-reuseport v0.2.0/go.mod h1:bvVho6eLMm6Bz5hmU0LYN3ixd3nPPvtIlaURZZgOY4k= +github.com/libp2p/go-sockaddr v0.0.2/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= +github.com/libp2p/go-yamux/v3 v3.1.2/go.mod h1:jeLEQgLXqE2YqX1ilAClIfCMDY+0uXQUKmmb/qp0gT4= +github.com/libp2p/zeroconf/v2 v2.2.0/go.mod h1:fuJqLnUwZTshS3U/bMRJ3+ow/v9oid1n0DmyYyNO1Xs= +github.com/lucas-clemente/quic-go v0.28.1/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0= +github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= +github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= +github.com/marten-seemann/qtls-go1-17 v0.1.2/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= +github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= +github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= +github.com/marten-seemann/qtls-go1-19 v0.1.0/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= +github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= +github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKoFL8DUUmalo2yJJUCxbPKtm8OKfqr2/FTNU= +github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiversx/mx-chain-communication-go v1.0.0 h1:ZGIIrWIE7RqpF7gvMfshH+CJUehviXzkWlxnpZ02efE= -github.com/multiversx/mx-chain-communication-go v1.0.0/go.mod h1:GPHOm4HSXbvC0IotMziWXQmhtsUe69ScBPYsb+mF9bk= +github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-base32 v0.0.4/go.mod h1:jNLFzjPZtp3aIARHbJRZIaPuspdH0J6q39uUM5pnABM= +github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= +github.com/multiformats/go-multiaddr v0.1.0/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= +github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= +github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= +github.com/multiformats/go-multiaddr v0.2.1/go.mod h1:s/Apk6IyxfvMjDafnhJgJ3/46z7tZ04iMk5wP4QMGGE= +github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u0xW5UouOmQQrn6a3Y= +github.com/multiformats/go-multiaddr v0.3.0/go.mod h1:dF9kph9wfJ+3VLAaeBqo9Of8x4fJxp6ggJGteB8HQTI= +github.com/multiformats/go-multiaddr v0.3.1/go.mod h1:uPbspcUPd5AfaP6ql3ujFY+QWzmBD8uLLL4bXW0XfGc= +github.com/multiformats/go-multiaddr v0.4.1/go.mod h1:3afI9HfVW8csiF8UZqtpYRiDyew8pRX7qLIGHu9FLuM= +github.com/multiformats/go-multiaddr v0.6.0/go.mod h1:F4IpaKZuPP360tOMn2Tpyu0At8w23aRyVqeK0DbFeGM= +github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= +github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= +github.com/multiformats/go-multiaddr-net v0.1.1/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= +github.com/multiformats/go-multiaddr-net v0.1.4/go.mod h1:ilNnaM9HbmVFqsb/qcNysjCu4PVONlrBZpHIrw/qQuA= +github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5rnK7YaOIEOPVDI9tsJbEA= +github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= +github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= +github.com/multiformats/go-multicodec v0.4.1/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= +github.com/multiformats/go-multicodec v0.5.0/go.mod h1:DiY2HFaEp5EhEXb/iYzVAunmyX/aSFMxq2KMKfWEues= +github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= +github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= +github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= +github.com/multiformats/go-multistream v0.3.3/go.mod h1:ODRoqamLUsETKS9BNcII4gcRsJBU5VAwRIv7O39cEXg= +github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= +github.com/multiversx/mx-chain-communication-go v1.0.2 h1:1AKdqFZNmigt1kcwYMl+L8fzolsb+WpeTX6yzpmvbV4= +github.com/multiversx/mx-chain-communication-go v1.0.2/go.mod h1:OnnSUjnSP87H5MtQtxn33FGnTVRVgo2Huo3ijmCgN2M= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1-0.20230510143029-ab37792342df/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.4 h1:BRXyajUevLU6zHszR8jnp2+7C2bAQBor51YTc4dp3YQ= github.com/multiversx/mx-chain-core-go v1.2.4/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= +github.com/multiversx/mx-chain-crypto-go v1.2.6/go.mod h1:rOj0Rr19HTOYt9YTeym7RKxlHt91NXln3LVKjHKVmA0= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= +github.com/multiversx/mx-chain-storage-go v1.0.8/go.mod h1:lEkFYFe6taKYxqO1einNgT1esY3K9Pj6vPnoCwV9C3U= github.com/multiversx/mx-chain-vm-common-go v1.4.1 h1:HHZF9zU4WsMbfLrCarx3ESM95caWUrPBleGHKdsbzgc= github.com/multiversx/mx-chain-vm-common-go v1.4.1/go.mod h1:K6yCdro8VohzYI6GwjGzTO+fJiPgO5coo2sgQb+zA24= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= +github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= +github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= +github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= +github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI= +github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU= +github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag= +github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg= +github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= +github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= +github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= +github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= +github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= +github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0= +github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= +github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= +github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= +github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= +github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= +github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= +github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.10 h1:p8Fspmz3iTctJstry1PYS3HVdllxnEzTEsgIgtxTrCk= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= +github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= +github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE= +github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= +github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= +github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= +github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= +go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U= +go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= +gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/scripts/observers/.env b/scripts/observers/.env index 5418c53d..d3627aa7 100644 --- a/scripts/observers/.env +++ b/scripts/observers/.env @@ -2,7 +2,7 @@ MX_CHAIN_DEPLOY_GO_URL=https://github.com/multiversx/mx-chain-deploy-go MX_CHAIN_PROXY_URL=https://github.com/multiversx/mx-chain-proxy-go NODE_GO_URL="https://github.com/multiversx/mx-chain-go" -NODE_GO_BRANCH="integrate-new-ws-client-server" +NODE_GO_BRANCH="rc/v1.6.0" WORKING_DIRECTORY="IndexerObservers" OBSERVER_DIR_PREFIX="observer_shard_" diff --git a/scripts/observers/config.py b/scripts/observers/config.py index 037cbce7..ff4c8c6d 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -249,11 +249,12 @@ def main(): # build binary mx-chain-go print("building node...") mx_chain_go_folder = working_dir / "mx-chain-go" - subprocess.check_call(["go", "build"], cwd=mx_chain_go_folder / "cmd/node") + flags = '-gcflags="all=-N -l"' + subprocess.check_call(["go", "build", flags], cwd=mx_chain_go_folder / "cmd/node") # build binary indexer print("building indexer...") - subprocess.check_call(["go", "build"], cwd="../../cmd/elasticindexer") + subprocess.check_call(["go", "build", flags], cwd="../../cmd/elasticindexer") # prepare observers config_folder = working_dir / "config" From b476c3e63393184a384db2a9efb212792fd5bf55 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 25 May 2023 15:33:56 +0300 Subject: [PATCH 072/189] Makefile --- scripts/observers/Makefile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 scripts/observers/Makefile diff --git a/scripts/observers/Makefile b/scripts/observers/Makefile new file mode 100644 index 00000000..61f9f708 --- /dev/null +++ b/scripts/observers/Makefile @@ -0,0 +1,16 @@ +# Help section +help: + @echo "Available targets:" + @echo " config - Configure the local testnet" + @echo " start - Start the local testnet" + @echo " stop - Stop the local testnet" + @echo " clean - Clean up the files of the local testnet" + +config: + python3 config.py +start: + python3 start.py +stop: + python3 stop.py +clean: + python3 clean.py From ebcbc1e5047c72c37b3302e05e393ecc160d929f Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 6 Jun 2023 16:48:22 +0300 Subject: [PATCH 073/189] remove data dispatcher --- mock/dispatcherStub.go | 39 ---- process/dataindexer/dataDispatcher.go | 197 ----------------- process/dataindexer/dataDispatcher_test.go | 209 ------------------ process/dataindexer/dataIndexer.go | 116 ++++++---- process/dataindexer/dataIndexer_test.go | 82 ++++--- process/dataindexer/interface.go | 9 - process/dataindexer/workItems/interface.go | 42 ---- .../dataindexer/workItems/workItemAccounts.go | 43 ---- .../workItems/workItemAccounts_test.go | 45 ---- .../dataindexer/workItems/workItemBlock.go | 90 -------- .../workItems/workItemBlock_test.go | 158 ------------- .../dataindexer/workItems/workItemRating.go | 34 --- .../workItems/workItemRating_test.go | 45 ---- .../workItems/workItemRemoveBlock.go | 50 ----- .../workItems/workItemRemoveBlock_test.go | 73 ------ .../dataindexer/workItems/workItemRounds.go | 34 --- .../workItems/workItemRounds_test.go | 45 ---- .../workItems/workItemValidators.go | 29 --- .../workItems/workItemValidators_test.go | 45 ---- 19 files changed, 118 insertions(+), 1267 deletions(-) delete mode 100644 mock/dispatcherStub.go delete mode 100644 process/dataindexer/dataDispatcher.go delete mode 100644 process/dataindexer/dataDispatcher_test.go delete mode 100644 process/dataindexer/workItems/interface.go delete mode 100644 process/dataindexer/workItems/workItemAccounts.go delete mode 100644 process/dataindexer/workItems/workItemAccounts_test.go delete mode 100644 process/dataindexer/workItems/workItemBlock.go delete mode 100644 process/dataindexer/workItems/workItemBlock_test.go delete mode 100644 process/dataindexer/workItems/workItemRating.go delete mode 100644 process/dataindexer/workItems/workItemRating_test.go delete mode 100644 process/dataindexer/workItems/workItemRemoveBlock.go delete mode 100644 process/dataindexer/workItems/workItemRemoveBlock_test.go delete mode 100644 process/dataindexer/workItems/workItemRounds.go delete mode 100644 process/dataindexer/workItems/workItemRounds_test.go delete mode 100644 process/dataindexer/workItems/workItemValidators.go delete mode 100644 process/dataindexer/workItems/workItemValidators_test.go diff --git a/mock/dispatcherStub.go b/mock/dispatcherStub.go deleted file mode 100644 index 3dd089cc..00000000 --- a/mock/dispatcherStub.go +++ /dev/null @@ -1,39 +0,0 @@ -package mock - -import ( - "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" -) - -// DispatcherMock - -type DispatcherMock struct { - StartIndexDataCalled func() - CloseCalled func() error - AddCalled func(item workItems.WorkItemHandler) -} - -// StartIndexData - -func (dm *DispatcherMock) StartIndexData() { - if dm.StartIndexDataCalled != nil { - dm.StartIndexDataCalled() - } -} - -// Close - -func (dm *DispatcherMock) Close() error { - if dm.CloseCalled != nil { - return dm.CloseCalled() - } - return nil -} - -// Add - -func (dm *DispatcherMock) Add(item workItems.WorkItemHandler) { - if dm.AddCalled != nil { - dm.AddCalled(item) - } -} - -// IsInterfaceNil returns true if there is no value under the interface -func (dm *DispatcherMock) IsInterfaceNil() bool { - return dm == nil -} diff --git a/process/dataindexer/dataDispatcher.go b/process/dataindexer/dataDispatcher.go deleted file mode 100644 index 55c4d51d..00000000 --- a/process/dataindexer/dataDispatcher.go +++ /dev/null @@ -1,197 +0,0 @@ -package dataindexer - -import ( - "context" - "errors" - "runtime/debug" - "sync" - "time" - - "github.com/multiversx/mx-chain-core-go/core/atomic" - "github.com/multiversx/mx-chain-core-go/core/check" - "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" - logger "github.com/multiversx/mx-chain-logger-go" -) - -var log = logger.GetOrCreate("indexer") - -const ( - durationBetweenErrorRetry = time.Second * 3 - closeTimeout = time.Second * 20 - backOffTime = time.Second * 10 - maxBackOff = time.Minute * 5 -) - -type dataDispatcher struct { - backOffTime time.Duration - chanWorkItems chan workItems.WorkItemHandler - cancelFunc func() - wasClosed *atomic.Flag - currentWriteDone chan struct{} - closeStartTime time.Time - mutexCloseStartTime sync.RWMutex -} - -// NewDataDispatcher creates a new dataDispatcher instance, capable of saving sequentially data in elasticsearch database -func NewDataDispatcher(cacheSize int) (*dataDispatcher, error) { - if cacheSize < 0 { - return nil, ErrNegativeCacheSize - } - - dd := &dataDispatcher{ - chanWorkItems: make(chan workItems.WorkItemHandler, cacheSize), - wasClosed: &atomic.Flag{}, - currentWriteDone: make(chan struct{}), - mutexCloseStartTime: sync.RWMutex{}, - } - - return dd, nil -} - -// StartIndexData will start index data in database -func (d *dataDispatcher) StartIndexData() { - var ctx context.Context - ctx, d.cancelFunc = context.WithCancel(context.Background()) - - go d.doDataDispatch(ctx) -} - -func (d *dataDispatcher) doDataDispatch(ctx context.Context) { - defer func() { - r := recover() - if r != nil { - log.Error("d.doDataDispatch", - "message", r, - "panic", string(debug.Stack())) - panic(r) - } - }() - - for { - select { - case <-ctx.Done(): - d.stopWorker() - return - case wi := <-d.chanWorkItems: - timeoutOnClose := d.doWork(wi) - if timeoutOnClose { - d.stopWorker() - return - } - } - } -} - -func (d *dataDispatcher) stopWorker() { - log.Debug("dispatcher's go routine is stopping...") - d.currentWriteDone <- struct{}{} -} - -// Close will close the endless running go routine -func (d *dataDispatcher) Close() error { - previousState := d.wasClosed.SetReturningPrevious() - if previousState { - return nil - } - - d.mutexCloseStartTime.Lock() - d.closeStartTime = time.Now() - d.mutexCloseStartTime.Unlock() - - if d.cancelFunc != nil { - d.cancelFunc() - } - - <-d.currentWriteDone - d.consumeRemainingItems() - return nil -} - -func (d *dataDispatcher) consumeRemainingItems() { - for { - select { - case wi := <-d.chanWorkItems: - isTimeout := d.doWork(wi) - if isTimeout { - return - } - default: - return - } - } -} - -// Add will add a new item in queue -func (d *dataDispatcher) Add(item workItems.WorkItemHandler) { - if check.IfNil(item) { - log.Warn("dataDispatcher.Add nil item: will do nothing") - return - } - if d.wasClosed.IsSet() { - log.Warn("dataDispatcher.Add cannot add item: channel chanWorkItems is closed") - return - } - - d.chanWorkItems <- item -} - -func (d *dataDispatcher) doWork(wi workItems.WorkItemHandler) bool { - for { - if d.exitIfTimeoutOnClose() { - log.Warn("dataDispatcher.doWork could not index item", - "error", "timeout") - return true - } - - err := wi.Save() - if errors.Is(err, ErrBackOff) { - log.Warn("dataDispatcher.doWork could not index item", - "received back off:", err.Error()) - - d.increaseBackOffTime() - time.Sleep(d.backOffTime) - - continue - } - - d.backOffTime = 0 - if err != nil { - log.Warn("dataDispatcher.doWork could not index item (will retry)", "error", err.Error()) - time.Sleep(durationBetweenErrorRetry) - - continue - } - - return false - } - -} - -func (d *dataDispatcher) exitIfTimeoutOnClose() bool { - if !d.wasClosed.IsSet() { - return false - } - - d.mutexCloseStartTime.RLock() - passedTime := time.Since(d.closeStartTime) - d.mutexCloseStartTime.RUnlock() - - return passedTime > closeTimeout -} - -func (d *dataDispatcher) increaseBackOffTime() { - if d.backOffTime == 0 { - d.backOffTime = backOffTime - return - } - if d.backOffTime >= maxBackOff { - return - } - - d.backOffTime += d.backOffTime / 5 -} - -// IsInterfaceNil returns true if there is no value under the interface -func (d *dataDispatcher) IsInterfaceNil() bool { - return d == nil -} diff --git a/process/dataindexer/dataDispatcher_test.go b/process/dataindexer/dataDispatcher_test.go deleted file mode 100644 index fb7d3c2b..00000000 --- a/process/dataindexer/dataDispatcher_test.go +++ /dev/null @@ -1,209 +0,0 @@ -package dataindexer - -import ( - "context" - "errors" - "fmt" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/multiversx/mx-chain-core-go/data/outport" - "github.com/multiversx/mx-chain-es-indexer-go/mock" - "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" - "github.com/stretchr/testify/require" -) - -func TestNewDataDispatcher_InvalidCacheSize(t *testing.T) { - t.Parallel() - - dataDist, err := NewDataDispatcher(-1) - - require.Nil(t, dataDist) - require.Equal(t, ErrNegativeCacheSize, err) -} - -func TestNewDataDispatcher(t *testing.T) { - t.Parallel() - - dispatcher, err := NewDataDispatcher(100) - require.NoError(t, err) - require.NotNil(t, dispatcher) -} - -func TestDataDispatcher_StartIndexDataClose(t *testing.T) { - t.Parallel() - - dispatcher, err := NewDataDispatcher(100) - require.NoError(t, err) - dispatcher.StartIndexData() - - called := false - wg := sync.WaitGroup{} - wg.Add(1) - elasticProc := &mock.ElasticProcessorStub{ - SaveRoundsInfoCalled: func(infos *outport.RoundsInfo) error { - called = true - wg.Done() - return nil - }, - SaveAccountsCalled: func(accounts *outport.Accounts) error { - time.Sleep(7 * time.Second) - return nil - }, - - SaveValidatorsRatingCalled: func(_ *outport.ValidatorsRating) error { - time.Sleep(6 * time.Second) - return nil - }, - } - dispatcher.Add(workItems.NewItemRounds(elasticProc, &outport.RoundsInfo{})) - wg.Wait() - - require.True(t, called) - - dispatcher.Add(workItems.NewItemAccounts(elasticProc, &outport.Accounts{})) - wg.Add(1) - dispatcher.Add(workItems.NewItemRounds(elasticProc, &outport.RoundsInfo{})) - dispatcher.Add(workItems.NewItemRating(elasticProc, &outport.ValidatorsRating{})) - wg.Add(1) - dispatcher.Add(workItems.NewItemRounds(elasticProc, &outport.RoundsInfo{})) - err = dispatcher.Close() - require.NoError(t, err) -} - -func TestDataDispatcher_Add(t *testing.T) { - t.Parallel() - - dispatcher, err := NewDataDispatcher(100) - require.NoError(t, err) - dispatcher.StartIndexData() - - calledCount := uint32(0) - wg := sync.WaitGroup{} - wg.Add(1) - elasticProc := &mock.ElasticProcessorStub{ - SaveRoundsInfoCalled: func(_ *outport.RoundsInfo) error { - if calledCount < 2 { - atomic.AddUint32(&calledCount, 1) - return fmt.Errorf("%w: wrapped error", ErrBackOff) - } - - atomic.AddUint32(&calledCount, 1) - wg.Done() - return nil - }, - } - - start := time.Now() - dispatcher.Add(workItems.NewItemRounds(elasticProc, &outport.RoundsInfo{})) - wg.Wait() - - timePassed := time.Since(start) - require.Greater(t, 2*int64(timePassed), int64(backOffTime)) - - require.Equal(t, uint32(3), atomic.LoadUint32(&calledCount)) - - err = dispatcher.Close() - require.NoError(t, err) -} - -func TestDataDispatcher_AddWithErrorShouldRetryTheReprocessing(t *testing.T) { - t.Parallel() - - dispatcher, err := NewDataDispatcher(100) - require.NoError(t, err) - dispatcher.StartIndexData() - - calledCount := uint32(0) - wg := sync.WaitGroup{} - wg.Add(1) - elasticProc := &mock.ElasticProcessorStub{ - SaveRoundsInfoCalled: func(_ *outport.RoundsInfo) error { - if calledCount < 2 { - atomic.AddUint32(&calledCount, 1) - return errors.New("generic error") - } - - atomic.AddUint32(&calledCount, 1) - wg.Done() - return nil - }, - } - - start := time.Now() - dispatcher.Add(workItems.NewItemRounds(elasticProc, &outport.RoundsInfo{})) - wg.Wait() - - timePassed := time.Since(start) - require.Greater(t, int64(timePassed), int64(2*durationBetweenErrorRetry)) - - require.Equal(t, uint32(3), atomic.LoadUint32(&calledCount)) - - err = dispatcher.Close() - require.NoError(t, err) -} - -func TestDataDispatcher_Close(t *testing.T) { - t.Parallel() - - dispatcher, err := NewDataDispatcher(100) - require.NoError(t, err) - dispatcher.StartIndexData() - - elasticProc := &mock.ElasticProcessorStub{ - SaveRoundsInfoCalled: func(_ *outport.RoundsInfo) error { - time.Sleep(1000*time.Millisecond + 200*time.Microsecond) - return nil - }, - } - - wg := &sync.WaitGroup{} - wg.Add(1) - ctx, cancelFunc := context.WithCancel(context.Background()) - go func(c context.Context, w *sync.WaitGroup) { - count := 0 - for { - select { - case <-c.Done(): - return - default: - count++ - if count == 105 { - w.Done() - } - dispatcher.Add(workItems.NewItemRounds(elasticProc, &outport.RoundsInfo{})) - time.Sleep(50 * time.Millisecond) - } - } - }(ctx, wg) - - wg.Wait() - - err = dispatcher.Close() - require.NoError(t, err) - - cancelFunc() -} - -func TestDataDispatcher_RecoverPanic(t *testing.T) { - t.Parallel() - - defer func() { - r := recover() - require.NotNil(t, r) - }() - - dispatcher, err := NewDataDispatcher(100) - require.NoError(t, err) - - elasticProc := &mock.ElasticProcessorStub{ - SaveRoundsInfoCalled: func(_ *outport.RoundsInfo) error { - panic(1) - }, - } - - dispatcher.Add(workItems.NewItemRounds(elasticProc, &outport.RoundsInfo{})) - dispatcher.doDataDispatch(context.Background()) -} diff --git a/process/dataindexer/dataIndexer.go b/process/dataindexer/dataIndexer.go index a2eab46d..49e20dcf 100644 --- a/process/dataindexer/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -1,26 +1,29 @@ package dataindexer import ( + "encoding/hex" + "fmt" + "time" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/marshal" - "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" + logger "github.com/multiversx/mx-chain-logger-go" ) +var log = logger.GetOrCreate("dataindexer") + // ArgDataIndexer is a structure that is used to store all the components that are needed to create an indexer type ArgDataIndexer struct { HeaderMarshaller marshal.Marshalizer - DataDispatcher DispatcherHandler ElasticProcessor ElasticProcessor BlockContainer BlockContainerHandler } type dataIndexer struct { - isNilIndexer bool - dispatcher DispatcherHandler elasticProcessor ElasticProcessor headerMarshaller marshal.Marshalizer blockContainer BlockContainerHandler @@ -34,8 +37,6 @@ func NewDataIndexer(arguments ArgDataIndexer) (*dataIndexer, error) { } dataIndexerObj := &dataIndexer{ - isNilIndexer: false, - dispatcher: arguments.DataDispatcher, elasticProcessor: arguments.ElasticProcessor, headerMarshaller: arguments.HeaderMarshaller, blockContainer: arguments.BlockContainer, @@ -45,9 +46,6 @@ func NewDataIndexer(arguments ArgDataIndexer) (*dataIndexer, error) { } func checkIndexerArgs(arguments ArgDataIndexer) error { - if check.IfNil(arguments.DataDispatcher) { - return ErrNilDataDispatcher - } if check.IfNil(arguments.ElasticProcessor) { return ErrNilElasticProcessor } @@ -77,21 +75,55 @@ func (di *dataIndexer) SaveBlock(outportBlock *outport.OutportBlock) error { return err } - wi := workItems.NewItemBlock( - di.elasticProcessor, - &outport.OutportBlockWithHeader{ - OutportBlock: outportBlock, - Header: header, - }, - ) - di.dispatcher.Add(wi) + headerHash := outportBlock.BlockData.HeaderHash + shardID := header.GetShardID() + headerNonce := header.GetNonce() + defer func(startTime time.Time, headerHash []byte, headerNonce uint64, shardID uint32) { + log.Debug("di.SaveBlockData", + "duration", time.Since(startTime), + "shardID", shardID, + "nonce", headerNonce, + "hash", headerHash, + ) + }(time.Now(), headerHash, headerNonce, shardID) + log.Debug("indexer: starting indexing block", "hash", headerHash, "nonce", headerNonce) + + if outportBlock.TransactionPool == nil { + outportBlock.TransactionPool = &outport.TransactionPool{} + } + + outportBlockWithHeader := &outport.OutportBlockWithHeader{ + OutportBlock: outportBlock, + Header: header, + } + err = di.elasticProcessor.SaveHeader(outportBlockWithHeader) + if err != nil { + return fmt.Errorf("%w when saving header block, hash %s, nonce %d", + err, hex.EncodeToString(headerHash), headerNonce) + } + + if len(outportBlock.BlockData.Body.MiniBlocks) == 0 { + return nil + } + + err = di.elasticProcessor.SaveMiniblocks(header, outportBlock.BlockData.Body) + if err != nil { + return fmt.Errorf("%w when saving miniblocks, block hash %s, nonce %d", + err, hex.EncodeToString(headerHash), headerNonce) + } + + err = di.elasticProcessor.SaveTransactions(outportBlockWithHeader) + if err != nil { + return fmt.Errorf("%w when saving transactions, block hash %s, nonce %d", + err, hex.EncodeToString(headerHash), headerNonce) + } return nil } // Close will stop goroutine that index data in database func (di *dataIndexer) Close() error { - return di.dispatcher.Close() + return nil } // RevertIndexedBlock will remove from database block and miniblocks @@ -101,52 +133,42 @@ func (di *dataIndexer) RevertIndexedBlock(blockData *outport.BlockData) error { return err } - wi := workItems.NewItemRemoveBlock( - di.elasticProcessor, - header, - blockData.Body, - ) - di.dispatcher.Add(wi) + err = di.elasticProcessor.RemoveHeader(header) + if err != nil { + return err + } - return nil + err = di.elasticProcessor.RemoveMiniblocks(header, blockData.Body) + if err != nil { + return err + } + + err = di.elasticProcessor.RemoveTransactions(header, blockData.Body) + if err != nil { + return err + } + + return di.elasticProcessor.RemoveAccountsESDT(header.GetTimeStamp(), header.GetShardID()) } // SaveRoundsInfo will save data about a slice of rounds in elasticsearch func (di *dataIndexer) SaveRoundsInfo(rounds *outport.RoundsInfo) error { - wi := workItems.NewItemRounds(di.elasticProcessor, rounds) - di.dispatcher.Add(wi) - - return nil + return di.elasticProcessor.SaveRoundsInfo(rounds) } // SaveValidatorsRating will save all validators rating info to elasticsearch func (di *dataIndexer) SaveValidatorsRating(ratingData *outport.ValidatorsRating) error { - wi := workItems.NewItemRating( - di.elasticProcessor, - ratingData, - ) - di.dispatcher.Add(wi) - - return nil + return di.elasticProcessor.SaveValidatorsRating(ratingData) } // SaveValidatorsPubKeys will save all validators public keys to elasticsearch func (di *dataIndexer) SaveValidatorsPubKeys(validatorsPubKeys *outport.ValidatorsPubKeys) error { - wi := workItems.NewItemValidators( - di.elasticProcessor, - validatorsPubKeys, - ) - di.dispatcher.Add(wi) - - return nil + return di.elasticProcessor.SaveShardValidatorsPubKeys(validatorsPubKeys) } // SaveAccounts will save the provided accounts func (di *dataIndexer) SaveAccounts(accounts *outport.Accounts) error { - wi := workItems.NewItemAccounts(di.elasticProcessor, accounts) - di.dispatcher.Add(wi) - - return nil + return di.elasticProcessor.SaveAccounts(accounts) } // FinalizedBlock returns nil diff --git a/process/dataindexer/dataIndexer_test.go b/process/dataindexer/dataIndexer_test.go index ce07eab2..ca91516e 100644 --- a/process/dataindexer/dataIndexer_test.go +++ b/process/dataindexer/dataIndexer_test.go @@ -5,31 +5,21 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" + coreData "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-es-indexer-go/mock" - "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" "github.com/stretchr/testify/require" ) func NewDataIndexerArguments() ArgDataIndexer { return ArgDataIndexer{ - DataDispatcher: &mock.DispatcherMock{}, ElasticProcessor: &mock.ElasticProcessorStub{}, HeaderMarshaller: &mock.MarshalizerMock{}, BlockContainer: &mock.BlockContainerStub{}, } } -func TestDataIndexer_NewIndexerWithNilDataDispatcherShouldErr(t *testing.T) { - arguments := NewDataIndexerArguments() - arguments.DataDispatcher = nil - ei, err := NewDataIndexer(arguments) - - require.Nil(t, ei) - require.Equal(t, ErrNilDataDispatcher, err) -} - func TestDataIndexer_NewIndexerWithNilElasticProcessorShouldErr(t *testing.T) { arguments := NewDataIndexerArguments() arguments.ElasticProcessor = nil @@ -58,19 +48,29 @@ func TestDataIndexer_NewIndexerWithCorrectParamsShouldWork(t *testing.T) { } func TestDataIndexer_SaveBlock(t *testing.T) { - called := false + count := 0 arguments := NewDataIndexerArguments() - arguments.DataDispatcher = &mock.DispatcherMock{ - AddCalled: func(item workItems.WorkItemHandler) { - called = true - }, - } arguments.BlockContainer = &mock.BlockContainerStub{ GetCalled: func(headerType core.HeaderType) (dataBlock.EmptyBlockCreator, error) { return dataBlock.NewEmptyHeaderV2Creator(), nil }, } + + arguments.ElasticProcessor = &mock.ElasticProcessorStub{ + SaveHeaderCalled: func(outportBlockWithHeader *outport.OutportBlockWithHeader) error { + count++ + return nil + }, + SaveMiniblocksCalled: func(header coreData.HeaderHandler, body *dataBlock.Body) error { + count++ + return nil + }, + SaveTransactionsCalled: func(outportBlockWithHeader *outport.OutportBlockWithHeader) error { + count++ + return nil + }, + } ei, _ := NewDataIndexer(arguments) args := &outport.OutportBlock{ @@ -82,20 +82,21 @@ func TestDataIndexer_SaveBlock(t *testing.T) { } err := ei.SaveBlock(args) require.Nil(t, err) - require.True(t, called) + require.Equal(t, 3, count) } func TestDataIndexer_SaveRoundInfo(t *testing.T) { called := false arguments := NewDataIndexerArguments() - arguments.DataDispatcher = &mock.DispatcherMock{ - AddCalled: func(item workItems.WorkItemHandler) { + + arguments.HeaderMarshaller = &mock.MarshalizerMock{Fail: true} + arguments.ElasticProcessor = &mock.ElasticProcessorStub{ + SaveRoundsInfoCalled: func(infos *outport.RoundsInfo) error { called = true + return nil }, } - - arguments.HeaderMarshaller = &mock.MarshalizerMock{Fail: true} ei, _ := NewDataIndexer(arguments) _ = ei.Close() @@ -108,9 +109,10 @@ func TestDataIndexer_SaveValidatorsPubKeys(t *testing.T) { called := false arguments := NewDataIndexerArguments() - arguments.DataDispatcher = &mock.DispatcherMock{ - AddCalled: func(item workItems.WorkItemHandler) { + arguments.ElasticProcessor = &mock.ElasticProcessorStub{ + SaveShardValidatorsPubKeysCalled: func(validators *outport.ValidatorsPubKeys) error { called = true + return nil }, } ei, _ := NewDataIndexer(arguments) @@ -129,9 +131,10 @@ func TestDataIndexer_SaveValidatorsRating(t *testing.T) { called := false arguments := NewDataIndexerArguments() - arguments.DataDispatcher = &mock.DispatcherMock{ - AddCalled: func(item workItems.WorkItemHandler) { + arguments.ElasticProcessor = &mock.ElasticProcessorStub{ + SaveValidatorsRatingCalled: func(validatorsRating *outport.ValidatorsRating) error { called = true + return nil }, } ei, _ := NewDataIndexer(arguments) @@ -142,18 +145,31 @@ func TestDataIndexer_SaveValidatorsRating(t *testing.T) { } func TestDataIndexer_RevertIndexedBlock(t *testing.T) { - called := false + count := 0 arguments := NewDataIndexerArguments() - arguments.DataDispatcher = &mock.DispatcherMock{ - AddCalled: func(item workItems.WorkItemHandler) { - called = true - }, - } arguments.BlockContainer = &mock.BlockContainerStub{ GetCalled: func(headerType core.HeaderType) (dataBlock.EmptyBlockCreator, error) { return dataBlock.NewEmptyHeaderV2Creator(), nil }} + arguments.ElasticProcessor = &mock.ElasticProcessorStub{ + RemoveHeaderCalled: func(header coreData.HeaderHandler) error { + count++ + return nil + }, + RemoveMiniblocksCalled: func(header coreData.HeaderHandler, body *dataBlock.Body) error { + count++ + return nil + }, + RemoveTransactionsCalled: func(header coreData.HeaderHandler, body *dataBlock.Body) error { + count++ + return nil + }, + RemoveAccountsESDTCalled: func(headerTimestamp uint64) error { + count++ + return nil + }, + } ei, _ := NewDataIndexer(arguments) err := ei.RevertIndexedBlock(&outport.BlockData{ @@ -162,5 +178,5 @@ func TestDataIndexer_RevertIndexedBlock(t *testing.T) { HeaderBytes: []byte("{}"), }) require.Nil(t, err) - require.True(t, called) + require.Equal(t, 4, count) } diff --git a/process/dataindexer/interface.go b/process/dataindexer/interface.go index 31a87cc7..08bdb000 100644 --- a/process/dataindexer/interface.go +++ b/process/dataindexer/interface.go @@ -8,17 +8,8 @@ import ( "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/marshal" - "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" ) -// DispatcherHandler defines the interface for the dispatcher that will manage when items are saved in elasticsearch database -type DispatcherHandler interface { - StartIndexData() - Close() error - Add(item workItems.WorkItemHandler) - IsInterfaceNil() bool -} - // ElasticProcessor defines the interface for the elastic search indexer type ElasticProcessor interface { SaveHeader(outportBlockWithHeader *outport.OutportBlockWithHeader) error diff --git a/process/dataindexer/workItems/interface.go b/process/dataindexer/workItems/interface.go deleted file mode 100644 index f6be410c..00000000 --- a/process/dataindexer/workItems/interface.go +++ /dev/null @@ -1,42 +0,0 @@ -package workItems - -import ( - coreData "github.com/multiversx/mx-chain-core-go/data" - "github.com/multiversx/mx-chain-core-go/data/block" - "github.com/multiversx/mx-chain-core-go/data/outport" -) - -// WorkItemHandler defines the interface for item that needs to be saved in elasticsearch database -type WorkItemHandler interface { - Save() error - IsInterfaceNil() bool -} - -type saveBlockIndexer interface { - SaveHeader(outportBlockWithHeader *outport.OutportBlockWithHeader) error - SaveMiniblocks(header coreData.HeaderHandler, body *block.Body) error - SaveTransactions(outportBlockWithHeader *outport.OutportBlockWithHeader) error -} - -type saveRatingIndexer interface { - SaveValidatorsRating(ratingData *outport.ValidatorsRating) error -} - -type removeIndexer interface { - RemoveHeader(header coreData.HeaderHandler) error - RemoveMiniblocks(header coreData.HeaderHandler, body *block.Body) error - RemoveTransactions(header coreData.HeaderHandler, body *block.Body) error - RemoveAccountsESDT(headerTimestamp uint64, shardID uint32) error -} - -type saveRounds interface { - SaveRoundsInfo(rounds *outport.RoundsInfo) error -} - -type saveValidatorsIndexer interface { - SaveShardValidatorsPubKeys(validatorsPubKeys *outport.ValidatorsPubKeys) error -} - -type saveAccountsIndexer interface { - SaveAccounts(accounts *outport.Accounts) error -} diff --git a/process/dataindexer/workItems/workItemAccounts.go b/process/dataindexer/workItems/workItemAccounts.go deleted file mode 100644 index 54888d20..00000000 --- a/process/dataindexer/workItems/workItemAccounts.go +++ /dev/null @@ -1,43 +0,0 @@ -package workItems - -import ( - "github.com/multiversx/mx-chain-core-go/data/outport" -) - -type itemAccounts struct { - indexer saveAccountsIndexer - accounts *outport.Accounts -} - -// NewItemAccounts will create a new instance of itemAccounts -func NewItemAccounts( - indexer saveAccountsIndexer, - accounts *outport.Accounts, -) WorkItemHandler { - return &itemAccounts{ - indexer: indexer, - accounts: accounts, - } -} - -// Save will save information about an account -func (wiv *itemAccounts) Save() error { - if wiv.accounts == nil { - return nil - } - - err := wiv.indexer.SaveAccounts(wiv.accounts) - if err != nil { - log.Warn("itemAccounts.Save", - "could not index account", - "error", err.Error()) - return err - } - - return nil -} - -// IsInterfaceNil returns true if there is no value under the interface -func (wiv *itemAccounts) IsInterfaceNil() bool { - return wiv == nil -} diff --git a/process/dataindexer/workItems/workItemAccounts_test.go b/process/dataindexer/workItems/workItemAccounts_test.go deleted file mode 100644 index 4f33d90a..00000000 --- a/process/dataindexer/workItems/workItemAccounts_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package workItems_test - -import ( - "errors" - "testing" - - "github.com/multiversx/mx-chain-core-go/data/outport" - "github.com/multiversx/mx-chain-es-indexer-go/mock" - "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" - "github.com/stretchr/testify/require" -) - -func TestItemAccounts_Save(t *testing.T) { - called := false - itemAccounts := workItems.NewItemAccounts( - &mock.ElasticProcessorStub{ - SaveAccountsCalled: func(_ *outport.Accounts) error { - called = true - return nil - }, - }, - &outport.Accounts{}, - ) - require.False(t, itemAccounts.IsInterfaceNil()) - - err := itemAccounts.Save() - require.NoError(t, err) - require.True(t, called) -} - -func TestItemAccounts_SaveAccountsShouldErr(t *testing.T) { - localErr := errors.New("local err") - itemAccounts := workItems.NewItemAccounts( - &mock.ElasticProcessorStub{ - SaveAccountsCalled: func(_ *outport.Accounts) error { - return localErr - }, - }, - &outport.Accounts{}, - ) - require.False(t, itemAccounts.IsInterfaceNil()) - - err := itemAccounts.Save() - require.Equal(t, localErr, err) -} diff --git a/process/dataindexer/workItems/workItemBlock.go b/process/dataindexer/workItems/workItemBlock.go deleted file mode 100644 index c551900f..00000000 --- a/process/dataindexer/workItems/workItemBlock.go +++ /dev/null @@ -1,90 +0,0 @@ -package workItems - -import ( - "encoding/hex" - "fmt" - "time" - - "github.com/multiversx/mx-chain-core-go/core/check" - "github.com/multiversx/mx-chain-core-go/data/outport" - logger "github.com/multiversx/mx-chain-logger-go" -) - -var log = logger.GetOrCreate("indexer/workItems") - -type itemBlock struct { - indexer saveBlockIndexer - outportBlockWithHeader *outport.OutportBlockWithHeader -} - -// NewItemBlock will create a new instance of ItemBlock -func NewItemBlock( - indexer saveBlockIndexer, - outportBlock *outport.OutportBlockWithHeader, -) WorkItemHandler { - return &itemBlock{ - indexer: indexer, - outportBlockWithHeader: outportBlock, - } -} - -// Save will prepare and save a block item in elasticsearch database -func (wib *itemBlock) Save() error { - if check.IfNilReflect(wib.outportBlockWithHeader) { - log.Warn("nil outportBlock block provided when trying to index block, will skip") - return nil - } - if check.IfNil(wib.outportBlockWithHeader.Header) { - log.Warn("nil header provided when trying to index block, will skip") - return nil - } - - headerNonce := wib.outportBlockWithHeader.Header.GetNonce() - headerHash := wib.outportBlockWithHeader.BlockData.HeaderHash - shardID := wib.outportBlockWithHeader.Header.GetShardID() - defer func(startTime time.Time, headerHash []byte, headerNonce uint64, shardID uint32) { - log.Debug("wib.SaveBlockData", - "duration", time.Since(startTime), - "shardID", shardID, - "nonce", headerNonce, - "hash", headerHash, - ) - }(time.Now(), headerHash, headerNonce, shardID) - - log.Debug("indexer: starting indexing block", - "hash", wib.outportBlockWithHeader.BlockData.HeaderHash, - "nonce", wib.outportBlockWithHeader.Header.GetNonce()) - - if wib.outportBlockWithHeader.TransactionPool == nil { - wib.outportBlockWithHeader.TransactionPool = &outport.TransactionPool{} - } - - err := wib.indexer.SaveHeader(wib.outportBlockWithHeader) - if err != nil { - return fmt.Errorf("%w when saving header block, hash %s, nonce %d", - err, hex.EncodeToString(headerHash), headerNonce) - } - - if len(wib.outportBlockWithHeader.BlockData.Body.MiniBlocks) == 0 { - return nil - } - - err = wib.indexer.SaveMiniblocks(wib.outportBlockWithHeader.Header, wib.outportBlockWithHeader.BlockData.Body) - if err != nil { - return fmt.Errorf("%w when saving miniblocks, block hash %s, nonce %d", - err, hex.EncodeToString(headerHash), headerNonce) - } - - err = wib.indexer.SaveTransactions(wib.outportBlockWithHeader) - if err != nil { - return fmt.Errorf("%w when saving transactions, block hash %s, nonce %d", - err, hex.EncodeToString(headerHash), headerNonce) - } - - return nil -} - -// IsInterfaceNil returns true if there is no value under the interface -func (wib *itemBlock) IsInterfaceNil() bool { - return wib == nil -} diff --git a/process/dataindexer/workItems/workItemBlock_test.go b/process/dataindexer/workItems/workItemBlock_test.go deleted file mode 100644 index edc6c794..00000000 --- a/process/dataindexer/workItems/workItemBlock_test.go +++ /dev/null @@ -1,158 +0,0 @@ -package workItems_test - -import ( - "errors" - "testing" - - "github.com/multiversx/mx-chain-core-go/data" - dataBlock "github.com/multiversx/mx-chain-core-go/data/block" - "github.com/multiversx/mx-chain-core-go/data/outport" - "github.com/multiversx/mx-chain-es-indexer-go/mock" - "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestItemBlock_SaveNilHeaderShouldRetNil(t *testing.T) { - itemBlock := workItems.NewItemBlock( - &mock.ElasticProcessorStub{}, - &outport.OutportBlockWithHeader{}, - ) - require.False(t, itemBlock.IsInterfaceNil()) - - err := itemBlock.Save() - assert.Nil(t, err) -} - -func TestItemBlock_SaveHeaderShouldErr(t *testing.T) { - localErr := errors.New("local err") - itemBlock := workItems.NewItemBlock( - &mock.ElasticProcessorStub{ - SaveHeaderCalled: func(_ *outport.OutportBlockWithHeader) error { - return localErr - }, - }, - &outport.OutportBlockWithHeader{ - OutportBlock: &outport.OutportBlock{ - BlockData: &outport.BlockData{}, - }, - Header: &dataBlock.Header{}, - }, - ) - require.False(t, itemBlock.IsInterfaceNil()) - - err := itemBlock.Save() - require.True(t, errors.Is(err, localErr)) -} - -func TestItemBlock_SaveNoMiniblocksShoulCallSaveHeader(t *testing.T) { - countCalled := 0 - itemBlock := workItems.NewItemBlock( - &mock.ElasticProcessorStub{ - SaveHeaderCalled: func(_ *outport.OutportBlockWithHeader) error { - countCalled++ - return nil - }, - SaveMiniblocksCalled: func(header data.HeaderHandler, body *dataBlock.Body) error { - countCalled++ - return nil - }, - SaveTransactionsCalled: func(_ *outport.OutportBlockWithHeader) error { - countCalled++ - return nil - }, - }, - &outport.OutportBlockWithHeader{ - OutportBlock: &outport.OutportBlock{ - BlockData: &outport.BlockData{ - Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{}}, - }, - }, - Header: &dataBlock.Header{}, - }, - ) - require.False(t, itemBlock.IsInterfaceNil()) - - err := itemBlock.Save() - require.NoError(t, err) - require.Equal(t, 1, countCalled) -} - -func TestItemBlock_SaveMiniblocksShouldErr(t *testing.T) { - localErr := errors.New("local err") - itemBlock := workItems.NewItemBlock( - &mock.ElasticProcessorStub{ - SaveMiniblocksCalled: func(header data.HeaderHandler, body *dataBlock.Body) error { - return localErr - }, - }, - &outport.OutportBlockWithHeader{ - OutportBlock: &outport.OutportBlock{ - BlockData: &outport.BlockData{ - Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{{}}}, - }, - }, - Header: &dataBlock.Header{}, - }, - ) - require.False(t, itemBlock.IsInterfaceNil()) - - err := itemBlock.Save() - require.True(t, errors.Is(err, localErr)) -} - -func TestItemBlock_SaveTransactionsShouldErr(t *testing.T) { - localErr := errors.New("local err") - itemBlock := workItems.NewItemBlock( - &mock.ElasticProcessorStub{ - SaveTransactionsCalled: func(_ *outport.OutportBlockWithHeader) error { - return localErr - }, - }, - &outport.OutportBlockWithHeader{ - OutportBlock: &outport.OutportBlock{ - BlockData: &outport.BlockData{ - Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{{}}}, - }, - }, - Header: &dataBlock.Header{}, - }, - ) - require.False(t, itemBlock.IsInterfaceNil()) - - err := itemBlock.Save() - require.True(t, errors.Is(err, localErr)) -} - -func TestItemBlock_SaveShouldWork(t *testing.T) { - countCalled := 0 - itemBlock := workItems.NewItemBlock( - &mock.ElasticProcessorStub{ - SaveHeaderCalled: func(_ *outport.OutportBlockWithHeader) error { - countCalled++ - return nil - }, - SaveMiniblocksCalled: func(header data.HeaderHandler, body *dataBlock.Body) error { - countCalled++ - return nil - }, - SaveTransactionsCalled: func(_ *outport.OutportBlockWithHeader) error { - countCalled++ - return nil - }, - }, - &outport.OutportBlockWithHeader{ - OutportBlock: &outport.OutportBlock{ - BlockData: &outport.BlockData{ - Body: &dataBlock.Body{MiniBlocks: []*dataBlock.MiniBlock{{}}}, - }, - }, - Header: &dataBlock.Header{}, - }, - ) - require.False(t, itemBlock.IsInterfaceNil()) - - err := itemBlock.Save() - require.NoError(t, err) - require.Equal(t, 3, countCalled) -} diff --git a/process/dataindexer/workItems/workItemRating.go b/process/dataindexer/workItems/workItemRating.go deleted file mode 100644 index 9f2e2f52..00000000 --- a/process/dataindexer/workItems/workItemRating.go +++ /dev/null @@ -1,34 +0,0 @@ -package workItems - -import ( - "github.com/multiversx/mx-chain-core-go/data/outport" -) - -type itemRating struct { - indexer saveRatingIndexer - ratingData *outport.ValidatorsRating -} - -// NewItemRating will create a new instance of itemRating -func NewItemRating(indexer saveRatingIndexer, ratingData *outport.ValidatorsRating) WorkItemHandler { - return &itemRating{ - indexer: indexer, - ratingData: ratingData, - } -} - -// IsInterfaceNil returns true if there is no value under the interface -func (wir *itemRating) IsInterfaceNil() bool { - return wir == nil -} - -// Save will save validators rating in elasticsearch database -func (wir *itemRating) Save() error { - err := wir.indexer.SaveValidatorsRating(wir.ratingData) - if err != nil { - log.Warn("itemRating.Save", "could not index validators rating", err.Error()) - return err - } - - return nil -} diff --git a/process/dataindexer/workItems/workItemRating_test.go b/process/dataindexer/workItems/workItemRating_test.go deleted file mode 100644 index 75f53111..00000000 --- a/process/dataindexer/workItems/workItemRating_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package workItems_test - -import ( - "errors" - "testing" - - "github.com/multiversx/mx-chain-core-go/data/outport" - "github.com/multiversx/mx-chain-es-indexer-go/mock" - "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" - "github.com/stretchr/testify/require" -) - -func TestItemRating_Save(t *testing.T) { - called := false - itemRating := workItems.NewItemRating( - &mock.ElasticProcessorStub{ - SaveValidatorsRatingCalled: func(_ *outport.ValidatorsRating) error { - called = true - return nil - }, - }, - &outport.ValidatorsRating{}, - ) - require.False(t, itemRating.IsInterfaceNil()) - - err := itemRating.Save() - require.NoError(t, err) - require.True(t, called) -} - -func TestItemRating_SaveShouldErr(t *testing.T) { - localErr := errors.New("local err") - itemRating := workItems.NewItemRating( - &mock.ElasticProcessorStub{ - SaveValidatorsRatingCalled: func(_ *outport.ValidatorsRating) error { - return localErr - }, - }, - &outport.ValidatorsRating{}, - ) - require.False(t, itemRating.IsInterfaceNil()) - - err := itemRating.Save() - require.Equal(t, localErr, err) -} diff --git a/process/dataindexer/workItems/workItemRemoveBlock.go b/process/dataindexer/workItems/workItemRemoveBlock.go deleted file mode 100644 index 0b1f6328..00000000 --- a/process/dataindexer/workItems/workItemRemoveBlock.go +++ /dev/null @@ -1,50 +0,0 @@ -package workItems - -import ( - "github.com/multiversx/mx-chain-core-go/data" - "github.com/multiversx/mx-chain-core-go/data/block" -) - -type itemRemoveBlock struct { - indexer removeIndexer - header data.HeaderHandler - body *block.Body -} - -// NewItemRemoveBlock will create a new instance of itemRemoveBlock -func NewItemRemoveBlock( - indexer removeIndexer, - header data.HeaderHandler, - body *block.Body, -) WorkItemHandler { - return &itemRemoveBlock{ - indexer: indexer, - header: header, - body: body, - } -} - -// IsInterfaceNil returns true if there is no value under the interface -func (wirb *itemRemoveBlock) IsInterfaceNil() bool { - return wirb == nil -} - -// Save will remove a block and miniblocks from elasticsearch database -func (wirb *itemRemoveBlock) Save() error { - err := wirb.indexer.RemoveHeader(wirb.header) - if err != nil { - return err - } - - err = wirb.indexer.RemoveMiniblocks(wirb.header, wirb.body) - if err != nil { - return err - } - - err = wirb.indexer.RemoveTransactions(wirb.header, wirb.body) - if err != nil { - return err - } - - return wirb.indexer.RemoveAccountsESDT(wirb.header.GetTimeStamp(), wirb.header.GetShardID()) -} diff --git a/process/dataindexer/workItems/workItemRemoveBlock_test.go b/process/dataindexer/workItems/workItemRemoveBlock_test.go deleted file mode 100644 index 8b58f9db..00000000 --- a/process/dataindexer/workItems/workItemRemoveBlock_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package workItems_test - -import ( - "errors" - "testing" - - "github.com/multiversx/mx-chain-core-go/data" - dataBlock "github.com/multiversx/mx-chain-core-go/data/block" - "github.com/multiversx/mx-chain-es-indexer-go/mock" - "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" - "github.com/stretchr/testify/require" -) - -func TestItemRemoveBlock_Save(t *testing.T) { - countCalled := 0 - itemRemove := workItems.NewItemRemoveBlock( - &mock.ElasticProcessorStub{ - RemoveHeaderCalled: func(header data.HeaderHandler) error { - countCalled++ - return nil - }, - RemoveMiniblocksCalled: func(header data.HeaderHandler, body *dataBlock.Body) error { - countCalled++ - return nil - }, - RemoveTransactionsCalled: func(header data.HeaderHandler, body *dataBlock.Body) error { - countCalled++ - return nil - }, - }, - &dataBlock.Header{}, - &dataBlock.Body{}, - ) - require.False(t, itemRemove.IsInterfaceNil()) - - err := itemRemove.Save() - require.NoError(t, err) - require.Equal(t, 3, countCalled) -} - -func TestItemRemoveBlock_SaveRemoveHeaderShouldErr(t *testing.T) { - localErr := errors.New("local err") - itemRemove := workItems.NewItemRemoveBlock( - &mock.ElasticProcessorStub{ - RemoveHeaderCalled: func(header data.HeaderHandler) error { - return localErr - }, - }, - &dataBlock.Header{}, - &dataBlock.Body{}, - ) - require.False(t, itemRemove.IsInterfaceNil()) - - err := itemRemove.Save() - require.Equal(t, localErr, err) -} - -func TestItemRemoveBlock_SaveRemoveMiniblocksShouldErr(t *testing.T) { - localErr := errors.New("local err") - itemRemove := workItems.NewItemRemoveBlock( - &mock.ElasticProcessorStub{ - RemoveMiniblocksCalled: func(header data.HeaderHandler, body *dataBlock.Body) error { - return localErr - }, - }, - &dataBlock.Header{}, - &dataBlock.Body{}, - ) - require.False(t, itemRemove.IsInterfaceNil()) - - err := itemRemove.Save() - require.Equal(t, localErr, err) -} diff --git a/process/dataindexer/workItems/workItemRounds.go b/process/dataindexer/workItems/workItemRounds.go deleted file mode 100644 index 5e078cd6..00000000 --- a/process/dataindexer/workItems/workItemRounds.go +++ /dev/null @@ -1,34 +0,0 @@ -package workItems - -import ( - "github.com/multiversx/mx-chain-core-go/data/outport" -) - -type itemRounds struct { - indexer saveRounds - rounds *outport.RoundsInfo -} - -// NewItemRounds will create a new instance of itemRounds -func NewItemRounds(indexer saveRounds, rounds *outport.RoundsInfo) WorkItemHandler { - return &itemRounds{ - indexer: indexer, - rounds: rounds, - } -} - -// Save will save in elasticsearch database information about rounds -func (wir *itemRounds) Save() error { - err := wir.indexer.SaveRoundsInfo(wir.rounds) - if err != nil { - log.Warn("itemRounds.Save", "could not index rounds info", err.Error()) - return err - } - - return nil -} - -// IsInterfaceNil returns true if there is no value under the interface -func (wir *itemRounds) IsInterfaceNil() bool { - return wir == nil -} diff --git a/process/dataindexer/workItems/workItemRounds_test.go b/process/dataindexer/workItems/workItemRounds_test.go deleted file mode 100644 index f8c2ac7f..00000000 --- a/process/dataindexer/workItems/workItemRounds_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package workItems_test - -import ( - "errors" - "testing" - - "github.com/multiversx/mx-chain-core-go/data/outport" - "github.com/multiversx/mx-chain-es-indexer-go/mock" - "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" - "github.com/stretchr/testify/require" -) - -func TestItemRounds_Save(t *testing.T) { - called := false - itemRounds := workItems.NewItemRounds( - &mock.ElasticProcessorStub{ - SaveRoundsInfoCalled: func(_ *outport.RoundsInfo) error { - called = true - return nil - }, - }, - &outport.RoundsInfo{}, - ) - require.False(t, itemRounds.IsInterfaceNil()) - - err := itemRounds.Save() - require.NoError(t, err) - require.True(t, called) -} - -func TestItemRounds_SaveRoundsShouldErr(t *testing.T) { - localErr := errors.New("local err") - itemRounds := workItems.NewItemRounds( - &mock.ElasticProcessorStub{ - SaveRoundsInfoCalled: func(_ *outport.RoundsInfo) error { - return localErr - }, - }, - &outport.RoundsInfo{}, - ) - require.False(t, itemRounds.IsInterfaceNil()) - - err := itemRounds.Save() - require.Equal(t, localErr, err) -} diff --git a/process/dataindexer/workItems/workItemValidators.go b/process/dataindexer/workItems/workItemValidators.go deleted file mode 100644 index a70d0239..00000000 --- a/process/dataindexer/workItems/workItemValidators.go +++ /dev/null @@ -1,29 +0,0 @@ -package workItems - -import "github.com/multiversx/mx-chain-core-go/data/outport" - -type itemValidators struct { - indexer saveValidatorsIndexer - validatorsPubKeys *outport.ValidatorsPubKeys -} - -// NewItemValidators will create a new instance of itemValidators -func NewItemValidators( - indexer saveValidatorsIndexer, - validatorsPubKeys *outport.ValidatorsPubKeys, -) WorkItemHandler { - return &itemValidators{ - indexer: indexer, - validatorsPubKeys: validatorsPubKeys, - } -} - -// Save will save information about validators -func (wiv *itemValidators) Save() error { - return wiv.indexer.SaveShardValidatorsPubKeys(wiv.validatorsPubKeys) -} - -// IsInterfaceNil returns true if there is no value under the interface -func (wiv *itemValidators) IsInterfaceNil() bool { - return wiv == nil -} diff --git a/process/dataindexer/workItems/workItemValidators_test.go b/process/dataindexer/workItems/workItemValidators_test.go deleted file mode 100644 index b188928a..00000000 --- a/process/dataindexer/workItems/workItemValidators_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package workItems_test - -import ( - "errors" - "testing" - - "github.com/multiversx/mx-chain-core-go/data/outport" - "github.com/multiversx/mx-chain-es-indexer-go/mock" - "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer/workItems" - "github.com/stretchr/testify/require" -) - -func TestItemValidators_Save(t *testing.T) { - called := false - itemValidators := workItems.NewItemValidators( - &mock.ElasticProcessorStub{ - SaveShardValidatorsPubKeysCalled: func(_ *outport.ValidatorsPubKeys) error { - called = true - return nil - }, - }, - &outport.ValidatorsPubKeys{}, - ) - require.False(t, itemValidators.IsInterfaceNil()) - - err := itemValidators.Save() - require.NoError(t, err) - require.True(t, called) -} - -func TestItemValidators_SaveValidatorsShouldErr(t *testing.T) { - localErr := errors.New("local err") - itemValidators := workItems.NewItemValidators( - &mock.ElasticProcessorStub{ - SaveShardValidatorsPubKeysCalled: func(_ *outport.ValidatorsPubKeys) error { - return localErr - }, - }, - &outport.ValidatorsPubKeys{}, - ) - require.False(t, itemValidators.IsInterfaceNil()) - - err := itemValidators.Save() - require.Equal(t, localErr, err) -} From 73017cfa0d53c22a2bbf4f78aad5d3f4a6d91e94 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 6 Jun 2023 16:50:50 +0300 Subject: [PATCH 074/189] fixes --- process/factory/indexerFactory.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index 76a59ae0..6ee5024c 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -54,13 +54,6 @@ func NewIndexer(args ArgsIndexerFactory) (dataindexer.Indexer, error) { return nil, err } - dispatcher, err := dataindexer.NewDataDispatcher(args.IndexerCacheSize) - if err != nil { - return nil, err - } - - dispatcher.StartIndexData() - blockContainer, err := createBlockCreatorsContainer() if err != nil { return nil, err @@ -69,7 +62,6 @@ func NewIndexer(args ArgsIndexerFactory) (dataindexer.Indexer, error) { arguments := dataindexer.ArgDataIndexer{ HeaderMarshaller: args.HeaderMarshaller, ElasticProcessor: elasticProcessor, - DataDispatcher: dispatcher, BlockContainer: blockContainer, } From db1ca8ebcccfb9649688d4ffef3bc235c5d185a9 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 6 Jun 2023 17:12:42 +0300 Subject: [PATCH 075/189] fixes after first review --- process/dataindexer/dataIndexer.go | 9 ++++++++- process/dataindexer/dataIndexer_test.go | 27 +++++++++++++++---------- process/dataindexer/errors.go | 3 --- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/process/dataindexer/dataIndexer.go b/process/dataindexer/dataIndexer.go index 49e20dcf..038b2495 100644 --- a/process/dataindexer/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -92,11 +92,18 @@ func (di *dataIndexer) SaveBlock(outportBlock *outport.OutportBlock) error { outportBlock.TransactionPool = &outport.TransactionPool{} } + return di.saveBlockData(outportBlock, header) +} + +func (di *dataIndexer) saveBlockData(outportBlock *outport.OutportBlock, header data.HeaderHandler) error { outportBlockWithHeader := &outport.OutportBlockWithHeader{ OutportBlock: outportBlock, Header: header, } - err = di.elasticProcessor.SaveHeader(outportBlockWithHeader) + + headerHash := outportBlock.BlockData.HeaderHash + headerNonce := header.GetNonce() + err := di.elasticProcessor.SaveHeader(outportBlockWithHeader) if err != nil { return fmt.Errorf("%w when saving header block, hash %s, nonce %d", err, hex.EncodeToString(headerHash), headerNonce) diff --git a/process/dataindexer/dataIndexer_test.go b/process/dataindexer/dataIndexer_test.go index ca91516e..9cc77a5f 100644 --- a/process/dataindexer/dataIndexer_test.go +++ b/process/dataindexer/dataIndexer_test.go @@ -48,7 +48,7 @@ func TestDataIndexer_NewIndexerWithCorrectParamsShouldWork(t *testing.T) { } func TestDataIndexer_SaveBlock(t *testing.T) { - count := 0 + countMap := map[int]int{} arguments := NewDataIndexerArguments() arguments.BlockContainer = &mock.BlockContainerStub{ @@ -59,15 +59,15 @@ func TestDataIndexer_SaveBlock(t *testing.T) { arguments.ElasticProcessor = &mock.ElasticProcessorStub{ SaveHeaderCalled: func(outportBlockWithHeader *outport.OutportBlockWithHeader) error { - count++ + countMap[0]++ return nil }, SaveMiniblocksCalled: func(header coreData.HeaderHandler, body *dataBlock.Body) error { - count++ + countMap[1]++ return nil }, SaveTransactionsCalled: func(outportBlockWithHeader *outport.OutportBlockWithHeader) error { - count++ + countMap[2]++ return nil }, } @@ -82,7 +82,9 @@ func TestDataIndexer_SaveBlock(t *testing.T) { } err := ei.SaveBlock(args) require.Nil(t, err) - require.Equal(t, 3, count) + require.Equal(t, 1, countMap[0]) + require.Equal(t, 1, countMap[1]) + require.Equal(t, 1, countMap[2]) } func TestDataIndexer_SaveRoundInfo(t *testing.T) { @@ -145,7 +147,7 @@ func TestDataIndexer_SaveValidatorsRating(t *testing.T) { } func TestDataIndexer_RevertIndexedBlock(t *testing.T) { - count := 0 + countMap := map[int]int{} arguments := NewDataIndexerArguments() arguments.BlockContainer = &mock.BlockContainerStub{ @@ -154,19 +156,19 @@ func TestDataIndexer_RevertIndexedBlock(t *testing.T) { }} arguments.ElasticProcessor = &mock.ElasticProcessorStub{ RemoveHeaderCalled: func(header coreData.HeaderHandler) error { - count++ + countMap[0]++ return nil }, RemoveMiniblocksCalled: func(header coreData.HeaderHandler, body *dataBlock.Body) error { - count++ + countMap[1]++ return nil }, RemoveTransactionsCalled: func(header coreData.HeaderHandler, body *dataBlock.Body) error { - count++ + countMap[2]++ return nil }, RemoveAccountsESDTCalled: func(headerTimestamp uint64) error { - count++ + countMap[3]++ return nil }, } @@ -178,5 +180,8 @@ func TestDataIndexer_RevertIndexedBlock(t *testing.T) { HeaderBytes: []byte("{}"), }) require.Nil(t, err) - require.Equal(t, 4, count) + require.Equal(t, 1, countMap[0]) + require.Equal(t, 1, countMap[1]) + require.Equal(t, 1, countMap[2]) + require.Equal(t, 1, countMap[3]) } diff --git a/process/dataindexer/errors.go b/process/dataindexer/errors.go index 6c33926c..cb7575b0 100644 --- a/process/dataindexer/errors.go +++ b/process/dataindexer/errors.go @@ -11,9 +11,6 @@ var ErrNilPubkeyConverter = errors.New("nil pubkey converter") // ErrNegativeDenominationValue signals that a negative denomination value has been provided var ErrNegativeDenominationValue = errors.New("negative denomination value") -// ErrNilDataDispatcher signals that an operation has been attempted to or with a nil data dispatcher implementation -var ErrNilDataDispatcher = errors.New("nil data dispatcher") - // ErrNilElasticProcessor signals that an operation has been attempted to or with a nil elastic processor implementation var ErrNilElasticProcessor = errors.New("nil elastic processor") From 299ae2143df54bc0cfd70a7e079f3c0ac2af097f Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 7 Jun 2023 14:00:13 +0300 Subject: [PATCH 076/189] update mx chain comm go --- cmd/elasticindexer/config/prefs.toml | 4 +++- config/config.go | 1 + factory/wsIndexerFactory.go | 11 ++++++----- go.mod | 2 +- go.sum | 4 ++-- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index 141b7a0a..2685e4b8 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -11,9 +11,11 @@ # Retry duration (receive/send ack signal) in seconds retry-duration-in-seconds = 5 # Signals if in case of data payload processing error, we should send the ack signal or not - blocking-ack-on-error = false + blocking-ack-on-error = true # After a message will be sent it will wait for an ack message if this flag is enabled with-acknowledge = true + # The duration in seconds to wait for an acknowledgment message, after this time passes an error will be returned + acknowledge-timeout-in-seconds = 50 [config.elastic-cluster] use-kibana = false diff --git a/config/config.go b/config/config.go index 301d41fb..e07b00b2 100644 --- a/config/config.go +++ b/config/config.go @@ -42,6 +42,7 @@ type ClusterConfig struct { RetryDurationInSec uint32 `toml:"retry-duration-in-seconds"` BlockingAckOnError bool `toml:"blocking-ack-on-error"` WithAcknowledge bool `toml:"with-acknowledge"` + AckTimeoutInSec uint32 `toml:"acknowledge-timeout-in-seconds"` } `toml:"web-socket"` ElasticCluster struct { UseKibana bool `toml:"use-kibana"` diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 3b2a9f60..ebb21c58 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -107,11 +107,12 @@ func prepareIndices(availableIndices, disabledIndices []string) []string { func createWsHost(clusterCfg config.ClusterConfig, wsMarshaller marshal.Marshalizer) (factoryHost.FullDuplexHost, error) { return factoryHost.CreateWebSocketHost(factoryHost.ArgsWebSocketHost{ WebSocketConfig: data.WebSocketConfig{ - URL: clusterCfg.Config.WebSocket.URL, - WithAcknowledge: clusterCfg.Config.WebSocket.WithAcknowledge, - Mode: clusterCfg.Config.WebSocket.Mode, - RetryDurationInSec: int(clusterCfg.Config.WebSocket.RetryDurationInSec), - BlockingAckOnError: clusterCfg.Config.WebSocket.BlockingAckOnError, + URL: clusterCfg.Config.WebSocket.URL, + WithAcknowledge: clusterCfg.Config.WebSocket.WithAcknowledge, + Mode: clusterCfg.Config.WebSocket.Mode, + RetryDurationInSec: int(clusterCfg.Config.WebSocket.RetryDurationInSec), + AcknowledgeTimeoutInSec: int(clusterCfg.Config.WebSocket.AckTimeoutInSec), + BlockingAckOnError: clusterCfg.Config.WebSocket.BlockingAckOnError, }, Marshaller: wsMarshaller, Log: log, diff --git a/go.mod b/go.mod index 0be3b866..610f0b66 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 - github.com/multiversx/mx-chain-communication-go v1.0.2 + github.com/multiversx/mx-chain-communication-go v1.0.3-0.20230607101542-b80e6e676772 github.com/multiversx/mx-chain-core-go v1.2.4 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.1 diff --git a/go.sum b/go.sum index e3abfc40..49ec5daf 100644 --- a/go.sum +++ b/go.sum @@ -471,8 +471,8 @@ github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= -github.com/multiversx/mx-chain-communication-go v1.0.2 h1:1AKdqFZNmigt1kcwYMl+L8fzolsb+WpeTX6yzpmvbV4= -github.com/multiversx/mx-chain-communication-go v1.0.2/go.mod h1:OnnSUjnSP87H5MtQtxn33FGnTVRVgo2Huo3ijmCgN2M= +github.com/multiversx/mx-chain-communication-go v1.0.3-0.20230607101542-b80e6e676772 h1:K5lIIlpjBckiQtNJLCpTHkaYAnEUBA3xSkQ6kC0MpUY= +github.com/multiversx/mx-chain-communication-go v1.0.3-0.20230607101542-b80e6e676772/go.mod h1:OnnSUjnSP87H5MtQtxn33FGnTVRVgo2Huo3ijmCgN2M= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1-0.20230510143029-ab37792342df/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= From 184cb0ef4d5cea7c3962226e36b9ea2337913641 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 7 Jun 2023 14:02:39 +0300 Subject: [PATCH 077/189] fixes --- process/dataindexer/dataIndexer.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/process/dataindexer/dataIndexer.go b/process/dataindexer/dataIndexer.go index 038b2495..85120858 100644 --- a/process/dataindexer/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -78,14 +78,15 @@ func (di *dataIndexer) SaveBlock(outportBlock *outport.OutportBlock) error { headerHash := outportBlock.BlockData.HeaderHash shardID := header.GetShardID() headerNonce := header.GetNonce() - defer func(startTime time.Time, headerHash []byte, headerNonce uint64, shardID uint32) { + startTime := time.Now() + defer func() { log.Debug("di.SaveBlockData", "duration", time.Since(startTime), "shardID", shardID, "nonce", headerNonce, "hash", headerHash, ) - }(time.Now(), headerHash, headerNonce, shardID) + }() log.Debug("indexer: starting indexing block", "hash", headerHash, "nonce", headerNonce) if outportBlock.TransactionPool == nil { From 55484bc101f4c855498ec84f8293f882efe54098 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 7 Jun 2023 14:04:33 +0300 Subject: [PATCH 078/189] remove IndexerCacheSize --- process/dataindexer/errors.go | 3 --- process/factory/indexerFactory.go | 4 ---- process/factory/indexerFactory_test.go | 10 ---------- 3 files changed, 17 deletions(-) diff --git a/process/dataindexer/errors.go b/process/dataindexer/errors.go index cb7575b0..9b4fbd0f 100644 --- a/process/dataindexer/errors.go +++ b/process/dataindexer/errors.go @@ -14,9 +14,6 @@ var ErrNegativeDenominationValue = errors.New("negative denomination value") // ErrNilElasticProcessor signals that an operation has been attempted to or with a nil elastic processor implementation var ErrNilElasticProcessor = errors.New("nil elastic processor") -// ErrNegativeCacheSize signals that an invalid cache size has been provided -var ErrNegativeCacheSize = errors.New("negative cache size") - // ErrEmptyEnabledIndexes signals that an empty slice of enables indexes has been provided var ErrEmptyEnabledIndexes = errors.New("empty enabled indexes slice") diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index 6ee5024c..2782b51d 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -27,7 +27,6 @@ type ArgsIndexerFactory struct { Enabled bool UseKibana bool ImportDB bool - IndexerCacheSize int Denomination int BulkRequestMaxSize int Url string @@ -105,9 +104,6 @@ func createElasticProcessor(args ArgsIndexerFactory) (dataindexer.ElasticProcess } func checkDataIndexerParams(arguments ArgsIndexerFactory) error { - if arguments.IndexerCacheSize < 0 { - return dataindexer.ErrNegativeCacheSize - } if check.IfNil(arguments.AddressPubkeyConverter) { return fmt.Errorf("%w when setting AddressPubkeyConverter in indexer", dataindexer.ErrNilPubkeyConverter) } diff --git a/process/factory/indexerFactory_test.go b/process/factory/indexerFactory_test.go index 3f153017..b6b35110 100644 --- a/process/factory/indexerFactory_test.go +++ b/process/factory/indexerFactory_test.go @@ -16,7 +16,6 @@ func createMockIndexerFactoryArgs() ArgsIndexerFactory { return ArgsIndexerFactory{ Enabled: true, - IndexerCacheSize: 100, Url: ts.URL, UserName: "", Password: "", @@ -36,15 +35,6 @@ func TestNewIndexerFactory(t *testing.T) { argsFunc func() ArgsIndexerFactory exError error }{ - { - name: "InvalidCacheSize", - argsFunc: func() ArgsIndexerFactory { - args := createMockIndexerFactoryArgs() - args.IndexerCacheSize = -1 - return args - }, - exError: dataindexer.ErrNegativeCacheSize, - }, { name: "NilAddressPubkeyConverter", argsFunc: func() ArgsIndexerFactory { From 5f49abfa8c319b3b3e5860072b2fd261b9cad1e6 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 7 Jun 2023 14:06:16 +0300 Subject: [PATCH 079/189] fix binary --- factory/wsIndexerFactory.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index ebb21c58..0d4a3278 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -13,10 +13,6 @@ import ( logger "github.com/multiversx/mx-chain-logger-go" ) -const ( - indexerCacheSize = 1 -) - var log = logger.GetOrCreate("elasticindexer") // CreateWsIndexer will create a new instance of wsindexer.WSClient @@ -69,7 +65,6 @@ func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig, wsMar return factory.NewIndexer(factory.ArgsIndexerFactory{ UseKibana: clusterCfg.Config.ElasticCluster.UseKibana, - IndexerCacheSize: indexerCacheSize, Denomination: cfg.Config.Economics.Denomination, BulkRequestMaxSize: clusterCfg.Config.ElasticCluster.BulkRequestMaxSizeInBytes, Url: clusterCfg.Config.ElasticCluster.URL, From 8d697dad7543818652f9db8659f40fb7eb1ca9a7 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 7 Jun 2023 15:55:00 +0300 Subject: [PATCH 080/189] request node settings --- cmd/elasticindexer/main.go | 14 ++++++++++++++ go.mod | 2 +- go.sum | 4 ++-- process/dataindexer/dataIndexer.go | 12 ++++++++++++ process/dataindexer/interface.go | 3 +++ process/elasticproc/elasticProcessor.go | 23 +++++++++++++++++++++-- process/wsindexer/indexer.go | 11 +++++++++++ process/wsindexer/interface.go | 2 ++ 8 files changed, 66 insertions(+), 5 deletions(-) diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index 67cc107d..a4241955 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -10,8 +10,10 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/core/closing" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-es-indexer-go/config" "github.com/multiversx/mx-chain-es-indexer-go/factory" + "github.com/multiversx/mx-chain-es-indexer-go/process/wsindexer" logger "github.com/multiversx/mx-chain-logger-go" "github.com/multiversx/mx-chain-logger-go/file" "github.com/urfave/cli" @@ -87,6 +89,8 @@ func startIndexer(ctx *cli.Context) error { return fmt.Errorf("%w while creating the indexer", err) } + requestSettings(wsHost, clusterCfg.Config.WebSocket.RetryDurationInSec) + interrupt := make(chan os.Signal, 1) signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM) @@ -104,6 +108,16 @@ func startIndexer(ctx *cli.Context) error { return nil } +func requestSettings(host wsindexer.WSClient, retryDurationInSec uint32) { + for { + err := host.Send([]byte{}, outport.TopicSettings) + if err != nil { + time.Sleep(time.Duration(retryDurationInSec) * time.Second) + log.Debug("request settings", "error", err) + } + } +} + func loadMainConfig(filepath string) (config.Config, error) { cfg := config.Config{} err := core.LoadTomlFile(&cfg, filepath) diff --git a/go.mod b/go.mod index 610f0b66..46cebd4c 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/multiversx/mx-chain-communication-go v1.0.3-0.20230607101542-b80e6e676772 - github.com/multiversx/mx-chain-core-go v1.2.4 + github.com/multiversx/mx-chain-core-go v1.2.6-0.20230529102659-223d4826d177 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.1 github.com/stretchr/testify v1.8.2 diff --git a/go.sum b/go.sum index 49ec5daf..e30422a2 100644 --- a/go.sum +++ b/go.sum @@ -476,8 +476,8 @@ github.com/multiversx/mx-chain-communication-go v1.0.3-0.20230607101542-b80e6e67 github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1-0.20230510143029-ab37792342df/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.4 h1:BRXyajUevLU6zHszR8jnp2+7C2bAQBor51YTc4dp3YQ= -github.com/multiversx/mx-chain-core-go v1.2.4/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= +github.com/multiversx/mx-chain-core-go v1.2.6-0.20230529102659-223d4826d177 h1:VVvMV4KufAF1uNxdTLODWwT+l+l2m4Cek8Dc4LgCo60= +github.com/multiversx/mx-chain-core-go v1.2.6-0.20230529102659-223d4826d177/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-crypto-go v1.2.6/go.mod h1:rOj0Rr19HTOYt9YTeym7RKxlHt91NXln3LVKjHKVmA0= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= diff --git a/process/dataindexer/dataIndexer.go b/process/dataindexer/dataIndexer.go index 85120858..516de44f 100644 --- a/process/dataindexer/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -189,6 +189,18 @@ func (di *dataIndexer) GetMarshaller() marshal.Marshalizer { return di.headerMarshaller } +// RegisterHandlerForSettingsRequest will do nothing +func (di *dataIndexer) RegisterHandlerForSettingsRequest(_ func()) error { + return nil +} + +// CurrentSettings will set the provided settings +func (di *dataIndexer) CurrentSettings(cfg outport.OutportConfig) error { + log.Debug("dataIndexer.CurrentSettings", "importDBMode", cfg.IsInImportDBMode) + + return di.elasticProcessor.SetOutportConfig(cfg) +} + // IsInterfaceNil returns true if there is no value under the interface func (di *dataIndexer) IsInterfaceNil() bool { return di == nil diff --git a/process/dataindexer/interface.go b/process/dataindexer/interface.go index 08bdb000..e32017e9 100644 --- a/process/dataindexer/interface.go +++ b/process/dataindexer/interface.go @@ -23,6 +23,7 @@ type ElasticProcessor interface { SaveRoundsInfo(rounds *outport.RoundsInfo) error SaveShardValidatorsPubKeys(validatorsPubKeys *outport.ValidatorsPubKeys) error SaveAccounts(accounts *outport.Accounts) error + SetOutportConfig(cfg outport.OutportConfig) error IsInterfaceNil() bool } @@ -55,6 +56,8 @@ type Indexer interface { SaveAccounts(accountsData *outport.Accounts) error FinalizedBlock(finalizedBlock *outport.FinalizedBlock) error GetMarshaller() marshal.Marshalizer + RegisterHandlerForSettingsRequest(handler func()) error + CurrentSettings(cfg outport.OutportConfig) error Close() error IsInterfaceNil() bool } diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index 295c14c2..b40c2a29 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" "fmt" + "sync" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" @@ -63,6 +64,7 @@ type elasticProcessor struct { bulkRequestMaxSize int importDB bool enabledIndexes map[string]struct{} + mutex sync.RWMutex elasticClient DatabaseClientHandler accountsProc DBAccountHandler blockProc DBBlockHandler @@ -408,7 +410,7 @@ func (ei *elasticProcessor) miniblocksInDBMap(mbs []*data.Miniblock) (map[string func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader) error { headerTimestamp := obh.Header.GetTimeStamp() - preparedResults := ei.transactionsProc.PrepareTransactionsForDatabase(obh.BlockData.Body, obh.Header, obh.TransactionPool, ei.importDB, obh.NumberOfShards) + preparedResults := ei.transactionsProc.PrepareTransactionsForDatabase(obh.BlockData.Body, obh.Header, obh.TransactionPool, ei.isImportDB(), obh.NumberOfShards) logsData := ei.logsAndEventsProc.ExtractDataFromLogs(obh.TransactionPool.Logs, preparedResults, headerTimestamp, obh.Header.GetShardID(), obh.NumberOfShards) buffers := data.NewBufferSlice(ei.bulkRequestMaxSize) @@ -417,7 +419,7 @@ func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader return err } - err = ei.prepareAndIndexOperations(preparedResults.Transactions, preparedResults.TxHashStatus, obh.Header, preparedResults.ScResults, buffers, ei.importDB) + err = ei.prepareAndIndexOperations(preparedResults.Transactions, preparedResults.TxHashStatus, obh.Header, preparedResults.ScResults, buffers, ei.isImportDB()) if err != nil { return err } @@ -815,6 +817,23 @@ func (ei *elasticProcessor) doBulkRequests(index string, buffSlice []*bytes.Buff return nil } +// SetOutportConfig will set the outport config +func (ei *elasticProcessor) SetOutportConfig(cfg outport.OutportConfig) error { + ei.mutex.Lock() + defer ei.mutex.Unlock() + + ei.importDB = cfg.IsInImportDBMode + + return nil +} + +func (ei *elasticProcessor) isImportDB() bool { + ei.mutex.RLock() + defer ei.mutex.RUnlock() + + return ei.importDB +} + // IsInterfaceNil returns true if there is no value under the interface func (ei *elasticProcessor) IsInterfaceNil() bool { return ei == nil diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index e98e65b4..5548d5e6 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -49,6 +49,7 @@ func (i *indexer) initActionsMap() { outport.TopicSaveValidatorsPubKeys: i.saveValidatorsPubKeys, outport.TopicSaveAccounts: i.saveAccounts, outport.TopicFinalizedBlock: i.finalizedBlock, + outport.TopicSettings: i.setSettings, } } @@ -127,6 +128,16 @@ func (i *indexer) finalizedBlock(_ []byte) error { return nil } +func (i *indexer) setSettings(marshalledData []byte) error { + settings := outport.OutportConfig{} + err := i.marshaller.Unmarshal(&settings, marshalledData) + if err != nil { + return err + } + + return i.di.CurrentSettings(settings) +} + // Close will close the indexer func (i *indexer) Close() error { return i.di.Close() diff --git a/process/wsindexer/interface.go b/process/wsindexer/interface.go index 22069859..54ffb6f3 100644 --- a/process/wsindexer/interface.go +++ b/process/wsindexer/interface.go @@ -6,6 +6,7 @@ import ( // WSClient defines what a websocket client should do type WSClient interface { + Send(message []byte, topic string) error Close() error } @@ -18,6 +19,7 @@ type DataIndexer interface { SaveValidatorsRating(ratingData *outport.ValidatorsRating) error SaveAccounts(accountsData *outport.Accounts) error FinalizedBlock(finalizedBlock *outport.FinalizedBlock) error + CurrentSettings(settings outport.OutportConfig) error Close() error IsInterfaceNil() bool } From 8950bdbc83adb075771bde6358864ae54a4f629e Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 7 Jun 2023 15:56:55 +0300 Subject: [PATCH 081/189] fix tests --- mock/elasticProcessorStub.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mock/elasticProcessorStub.go b/mock/elasticProcessorStub.go index d8aa1000..e39b3e8c 100644 --- a/mock/elasticProcessorStub.go +++ b/mock/elasticProcessorStub.go @@ -111,6 +111,10 @@ func (eim *ElasticProcessorStub) SaveAccounts(accounts *outport.Accounts) error return nil } +func (eim *ElasticProcessorStub) SetOutportConfig(_ outport.OutportConfig) error { + return nil +} + // IsInterfaceNil returns true if there is no value under the interface func (eim *ElasticProcessorStub) IsInterfaceNil() bool { return eim == nil From d6bc468d3614de8b4ebc43395a56e1601f9e7e19 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 7 Jun 2023 15:57:09 +0300 Subject: [PATCH 082/189] comment --- mock/elasticProcessorStub.go | 1 + 1 file changed, 1 insertion(+) diff --git a/mock/elasticProcessorStub.go b/mock/elasticProcessorStub.go index e39b3e8c..7c0cd091 100644 --- a/mock/elasticProcessorStub.go +++ b/mock/elasticProcessorStub.go @@ -111,6 +111,7 @@ func (eim *ElasticProcessorStub) SaveAccounts(accounts *outport.Accounts) error return nil } +// SetOutportConfig - func (eim *ElasticProcessorStub) SetOutportConfig(_ outport.OutportConfig) error { return nil } From 392de413181b7d36b62b9e3a93a5c9422f27a366 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 7 Jun 2023 18:03:53 +0300 Subject: [PATCH 083/189] fixes --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 46cebd4c..937a679f 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 - github.com/multiversx/mx-chain-communication-go v1.0.3-0.20230607101542-b80e6e676772 + github.com/multiversx/mx-chain-communication-go v1.0.3-0.20230607150138-28d0e4242a47 github.com/multiversx/mx-chain-core-go v1.2.6-0.20230529102659-223d4826d177 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.1 diff --git a/go.sum b/go.sum index e30422a2..75558c3f 100644 --- a/go.sum +++ b/go.sum @@ -471,8 +471,8 @@ github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= -github.com/multiversx/mx-chain-communication-go v1.0.3-0.20230607101542-b80e6e676772 h1:K5lIIlpjBckiQtNJLCpTHkaYAnEUBA3xSkQ6kC0MpUY= -github.com/multiversx/mx-chain-communication-go v1.0.3-0.20230607101542-b80e6e676772/go.mod h1:OnnSUjnSP87H5MtQtxn33FGnTVRVgo2Huo3ijmCgN2M= +github.com/multiversx/mx-chain-communication-go v1.0.3-0.20230607150138-28d0e4242a47 h1:qizL4OFH8IPumgoP2gKrxamS+Oi6NMKICCrvNRp8Ctw= +github.com/multiversx/mx-chain-communication-go v1.0.3-0.20230607150138-28d0e4242a47/go.mod h1:OnnSUjnSP87H5MtQtxn33FGnTVRVgo2Huo3ijmCgN2M= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1-0.20230510143029-ab37792342df/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= From 09aab246e02150878dbe2f85e2be889d7d24c0c7 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 8 Jun 2023 12:50:30 +0300 Subject: [PATCH 084/189] bug fixes and script improves --- cmd/elasticindexer/main.go | 3 +++ scripts/observers/.env | 4 +++- scripts/observers/config.py | 12 +++++++----- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index a4241955..5aea71d5 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -113,8 +113,11 @@ func requestSettings(host wsindexer.WSClient, retryDurationInSec uint32) { err := host.Send([]byte{}, outport.TopicSettings) if err != nil { time.Sleep(time.Duration(retryDurationInSec) * time.Second) + log.Debug("request settings", "error", err) + continue } + return } } diff --git a/scripts/observers/.env b/scripts/observers/.env index d3627aa7..685333ce 100644 --- a/scripts/observers/.env +++ b/scripts/observers/.env @@ -2,7 +2,7 @@ MX_CHAIN_DEPLOY_GO_URL=https://github.com/multiversx/mx-chain-deploy-go MX_CHAIN_PROXY_URL=https://github.com/multiversx/mx-chain-proxy-go NODE_GO_URL="https://github.com/multiversx/mx-chain-go" -NODE_GO_BRANCH="rc/v1.6.0" +NODE_GO_BRANCH="handle-settings-in-outport-driver" WORKING_DIRECTORY="IndexerObservers" OBSERVER_DIR_PREFIX="observer_shard_" @@ -16,4 +16,6 @@ OBSERVERS_START_PORT=9500 PROXY_PORT=7950 +ACK_TIMEOUT_IN_SECONDS=10 + INDEXER_BINARY_SERVER=true diff --git a/scripts/observers/config.py b/scripts/observers/config.py index ff4c8c6d..9aea50d3 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -21,10 +21,11 @@ def update_toml_indexer(path, shard_id): prefs_data['config']['web-socket']['mode'] = "server" if shard_id != METACHAIN: - prefs_data['config']['web-socket']['url'] = "localhost:" + str(port) + prefs_data['config']['web-socket']['url'] = f"localhost:{str(port)}" else: - prefs_data['config']['web-socket']['url'] = "localhost:" + str(meta_port) + prefs_data['config']['web-socket']['url'] = f"localhost:{str(meta_port)}" prefs_data['config']['web-socket']['data-marshaller-type'] = str(os.getenv('WS_MARSHALLER_TYPE')) + prefs_data['config']['web-socket']['acknowledge-timeout-in-seconds'] = int(os.getenv('ACK_TIMEOUT_IN_SECONDS')) f = open(path_prefs, 'w') toml.dump(prefs_data, f) @@ -61,16 +62,17 @@ def update_toml_node(path, shard_id): is_indexer_server = os.getenv('INDEXER_BINARY_SERVER') if is_indexer_server: - external_data['HostDriverConfig']['IsServer'] = False + external_data['HostDriverConfig']['Mode'] = "client" port = WS_PORT_BASE meta_port = WS_PORT_BASE if shard_id != METACHAIN: - external_data['HostDriverConfig']['URL'] = "localhost:" + str(port) + external_data['HostDriverConfig']['URL'] = f"localhost:{str(port)}" else: - external_data['HostDriverConfig']['URL'] = "localhost:" + str(meta_port) + external_data['HostDriverConfig']['URL'] = f"localhost:{str(meta_port)}" external_data['HostDriverConfig']['MarshallerType'] = str(os.getenv('WS_MARSHALLER_TYPE')) + external_data['HostDriverConfig']['AcknowledgeTimeoutInSec'] = int(os.getenv('ACK_TIMEOUT_IN_SECONDS')) f = open(path_external, 'w') toml.dump(external_data, f) f.close() From b4e7d733f4d8ff64689c4b5c01fac0405a339950 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 8 Jun 2023 13:30:15 +0300 Subject: [PATCH 085/189] small refactoring --- scripts/observers/config.py | 10 +++++----- scripts/observers/stop.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/observers/config.py b/scripts/observers/config.py index 9aea50d3..eacf2f8f 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -199,10 +199,10 @@ def prepare_proxy(working_dir): config_data['Observers'].append(meta_observer) num_of_shards = int(os.getenv('NUM_OF_SHARDS')) - for shardID in range(num_of_shards): - shard_observer_port = observers_start_port + shardID + 1 + for shard_id in range(num_of_shards): + shard_observer_port = observers_start_port + shard_id + 1 meta_observer = { - 'ShardId': shardID, + 'ShardId': shard_id, 'Address': f'http://127.0.0.1:{shard_observer_port}', } config_data['Observers'].append(meta_observer) @@ -264,8 +264,8 @@ def main(): prepare_observer(METACHAIN, working_dir, config_folder) prepare_indexer_server(METACHAIN, working_dir) - for shardID in range(num_of_shards): - prepare_observer(shardID, working_dir, config_folder) + for shard_id in range(num_of_shards): + prepare_observer(shard_id, working_dir, config_folder) if __name__ == "__main__": diff --git a/scripts/observers/stop.py b/scripts/observers/stop.py index f4410e31..95cc29ae 100644 --- a/scripts/observers/stop.py +++ b/scripts/observers/stop.py @@ -22,7 +22,7 @@ def main(): os.system(f'screen -X -S indexer{shard_id} quit') if is_indexer_server: - os.system(f'screen -X -S indexerserver quit') + os.system("screen -X -S indexerserver quit") print("done") From 3225e78cd3fd9955f39152ec05411c35d88e8667 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 8 Jun 2023 14:36:51 +0300 Subject: [PATCH 086/189] extend Makefile --- scripts/observers/Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/observers/Makefile b/scripts/observers/Makefile index 61f9f708..fa48e6a3 100644 --- a/scripts/observers/Makefile +++ b/scripts/observers/Makefile @@ -5,6 +5,7 @@ help: @echo " start - Start the local testnet" @echo " stop - Stop the local testnet" @echo " clean - Clean up the files of the local testnet" + @echo " cluster - Start an Elasticsearch cluster on port 9200" config: python3 config.py @@ -14,3 +15,7 @@ stop: python3 stop.py clean: python3 clean.py + cd .. && /bin/bash script.sh delete +cluster: + cd .. && /bin/bash script.sh start + From 10e235cbff0ebac707707ba818203ca5defbcf67 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 9 Jun 2023 10:54:45 +0300 Subject: [PATCH 087/189] fixes after review --- cmd/elasticindexer/main.go | 2 +- process/dataindexer/dataIndexer.go | 4 ++-- process/dataindexer/interface.go | 2 +- process/wsindexer/indexer.go | 2 +- process/wsindexer/interface.go | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index 5aea71d5..e02a625c 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -114,7 +114,7 @@ func requestSettings(host wsindexer.WSClient, retryDurationInSec uint32) { if err != nil { time.Sleep(time.Duration(retryDurationInSec) * time.Second) - log.Debug("request settings", "error", err) + log.Debug("unable to request settings - will retry", "error", err) continue } return diff --git a/process/dataindexer/dataIndexer.go b/process/dataindexer/dataIndexer.go index 516de44f..6388f04c 100644 --- a/process/dataindexer/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -195,8 +195,8 @@ func (di *dataIndexer) RegisterHandlerForSettingsRequest(_ func()) error { } // CurrentSettings will set the provided settings -func (di *dataIndexer) CurrentSettings(cfg outport.OutportConfig) error { - log.Debug("dataIndexer.CurrentSettings", "importDBMode", cfg.IsInImportDBMode) +func (di *dataIndexer) SetCurrentSettings(cfg outport.OutportConfig) error { + log.Debug("dataIndexer.SetCurrentSettings", "importDBMode", cfg.IsInImportDBMode) return di.elasticProcessor.SetOutportConfig(cfg) } diff --git a/process/dataindexer/interface.go b/process/dataindexer/interface.go index e32017e9..5bf8c86c 100644 --- a/process/dataindexer/interface.go +++ b/process/dataindexer/interface.go @@ -57,7 +57,7 @@ type Indexer interface { FinalizedBlock(finalizedBlock *outport.FinalizedBlock) error GetMarshaller() marshal.Marshalizer RegisterHandlerForSettingsRequest(handler func()) error - CurrentSettings(cfg outport.OutportConfig) error + SetCurrentSettings(cfg outport.OutportConfig) error Close() error IsInterfaceNil() bool } diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index 5548d5e6..5dd50ab8 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -135,7 +135,7 @@ func (i *indexer) setSettings(marshalledData []byte) error { return err } - return i.di.CurrentSettings(settings) + return i.di.SetCurrentSettings(settings) } // Close will close the indexer diff --git a/process/wsindexer/interface.go b/process/wsindexer/interface.go index 54ffb6f3..1c23cc7d 100644 --- a/process/wsindexer/interface.go +++ b/process/wsindexer/interface.go @@ -19,7 +19,7 @@ type DataIndexer interface { SaveValidatorsRating(ratingData *outport.ValidatorsRating) error SaveAccounts(accountsData *outport.Accounts) error FinalizedBlock(finalizedBlock *outport.FinalizedBlock) error - CurrentSettings(settings outport.OutportConfig) error + SetCurrentSettings(settings outport.OutportConfig) error Close() error IsInterfaceNil() bool } From b20f7655883b30f1c6ee86a63baefb9960cc2c92 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 9 Jun 2023 11:05:21 +0300 Subject: [PATCH 088/189] fix comment --- process/dataindexer/dataIndexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process/dataindexer/dataIndexer.go b/process/dataindexer/dataIndexer.go index 6388f04c..0e656a69 100644 --- a/process/dataindexer/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -194,7 +194,7 @@ func (di *dataIndexer) RegisterHandlerForSettingsRequest(_ func()) error { return nil } -// CurrentSettings will set the provided settings +// SetCurrentSettings will set the provided settings func (di *dataIndexer) SetCurrentSettings(cfg outport.OutportConfig) error { log.Debug("dataIndexer.SetCurrentSettings", "importDBMode", cfg.IsInImportDBMode) From ad4bec61cba68bc4dda33772b9e9c3b17830798c Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 9 Jun 2023 12:01:55 +0300 Subject: [PATCH 089/189] fixes after review --- cmd/elasticindexer/main.go | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index e02a625c..01120cf1 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -89,12 +89,14 @@ func startIndexer(ctx *cli.Context) error { return fmt.Errorf("%w while creating the indexer", err) } - requestSettings(wsHost, clusterCfg.Config.WebSocket.RetryDurationInSec) - interrupt := make(chan os.Signal, 1) signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM) - <-interrupt + closed := requestSettings(wsHost, clusterCfg.Config.WebSocket.RetryDurationInSec, interrupt) + if !closed { + <-interrupt + } + log.Info("closing app at user's signal") err = wsHost.Close() if err != nil { @@ -108,16 +110,24 @@ func startIndexer(ctx *cli.Context) error { return nil } -func requestSettings(host wsindexer.WSClient, retryDurationInSec uint32) { - for { - err := host.Send([]byte{}, outport.TopicSettings) - if err != nil { - time.Sleep(time.Duration(retryDurationInSec) * time.Second) +func requestSettings(host wsindexer.WSClient, retryDurationInSec uint32, close chan os.Signal) bool { + timer := time.NewTimer(0) + defer timer.Stop() + for { + select { + case <-timer.C: + err := host.Send([]byte{}, outport.TopicSettings) + if err == nil { + return false + } log.Debug("unable to request settings - will retry", "error", err) - continue + + timer.Reset(time.Duration(retryDurationInSec) * time.Second) + case <-close: + log.Debug("closed") + return true } - return } } From 8757b56e2ff3a6634da5ea1137833c08d60a85b8 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 9 Jun 2023 12:09:12 +0300 Subject: [PATCH 090/189] remove log --- cmd/elasticindexer/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index 01120cf1..5e87632f 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -125,7 +125,6 @@ func requestSettings(host wsindexer.WSClient, retryDurationInSec uint32, close c timer.Reset(time.Duration(retryDurationInSec) * time.Second) case <-close: - log.Debug("closed") return true } } From 820c92bc89ba9b67955052115686396c0fb0ea1c Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 9 Jun 2023 12:27:26 +0300 Subject: [PATCH 091/189] small fixes --- cmd/elasticindexer/main.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index 5e87632f..5b8b9026 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -92,7 +92,8 @@ func startIndexer(ctx *cli.Context) error { interrupt := make(chan os.Signal, 1) signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM) - closed := requestSettings(wsHost, clusterCfg.Config.WebSocket.RetryDurationInSec, interrupt) + retryDuration := time.Duration(clusterCfg.Config.WebSocket.RetryDurationInSec) * time.Second + closed := requestSettings(wsHost, retryDuration, interrupt) if !closed { <-interrupt } @@ -110,20 +111,21 @@ func startIndexer(ctx *cli.Context) error { return nil } -func requestSettings(host wsindexer.WSClient, retryDurationInSec uint32, close chan os.Signal) bool { +func requestSettings(host wsindexer.WSClient, retryDuration time.Duration, close chan os.Signal) bool { timer := time.NewTimer(0) defer timer.Stop() + emptyMessage := make([]byte, 0) for { select { case <-timer.C: - err := host.Send([]byte{}, outport.TopicSettings) + err := host.Send(emptyMessage, outport.TopicSettings) if err == nil { return false } log.Debug("unable to request settings - will retry", "error", err) - timer.Reset(time.Duration(retryDurationInSec) * time.Second) + timer.Reset(retryDuration) case <-close: return true } From c71e4c2ca053c767b657b6dd2795e6d70deaaa27 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 12 Jun 2023 12:15:15 +0300 Subject: [PATCH 092/189] update interface --- process/dataindexer/dataIndexer.go | 2 +- process/dataindexer/interface.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/process/dataindexer/dataIndexer.go b/process/dataindexer/dataIndexer.go index 0e656a69..a7f0750e 100644 --- a/process/dataindexer/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -190,7 +190,7 @@ func (di *dataIndexer) GetMarshaller() marshal.Marshalizer { } // RegisterHandlerForSettingsRequest will do nothing -func (di *dataIndexer) RegisterHandlerForSettingsRequest(_ func()) error { +func (di *dataIndexer) RegisterHandlerForSettingsRequest(_ func() error) error { return nil } diff --git a/process/dataindexer/interface.go b/process/dataindexer/interface.go index 5bf8c86c..dd5def53 100644 --- a/process/dataindexer/interface.go +++ b/process/dataindexer/interface.go @@ -56,7 +56,7 @@ type Indexer interface { SaveAccounts(accountsData *outport.Accounts) error FinalizedBlock(finalizedBlock *outport.FinalizedBlock) error GetMarshaller() marshal.Marshalizer - RegisterHandlerForSettingsRequest(handler func()) error + RegisterHandlerForSettingsRequest(handler func() error) error SetCurrentSettings(cfg outport.OutportConfig) error Close() error IsInterfaceNil() bool From f65a9be1c6c18fa2d5fc10336a03d3dc00f0131d Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 12 Jun 2023 13:47:06 +0300 Subject: [PATCH 093/189] refactoring --- process/dataindexer/dataIndexer.go | 4 ++-- process/dataindexer/interface.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/process/dataindexer/dataIndexer.go b/process/dataindexer/dataIndexer.go index a7f0750e..6e571b08 100644 --- a/process/dataindexer/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -189,8 +189,8 @@ func (di *dataIndexer) GetMarshaller() marshal.Marshalizer { return di.headerMarshaller } -// RegisterHandlerForSettingsRequest will do nothing -func (di *dataIndexer) RegisterHandlerForSettingsRequest(_ func() error) error { +// RegisterHandler will do nothing +func (di *dataIndexer) RegisterHandler(_ func() error, _ string) error { return nil } diff --git a/process/dataindexer/interface.go b/process/dataindexer/interface.go index dd5def53..e72aaf78 100644 --- a/process/dataindexer/interface.go +++ b/process/dataindexer/interface.go @@ -56,7 +56,7 @@ type Indexer interface { SaveAccounts(accountsData *outport.Accounts) error FinalizedBlock(finalizedBlock *outport.FinalizedBlock) error GetMarshaller() marshal.Marshalizer - RegisterHandlerForSettingsRequest(handler func() error) error + RegisterHandler(handler func() error, topic string) error SetCurrentSettings(cfg outport.OutportConfig) error Close() error IsInterfaceNil() bool From 50828325da4a14fb95d152a844a0c9423029517e Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 14 Jun 2023 12:25:58 +0300 Subject: [PATCH 094/189] update go mod --- go.mod | 4 ++-- go.sum | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 937a679f..325b7928 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 - github.com/multiversx/mx-chain-communication-go v1.0.3-0.20230607150138-28d0e4242a47 - github.com/multiversx/mx-chain-core-go v1.2.6-0.20230529102659-223d4826d177 + github.com/multiversx/mx-chain-communication-go v1.0.3 + github.com/multiversx/mx-chain-core-go v1.2.7 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.1 github.com/stretchr/testify v1.8.2 diff --git a/go.sum b/go.sum index 75558c3f..9fcf9ca6 100644 --- a/go.sum +++ b/go.sum @@ -471,13 +471,13 @@ github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= -github.com/multiversx/mx-chain-communication-go v1.0.3-0.20230607150138-28d0e4242a47 h1:qizL4OFH8IPumgoP2gKrxamS+Oi6NMKICCrvNRp8Ctw= -github.com/multiversx/mx-chain-communication-go v1.0.3-0.20230607150138-28d0e4242a47/go.mod h1:OnnSUjnSP87H5MtQtxn33FGnTVRVgo2Huo3ijmCgN2M= +github.com/multiversx/mx-chain-communication-go v1.0.3 h1:42LkNQX+PdT4UEDO2/kbOMT/iDpEMfE0ncNldFTyFD4= +github.com/multiversx/mx-chain-communication-go v1.0.3/go.mod h1:7oTI77XfWmRWwVEbCq+pjH5CO3mJ6vEiHGMvQv6vF3Y= github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.1-0.20230510143029-ab37792342df/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.6-0.20230529102659-223d4826d177 h1:VVvMV4KufAF1uNxdTLODWwT+l+l2m4Cek8Dc4LgCo60= -github.com/multiversx/mx-chain-core-go v1.2.6-0.20230529102659-223d4826d177/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= +github.com/multiversx/mx-chain-core-go v1.2.5/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= +github.com/multiversx/mx-chain-core-go v1.2.7 h1:cP4h/B4Arnz0K6ieRd1OAf+z5gBejiMA6lzcWewR7kk= +github.com/multiversx/mx-chain-core-go v1.2.7/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-crypto-go v1.2.6/go.mod h1:rOj0Rr19HTOYt9YTeym7RKxlHt91NXln3LVKjHKVmA0= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= From 1ef42b8f48aa2aed2c7e4978cfa81a3186b48c4f Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 28 Jun 2023 18:10:28 +0300 Subject: [PATCH 095/189] update scripts --- scripts/observers/.env | 2 +- scripts/observers/config.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/observers/.env b/scripts/observers/.env index 685333ce..8d3c3713 100644 --- a/scripts/observers/.env +++ b/scripts/observers/.env @@ -2,7 +2,7 @@ MX_CHAIN_DEPLOY_GO_URL=https://github.com/multiversx/mx-chain-deploy-go MX_CHAIN_PROXY_URL=https://github.com/multiversx/mx-chain-proxy-go NODE_GO_URL="https://github.com/multiversx/mx-chain-go" -NODE_GO_BRANCH="handle-settings-in-outport-driver" +NODE_GO_BRANCH="multiple-instances-host-driver" WORKING_DIRECTORY="IndexerObservers" OBSERVER_DIR_PREFIX="observer_shard_" diff --git a/scripts/observers/config.py b/scripts/observers/config.py index eacf2f8f..0477054f 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -55,24 +55,24 @@ def update_toml_node(path, shard_id): # external.toml path_external = path / "external.toml" external_data = toml.load(str(path_external)) - external_data['HostDriverConfig']['Enabled'] = True + external_data['HostDriversConfig'][0]['Enabled'] = True port = WS_PORT_BASE + shard_id meta_port = WS_METACHAIN_PORT is_indexer_server = os.getenv('INDEXER_BINARY_SERVER') if is_indexer_server: - external_data['HostDriverConfig']['Mode'] = "client" + external_data['HostDriversConfig'][0]['Mode'] = "client" port = WS_PORT_BASE meta_port = WS_PORT_BASE if shard_id != METACHAIN: - external_data['HostDriverConfig']['URL'] = f"localhost:{str(port)}" + external_data['HostDriversConfig'][0]['URL'] = f"localhost:{str(port)}" else: - external_data['HostDriverConfig']['URL'] = f"localhost:{str(meta_port)}" + external_data['HostDriversConfig'][0]['URL'] = f"localhost:{str(meta_port)}" - external_data['HostDriverConfig']['MarshallerType'] = str(os.getenv('WS_MARSHALLER_TYPE')) - external_data['HostDriverConfig']['AcknowledgeTimeoutInSec'] = int(os.getenv('ACK_TIMEOUT_IN_SECONDS')) + external_data['HostDriversConfig'][0]['MarshallerType'] = str(os.getenv('WS_MARSHALLER_TYPE')) + external_data['HostDriversConfig'][0]['AcknowledgeTimeoutInSec'] = int(os.getenv('ACK_TIMEOUT_IN_SECONDS')) f = open(path_external, 'w') toml.dump(external_data, f) f.close() From b4023ce4ca9907b56bef9a92ab99308eea449e4e Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 29 Jun 2023 16:32:22 +0300 Subject: [PATCH 096/189] change branch --- scripts/observers/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/observers/.env b/scripts/observers/.env index 8d3c3713..f888c87d 100644 --- a/scripts/observers/.env +++ b/scripts/observers/.env @@ -2,7 +2,7 @@ MX_CHAIN_DEPLOY_GO_URL=https://github.com/multiversx/mx-chain-deploy-go MX_CHAIN_PROXY_URL=https://github.com/multiversx/mx-chain-proxy-go NODE_GO_URL="https://github.com/multiversx/mx-chain-go" -NODE_GO_BRANCH="multiple-instances-host-driver" +NODE_GO_BRANCH="rc/v1.6.0" WORKING_DIRECTORY="IndexerObservers" OBSERVER_DIR_PREFIX="observer_shard_" From f9c0c7c1e1d7a5b5c29171cd1a271380aa7ff949 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 3 Jul 2023 10:50:06 +0300 Subject: [PATCH 097/189] initial implementation --- data/logs.go | 8 + data/transaction.go | 3 +- go.mod | 2 +- go.sum | 4 +- mock/dbTransactionsHandlerStub.go | 2 +- process/elasticproc/elasticProcessor.go | 12 +- process/elasticproc/interface.go | 2 +- .../logsevents/informativeLogsProcessor.go | 54 +++++-- .../informativeLogsProcessor_test.go | 5 +- process/elasticproc/logsevents/interface.go | 7 + .../logsevents/logsAndEventsProcessor.go | 1 + process/elasticproc/logsevents/logsData.go | 2 + process/elasticproc/logsevents/statusInfo.go | 37 +++++ .../elasticproc/logsevents/statusInfo_test.go | 1 + .../transactions/scrsDataToTransactions.go | 148 +++--------------- .../scrsDataToTransactions_test.go | 14 +- process/elasticproc/transactions/serialize.go | 38 ++++- .../transactions/serialize_test.go | 6 +- .../transactions/transactionsProcessor.go | 3 +- .../transactionsProcessor_test.go | 2 +- 20 files changed, 170 insertions(+), 181 deletions(-) create mode 100644 process/elasticproc/logsevents/statusInfo.go create mode 100644 process/elasticproc/logsevents/statusInfo_test.go diff --git a/data/logs.go b/data/logs.go index d8a62ec6..a3baf0d5 100644 --- a/data/logs.go +++ b/data/logs.go @@ -24,12 +24,20 @@ type Event struct { Order int `json:"order"` } +// StatusInfo holds the fields for the transaction status +type StatusInfo struct { + CompletedEvent bool `json:"completedEvent"` + ErrorEvent bool `json:"errorEvent"` + Status string `json:"status"` +} + // PreparedLogsResults is the DTO that holds all the results after processing type PreparedLogsResults struct { Tokens TokensHandler TokensSupply TokensHandler ScDeploys map[string]*ScDeployInfo Delegators map[string]*Delegator + TxHashStatusInfo map[string]*StatusInfo TokensInfo []*TokenInfo NFTsDataUpdates []*NFTDataUpdate TokenRolesAndProperties *tokeninfo.TokenRolesAndProperties diff --git a/data/transaction.go b/data/transaction.go index fc00b972..04b7364f 100644 --- a/data/transaction.go +++ b/data/transaction.go @@ -46,6 +46,8 @@ type Transaction struct { Version uint32 `json:"version,omitempty"` GuardianAddress string `json:"guardian,omitempty"` GuardianSignature string `json:"guardianSignature,omitempty"` + ErrorEvent bool `json:"errorEvent,omitempty"` + CompletedEvent bool `json:"completedEvent,omitempty"` SmartContractResults []*ScResult `json:"-"` Hash string `json:"-"` BlockHash string `json:"-"` @@ -67,7 +69,6 @@ type PreparedResults struct { Transactions []*Transaction ScResults []*ScResult Receipts []*Receipt - TxHashStatus map[string]string TxHashFee map[string]*FeeData } diff --git a/go.mod b/go.mod index 325b7928..0d81c4a4 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/multiversx/mx-chain-communication-go v1.0.3 - github.com/multiversx/mx-chain-core-go v1.2.7 + github.com/multiversx/mx-chain-core-go v1.2.8-0.20230630085500-96880f1bce4c github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.1 github.com/stretchr/testify v1.8.2 diff --git a/go.sum b/go.sum index 9fcf9ca6..dc0ea0bb 100644 --- a/go.sum +++ b/go.sum @@ -476,8 +476,8 @@ github.com/multiversx/mx-chain-communication-go v1.0.3/go.mod h1:7oTI77XfWmRWwVE github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.5/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= -github.com/multiversx/mx-chain-core-go v1.2.7 h1:cP4h/B4Arnz0K6ieRd1OAf+z5gBejiMA6lzcWewR7kk= -github.com/multiversx/mx-chain-core-go v1.2.7/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= +github.com/multiversx/mx-chain-core-go v1.2.8-0.20230630085500-96880f1bce4c h1:UfPvZLUYZUsI+ZtxHFroyr3q+CDPgkT0n3eqKNdrJuM= +github.com/multiversx/mx-chain-core-go v1.2.8-0.20230630085500-96880f1bce4c/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-crypto-go v1.2.6/go.mod h1:rOj0Rr19HTOYt9YTeym7RKxlHt91NXln3LVKjHKVmA0= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= diff --git a/mock/dbTransactionsHandlerStub.go b/mock/dbTransactionsHandlerStub.go index 15b50e11..a810af5d 100644 --- a/mock/dbTransactionsHandlerStub.go +++ b/mock/dbTransactionsHandlerStub.go @@ -43,7 +43,7 @@ func (tps *DBTransactionProcessorStub) SerializeReceipts(recs []*data.Receipt, b } // SerializeTransactions - -func (tps *DBTransactionProcessorStub) SerializeTransactions(_ []*data.Transaction, _ map[string]string, _ uint32, _ *data.BufferSlice, _ string) error { +func (tps *DBTransactionProcessorStub) SerializeTransactions(_ []*data.Transaction, _ map[string]*data.StatusInfo, _ uint32, _ *data.BufferSlice, _ string) error { return nil } diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index b40c2a29..d70fff04 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -414,12 +414,12 @@ func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader logsData := ei.logsAndEventsProc.ExtractDataFromLogs(obh.TransactionPool.Logs, preparedResults, headerTimestamp, obh.Header.GetShardID(), obh.NumberOfShards) buffers := data.NewBufferSlice(ei.bulkRequestMaxSize) - err := ei.indexTransactions(preparedResults.Transactions, preparedResults.TxHashStatus, obh.Header, buffers) + err := ei.indexTransactions(preparedResults.Transactions, logsData.TxHashStatusInfo, obh.Header, buffers) if err != nil { return err } - err = ei.prepareAndIndexOperations(preparedResults.Transactions, preparedResults.TxHashStatus, obh.Header, preparedResults.ScResults, buffers, ei.isImportDB()) + err = ei.prepareAndIndexOperations(preparedResults.Transactions, logsData.TxHashStatusInfo, obh.Header, preparedResults.ScResults, buffers, ei.isImportDB()) if err != nil { return err } @@ -539,17 +539,17 @@ func (ei *elasticProcessor) indexScDeploys(deployData map[string]*data.ScDeployI return ei.logsAndEventsProc.SerializeSCDeploys(deployData, buffSlice, elasticIndexer.SCDeploysIndex) } -func (ei *elasticProcessor) indexTransactions(txs []*data.Transaction, txHashStatus map[string]string, header coreData.HeaderHandler, bytesBuff *data.BufferSlice) error { +func (ei *elasticProcessor) indexTransactions(txs []*data.Transaction, txHashStatusInfo map[string]*data.StatusInfo, header coreData.HeaderHandler, bytesBuff *data.BufferSlice) error { if !ei.isIndexEnabled(elasticIndexer.TransactionsIndex) { return nil } - return ei.transactionsProc.SerializeTransactions(txs, txHashStatus, header.GetShardID(), bytesBuff, elasticIndexer.TransactionsIndex) + return ei.transactionsProc.SerializeTransactions(txs, txHashStatusInfo, header.GetShardID(), bytesBuff, elasticIndexer.TransactionsIndex) } func (ei *elasticProcessor) prepareAndIndexOperations( txs []*data.Transaction, - txHashStatus map[string]string, + txHashStatusInfo map[string]*data.StatusInfo, header coreData.HeaderHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice, @@ -561,7 +561,7 @@ func (ei *elasticProcessor) prepareAndIndexOperations( processedTxs, processedSCRs := ei.operationsProc.ProcessTransactionsAndSCRs(txs, scrs, isImportDB, header.GetShardID()) - err := ei.transactionsProc.SerializeTransactions(processedTxs, txHashStatus, header.GetShardID(), buffSlice, elasticIndexer.OperationsIndex) + err := ei.transactionsProc.SerializeTransactions(processedTxs, txHashStatusInfo, header.GetShardID(), buffSlice, elasticIndexer.OperationsIndex) if err != nil { return err } diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index eeb12cd4..f7fcef17 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -66,7 +66,7 @@ type DBTransactionsHandler interface { GetHexEncodedHashesForRemove(header coreData.HeaderHandler, body *block.Body) ([]string, []string) SerializeReceipts(receipts []*data.Receipt, buffSlice *data.BufferSlice, index string) error - SerializeTransactions(transactions []*data.Transaction, txHashStatus map[string]string, selfShardID uint32, buffSlice *data.BufferSlice, index string) error + SerializeTransactions(transactions []*data.Transaction, txHashStatusInfo map[string]*data.StatusInfo, selfShardID uint32, buffSlice *data.BufferSlice, index string) error SerializeTransactionsFeeData(txHashRefund map[string]*data.FeeData, buffSlice *data.BufferSlice, index string) error SerializeScResults(scResults []*data.ScResult, buffSlice *data.BufferSlice, index string) error } diff --git a/process/elasticproc/logsevents/informativeLogsProcessor.go b/process/elasticproc/logsevents/informativeLogsProcessor.go index 1e64bf8b..d9725dc2 100644 --- a/process/elasticproc/logsevents/informativeLogsProcessor.go +++ b/process/elasticproc/logsevents/informativeLogsProcessor.go @@ -1,12 +1,9 @@ package logsevents import ( + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/transaction" -) - -const ( - writeLogOperation = "writeLog" - signalErrorOperation = "signalError" + "github.com/multiversx/mx-chain-es-indexer-go/data" ) type informativeLogsProcessor struct { @@ -16,8 +13,10 @@ type informativeLogsProcessor struct { func newInformativeLogsProcessor() *informativeLogsProcessor { return &informativeLogsProcessor{ operations: map[string]struct{}{ - writeLogOperation: {}, - signalErrorOperation: {}, + core.WriteLogIdentifier: {}, + core.SignalErrorOperation: {}, + core.CompletedTxEventIdentifier: {}, + core.InternalVMErrorsOperation: {}, }, } } @@ -31,19 +30,22 @@ func (ilp *informativeLogsProcessor) processEvent(args *argsProcessEvent) argOut tx, ok := args.txs[args.txHashHexEncoded] if !ok { - return argOutputProcessEvent{ - processed: true, - } + return processEventNoTx(args) } switch identifier { - case writeLogOperation: + case core.CompletedTxEventIdentifier: + { + tx.CompletedEvent = true + } + case core.WriteLogIdentifier: { tx.Status = transaction.TxStatusSuccess.String() } - case signalErrorOperation: + case core.SignalErrorOperation, core.InternalVMErrorsOperation: { tx.Status = transaction.TxStatusFail.String() + tx.ErrorEvent = true } } @@ -51,3 +53,31 @@ func (ilp *informativeLogsProcessor) processEvent(args *argsProcessEvent) argOut processed: true, } } + +func processEventNoTx(args *argsProcessEvent) argOutputProcessEvent { + scr, ok := args.scrs[args.txHashHexEncoded] + if !ok { + return argOutputProcessEvent{ + processed: true, + } + } + + record := &data.StatusInfo{} + switch string(args.event.GetIdentifier()) { + case core.CompletedTxEventIdentifier: + { + record.CompletedEvent = true + } + case core.SignalErrorOperation, core.InternalVMErrorsOperation: + { + record.Status = transaction.TxStatusFail.String() + record.ErrorEvent = true + } + } + + args.txHashStatusInfoProc.addRecord(scr.OriginalTxHash, record) + + return argOutputProcessEvent{ + processed: true, + } +} diff --git a/process/elasticproc/logsevents/informativeLogsProcessor_test.go b/process/elasticproc/logsevents/informativeLogsProcessor_test.go index 6e4e64c5..e1ec5d1e 100644 --- a/process/elasticproc/logsevents/informativeLogsProcessor_test.go +++ b/process/elasticproc/logsevents/informativeLogsProcessor_test.go @@ -3,6 +3,7 @@ package logsevents import ( "testing" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/stretchr/testify/require" @@ -40,7 +41,7 @@ func TestInformativeLogsProcessorWriteLog(t *testing.T) { event := &transaction.Event{ Address: []byte("addr"), - Identifier: []byte(writeLogOperation), + Identifier: []byte(core.WriteLogIdentifier), } args := &argsProcessEvent{ timestamp: 1234, @@ -73,7 +74,7 @@ func TestInformativeLogsProcessorSignalError(t *testing.T) { event := &transaction.Event{ Address: []byte("addr"), - Identifier: []byte(signalErrorOperation), + Identifier: []byte(core.SignalErrorOperation), } args := &argsProcessEvent{ timestamp: 1234, diff --git a/process/elasticproc/logsevents/interface.go b/process/elasticproc/logsevents/interface.go index c6092353..b3d4d9ba 100644 --- a/process/elasticproc/logsevents/interface.go +++ b/process/elasticproc/logsevents/interface.go @@ -10,10 +10,12 @@ type argsProcessEvent struct { txHashHexEncoded string scDeploys map[string]*data.ScDeployInfo txs map[string]*data.Transaction + scrs map[string]*data.ScResult event coreData.EventHandler tokens data.TokensHandler tokensSupply data.TokensHandler tokenRolesAndProperties *tokeninfo.TokenRolesAndProperties + txHashStatusInfoProc txHashStatusInfoHandler timestamp uint64 logAddress []byte selfShardID uint32 @@ -30,3 +32,8 @@ type argOutputProcessEvent struct { type eventsProcessor interface { processEvent(args *argsProcessEvent) argOutputProcessEvent } + +type txHashStatusInfoHandler interface { + addRecord(hash string, statusInfo *data.StatusInfo) + getAllRecords() map[string]*data.StatusInfo +} diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor.go b/process/elasticproc/logsevents/logsAndEventsProcessor.go index 2efbced0..a8c06e19 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor.go @@ -123,6 +123,7 @@ func (lep *logsAndEventsProcessor) ExtractDataFromLogs( Delegators: lep.logsData.delegators, NFTsDataUpdates: lep.logsData.nftsDataUpdates, TokenRolesAndProperties: lep.logsData.tokenRolesAndProperties, + TxHashStatusInfo: lep.logsData.txHashStatusInfoProc.getAllRecords(), } } diff --git a/process/elasticproc/logsevents/logsData.go b/process/elasticproc/logsevents/logsData.go index 0936e0f0..2d0aab4e 100644 --- a/process/elasticproc/logsevents/logsData.go +++ b/process/elasticproc/logsevents/logsData.go @@ -8,6 +8,7 @@ import ( type logsData struct { timestamp uint64 + txHashStatusInfoProc txHashStatusInfoHandler tokens data.TokensHandler tokensSupply data.TokensHandler txsMap map[string]*data.Transaction @@ -36,6 +37,7 @@ func newLogsData( ld.delegators = make(map[string]*data.Delegator) ld.nftsDataUpdates = make([]*data.NFTDataUpdate, 0) ld.tokenRolesAndProperties = tokeninfo.NewTokenRolesAndProperties() + ld.txHashStatusInfoProc = newTxHashStatusInfo() return ld } diff --git a/process/elasticproc/logsevents/statusInfo.go b/process/elasticproc/logsevents/statusInfo.go new file mode 100644 index 00000000..e440c555 --- /dev/null +++ b/process/elasticproc/logsevents/statusInfo.go @@ -0,0 +1,37 @@ +package logsevents + +import ( + "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-es-indexer-go/data" +) + +type txHashStatusInfoProc struct { + hashStatusInfo map[string]*data.StatusInfo +} + +// NewTxHashStatusInfo will create a new instance of TxHashStatusInfo +func newTxHashStatusInfo() *txHashStatusInfoProc { + return &txHashStatusInfoProc{ + hashStatusInfo: make(map[string]*data.StatusInfo), + } +} + +// AddRecord will add a new record for the given hash +func (ths *txHashStatusInfoProc) addRecord(hash string, statusInfo *data.StatusInfo) { + statusInfoFromMap, found := ths.hashStatusInfo[hash] + if !found { + ths.hashStatusInfo[hash] = statusInfo + } + + if statusInfoFromMap.Status != transaction.TxStatusFail.String() { + statusInfoFromMap.Status = statusInfo.Status + } + + statusInfoFromMap.ErrorEvent = statusInfoFromMap.ErrorEvent || statusInfo.ErrorEvent + statusInfoFromMap.CompletedEvent = statusInfoFromMap.CompletedEvent || statusInfo.CompletedEvent +} + +// GetAllRecords will return all the records +func (ths *txHashStatusInfoProc) getAllRecords() map[string]*data.StatusInfo { + return ths.hashStatusInfo +} diff --git a/process/elasticproc/logsevents/statusInfo_test.go b/process/elasticproc/logsevents/statusInfo_test.go new file mode 100644 index 00000000..2fd5536f --- /dev/null +++ b/process/elasticproc/logsevents/statusInfo_test.go @@ -0,0 +1 @@ +package logsevents diff --git a/process/elasticproc/transactions/scrsDataToTransactions.go b/process/elasticproc/transactions/scrsDataToTransactions.go index becb2299..08383872 100644 --- a/process/elasticproc/transactions/scrsDataToTransactions.go +++ b/process/elasticproc/transactions/scrsDataToTransactions.go @@ -3,13 +3,9 @@ package transactions import ( "encoding/hex" "math/big" - "strings" - "github.com/multiversx/mx-chain-core-go/core" - "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" - vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) const ( @@ -17,25 +13,11 @@ const ( ) type scrsDataToTransactions struct { - retCodes []string balanceConverter dataindexer.BalanceConverter } func newScrsDataToTransactions(balanceConverter dataindexer.BalanceConverter) *scrsDataToTransactions { return &scrsDataToTransactions{ - retCodes: []string{ - vmcommon.FunctionNotFound.String(), - vmcommon.FunctionWrongSignature.String(), - vmcommon.ContractNotFound.String(), - vmcommon.UserError.String(), - vmcommon.OutOfGas.String(), - vmcommon.AccountCollision.String(), - vmcommon.OutOfFunds.String(), - vmcommon.CallStackOverFlow.String(), - vmcommon.ContractInvalid.String(), - vmcommon.ExecutionFailed.String(), - vmcommon.UpgradeFailed.String(), - }, balanceConverter: balanceConverter, } } @@ -65,127 +47,35 @@ func (st *scrsDataToTransactions) processTransactionsAfterSCRsWereAttached(trans if len(tx.SmartContractResults) == 0 { continue } - - st.fillTxWithSCRsFields(tx) - } -} - -func (st *scrsDataToTransactions) fillTxWithSCRsFields(tx *data.Transaction) { - tx.HasSCR = true - - if isRelayedTx(tx) { - return - } - - // ignore invalid transaction because status and gas fields were already set - if tx.Status == transaction.TxStatusInvalid.String() { - return - } - - if hasSuccessfulSCRs(tx) { - return + tx.HasSCR = true } - - if hasCrossShardPendingTransfer(tx) { - return - } - - if st.hasSCRWithErrorCode(tx) { - tx.Status = transaction.TxStatusFail.String() - } -} - -func (st *scrsDataToTransactions) hasSCRWithErrorCode(tx *data.Transaction) bool { - for _, scr := range tx.SmartContractResults { - for _, codeStr := range st.retCodes { - if strings.Contains(string(scr.Data), hex.EncodeToString([]byte(codeStr))) || - scr.ReturnMessage == codeStr { - return true - } - } - } - - return false -} - -func hasSuccessfulSCRs(tx *data.Transaction) bool { - for _, scr := range tx.SmartContractResults { - if isScResultSuccessful(scr.Data) { - return true - } - } - - return false } -func hasCrossShardPendingTransfer(tx *data.Transaction) bool { - for _, scr := range tx.SmartContractResults { - splitData := strings.Split(string(scr.Data), data.AtSeparator) - if len(splitData) < 2 { - continue - } - - isMultiTransferOrNFTTransfer := splitData[0] == core.BuiltInFunctionESDTNFTTransfer || splitData[0] == core.BuiltInFunctionMultiESDTNFTTransfer - if !isMultiTransferOrNFTTransfer { - continue - } - - if scr.SenderShard != scr.ReceiverShard { - return true - } - } - - return false -} - -func (st *scrsDataToTransactions) processSCRsWithoutTx(scrs []*data.ScResult) (map[string]string, map[string]*data.FeeData) { - txHashStatus := make(map[string]string) +func (st *scrsDataToTransactions) processSCRsWithoutTx(scrs []*data.ScResult) map[string]*data.FeeData { txHashRefund := make(map[string]*data.FeeData) for _, scr := range scrs { - if scr.InitialTxGasUsed != 0 { - var feeNum float64 - var err error - initialTxFeeBig, ok := big.NewInt(0).SetString(scr.InitialTxFee, 10) - if ok { - feeNum, err = st.balanceConverter.ComputeESDTBalanceAsFloat(initialTxFeeBig) - } - if err != nil { - log.Warn("scrsDataToTransactions.processSCRsWithoutTx: cannot compute fee as num", - "initial Tx fee", initialTxFeeBig, "error", err) - } - - txHashRefund[scr.OriginalTxHash] = &data.FeeData{ - FeeNum: feeNum, - Fee: scr.InitialTxFee, - GasUsed: scr.InitialTxGasUsed, - Receiver: scr.Receiver, - } - } - - if !st.isESDTNFTTransferOrMultiTransferWithError(string(scr.Data)) { + if scr.InitialTxGasUsed == 0 { continue } - txHashStatus[scr.OriginalTxHash] = transaction.TxStatusFail.String() - } - - return txHashStatus, txHashRefund -} - -func (st *scrsDataToTransactions) isESDTNFTTransferOrMultiTransferWithError(scrData string) bool { - splitData := strings.Split(scrData, data.AtSeparator) - isMultiTransferOrNFTTransfer := splitData[0] == core.BuiltInFunctionESDTNFTTransfer || splitData[0] == core.BuiltInFunctionMultiESDTNFTTransfer - if !isMultiTransferOrNFTTransfer || len(splitData) < minNumOfArgumentsNFTTransferORMultiTransfer { - return false - } + var feeNum float64 + var err error + initialTxFeeBig, ok := big.NewInt(0).SetString(scr.InitialTxFee, 10) + if ok { + feeNum, err = st.balanceConverter.ComputeESDTBalanceAsFloat(initialTxFeeBig) + } + if err != nil { + log.Warn("scrsDataToTransactions.processSCRsWithoutTx: cannot compute fee as num", + "initial Tx fee", initialTxFeeBig, "error", err) + } - latestArgumentFromDataField := splitData[len(splitData)-1] - for _, retCode := range st.retCodes { - isWithError := latestArgumentFromDataField == hex.EncodeToString([]byte(retCode)) - if isWithError { - return true + txHashRefund[scr.OriginalTxHash] = &data.FeeData{ + FeeNum: feeNum, + Fee: scr.InitialTxFee, + GasUsed: scr.InitialTxGasUsed, + Receiver: scr.Receiver, } } - return false + return txHashRefund } diff --git a/process/elasticproc/transactions/scrsDataToTransactions_test.go b/process/elasticproc/transactions/scrsDataToTransactions_test.go index fddfbb1a..4858c642 100644 --- a/process/elasticproc/transactions/scrsDataToTransactions_test.go +++ b/process/elasticproc/transactions/scrsDataToTransactions_test.go @@ -88,19 +88,7 @@ func TestProcessTransactionsAfterSCRsWereAttached(t *testing.T) { } scrsDataToTxs.processTransactionsAfterSCRsWereAttached(txs) - require.Equal(t, "fail", tx1.Status) + require.Equal(t, "", tx1.Status) require.Equal(t, tx1.GasLimit, tx1.GasUsed) require.Equal(t, "168805000000000", tx1.Fee) } - -func TestIsESDTNFTTransferWithUserError(t *testing.T) { - t.Parallel() - - bc, _ := converters.NewBalanceConverter(18) - scrsDataToTxs := newScrsDataToTransactions(bc) - - require.False(t, scrsDataToTxs.isESDTNFTTransferOrMultiTransferWithError("ESDTNFTTransfer@45474c444d4558462d333766616239@06f5@045d2bd2629df0d2ea@0801120a00045d2bd2629df0d2ea226408f50d1a2000000000000000000500e809539d1d8febc54df4e6fe826fdc8ab6c88cf07ceb32003a3b00000007401c82df9c05a80000000000000407000000000000040f010000000009045d2bd2629df0d2ea0000000000000009045d2bd2629df0d2ea@636c61696d52657761726473")) - require.False(t, scrsDataToTxs.isESDTNFTTransferOrMultiTransferWithError("ESDTTransfer@4d45582d623662623764@74b7e37e3c2efe5f11@")) - require.False(t, scrsDataToTxs.isESDTNFTTransferOrMultiTransferWithError("ESDTNFTTransfer@45474c444d4558462d333766616239@070f@045d2bd2629df0d2ea@0801120a00045d2bd2629df0d2ea2264088f0e1a2000000000000000000500e809539d1d8febc54df4e6fe826fdc8ab6c88cf07ceb32003a3b000000074034d62af2b6930000000000000407000000000000040f010000000009045d2bd2629df0d2ea0000000000000009045d2bd2629df0d2ea@")) - require.True(t, scrsDataToTxs.isESDTNFTTransferOrMultiTransferWithError("MultiESDTNFTTransfer@02@5745474c442d626434643739@00@38e62046fb1a0000@584d45582d666461333535@07@0801120c00048907e58284c28e898e2922520807120a4d45582d3435356335371a20000000000000000005007afb2c871d1647372fd53a9eb3e53e5a8ec9251cb05532003a1e0000000a4d45582d343535633537000000000000000000000000000008e8@657865637574696f6e206661696c6564")) -} diff --git a/process/elasticproc/transactions/serialize.go b/process/elasticproc/transactions/serialize.go index 0d01c51b..3d3429bf 100644 --- a/process/elasticproc/transactions/serialize.go +++ b/process/elasticproc/transactions/serialize.go @@ -80,7 +80,7 @@ func (tdp *txsDatabaseProcessor) SerializeTransactionsFeeData(txHashRefund map[s // SerializeTransactions will serialize the transactions in a way that Elasticsearch expects a bulk request func (tdp *txsDatabaseProcessor) SerializeTransactions( transactions []*data.Transaction, - txHashStatus map[string]string, + txHashStatusInfo map[string]*data.StatusInfo, selfShardID uint32, buffSlice *data.BufferSlice, index string, @@ -97,7 +97,7 @@ func (tdp *txsDatabaseProcessor) SerializeTransactions( } } - err := serializeTxHashStatus(buffSlice, txHashStatus, index) + err := serializeTxHashStatus(buffSlice, txHashStatusInfo, index) if err != nil { return err } @@ -105,22 +105,46 @@ func (tdp *txsDatabaseProcessor) SerializeTransactions( return nil } -func serializeTxHashStatus(buffSlice *data.BufferSlice, txHashStatus map[string]string, index string) error { - for txHash, status := range txHashStatus { +func serializeTxHashStatus(buffSlice *data.BufferSlice, txHashStatusInfo map[string]*data.StatusInfo, index string) error { + for txHash, statusInfo := range txHashStatusInfo { metaData := []byte(fmt.Sprintf(`{"update":{ "_index":"%s","_id":"%s"}}%s`, index, txHash, "\n")) newTx := &data.Transaction{ - Status: status, + Status: statusInfo.Status, + ErrorEvent: statusInfo.ErrorEvent, + CompletedEvent: statusInfo.CompletedEvent, } marshaledTx, err := json.Marshal(newTx) if err != nil { return err } + marshaledStatusInfo, err := json.Marshal(statusInfo) + if err != nil { + return err + } codeToExecute := ` - ctx._source.status = params.status + if (params.statusInfo.status != "") { + ctx._source.status = params.statusInfo.status + } + + if (ctx._source.containsKey('completedEvent')) { + if (!ctx._source.completedEvent) { + ctx._source.completedEvent = params.statusInfo.completedEvent + } + } else { + ctx._source.completedEvent = params.statusInfo.completedEvent + } + + if (ctx._source.containsKey('errorEvent')) { + if (!ctx._source.errorEvent) { + ctx._source.errorEvent = params.statusInfo.errorEvent + } + } else { + ctx._source.errorEvent = params.statusInfo.errorEvent + } ` - serializedData := []byte(fmt.Sprintf(`{"script": {"source": "%s","lang": "painless","params": {"status": "%s"}},"upsert": %s }`, converters.FormatPainlessSource(codeToExecute), converters.JsonEscape(status), string(marshaledTx))) + serializedData := []byte(fmt.Sprintf(`{"script": {"source": "%s","lang": "painless","params": {"statusInfo": %s}},"upsert": %s }`, converters.FormatPainlessSource(codeToExecute), string(marshaledStatusInfo), string(marshaledTx))) err = buffSlice.PutData(metaData, serializedData) if err != nil { return err diff --git a/process/elasticproc/transactions/serialize_test.go b/process/elasticproc/transactions/serialize_test.go index d81dd6aa..bccad6ca 100644 --- a/process/elasticproc/transactions/serialize_test.go +++ b/process/elasticproc/transactions/serialize_test.go @@ -81,7 +81,7 @@ func TestSerializeTransactionsIntraShardTx(t *testing.T) { err := (&txsDatabaseProcessor{}).SerializeTransactions([]*data.Transaction{{ Hash: "txHash", SmartContractResults: []*data.ScResult{{}}, - }}, map[string]string{}, 0, buffSlice, "transactions") + }}, map[string]*data.StatusInfo{}, 0, buffSlice, "transactions") require.Nil(t, err) expectedBuff := `{ "index" : { "_index":"transactions", "_id" : "txHash" } } @@ -100,7 +100,7 @@ func TestSerializeTransactionCrossShardTxSource(t *testing.T) { ReceiverShard: 1, SmartContractResults: []*data.ScResult{{}}, Version: 1, - }}, map[string]string{}, 0, buffSlice, "transactions") + }}, map[string]*data.StatusInfo{}, 0, buffSlice, "transactions") require.Nil(t, err) expectedBuff := `{"update":{ "_index":"transactions", "_id":"txHash"}} @@ -119,7 +119,7 @@ func TestSerializeTransactionsCrossShardTxDestination(t *testing.T) { ReceiverShard: 0, SmartContractResults: []*data.ScResult{{}}, Version: 1, - }}, map[string]string{}, 0, buffSlice, "transactions") + }}, map[string]*data.StatusInfo{}, 0, buffSlice, "transactions") require.Nil(t, err) expectedBuff := `{ "index" : { "_index":"transactions", "_id" : "txHash" } } diff --git a/process/elasticproc/transactions/transactionsProcessor.go b/process/elasticproc/transactions/transactionsProcessor.go index d8f089e7..3c7494cf 100644 --- a/process/elasticproc/transactions/transactionsProcessor.go +++ b/process/elasticproc/transactions/transactionsProcessor.go @@ -124,7 +124,7 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( srcsNoTxInCurrentShard := tdp.scrsDataToTxs.attachSCRsToTransactionsAndReturnSCRsWithoutTx(normalTxs, dbSCResults) tdp.scrsDataToTxs.processTransactionsAfterSCRsWereAttached(normalTxs) - txHashStatus, txHashFee := tdp.scrsDataToTxs.processSCRsWithoutTx(srcsNoTxInCurrentShard) + txHashFee := tdp.scrsDataToTxs.processSCRsWithoutTx(srcsNoTxInCurrentShard) sliceNormalTxs := convertMapTxsToSlice(normalTxs) sliceRewardsTxs := convertMapTxsToSlice(rewardsTxs) @@ -134,7 +134,6 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( Transactions: txsSlice, ScResults: dbSCResults, Receipts: dbReceipts, - TxHashStatus: txHashStatus, TxHashFee: txHashFee, } } diff --git a/process/elasticproc/transactions/transactionsProcessor_test.go b/process/elasticproc/transactions/transactionsProcessor_test.go index d33c9571..f4eb6b66 100644 --- a/process/elasticproc/transactions/transactionsProcessor_test.go +++ b/process/elasticproc/transactions/transactionsProcessor_test.go @@ -662,6 +662,6 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { } res = txDbProc.PrepareTransactionsForDatabase(body, header, pool, false, 3) - require.Equal(t, "fail", res.Transactions[0].Status) + require.Equal(t, "success", res.Transactions[0].Status) require.Equal(t, 1, len(res.ScResults)) } From 5757741bd596d40c5229d539ee4edf46b4c8edbe Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 3 Jul 2023 11:01:53 +0300 Subject: [PATCH 098/189] unit tests status info processor --- process/elasticproc/logsevents/statusInfo.go | 1 + .../elasticproc/logsevents/statusInfo_test.go | 46 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/process/elasticproc/logsevents/statusInfo.go b/process/elasticproc/logsevents/statusInfo.go index e440c555..2ad2a5df 100644 --- a/process/elasticproc/logsevents/statusInfo.go +++ b/process/elasticproc/logsevents/statusInfo.go @@ -21,6 +21,7 @@ func (ths *txHashStatusInfoProc) addRecord(hash string, statusInfo *data.StatusI statusInfoFromMap, found := ths.hashStatusInfo[hash] if !found { ths.hashStatusInfo[hash] = statusInfo + return } if statusInfoFromMap.Status != transaction.TxStatusFail.String() { diff --git a/process/elasticproc/logsevents/statusInfo_test.go b/process/elasticproc/logsevents/statusInfo_test.go index 2fd5536f..f244c11c 100644 --- a/process/elasticproc/logsevents/statusInfo_test.go +++ b/process/elasticproc/logsevents/statusInfo_test.go @@ -1 +1,47 @@ package logsevents + +import ( + "testing" + + "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/stretchr/testify/require" +) + +func TestStatusInfoAddRecord(t *testing.T) { + t.Parallel() + + statusInfOProc := newTxHashStatusInfo() + + txHash := "txHash1" + statusInfOProc.addRecord(txHash, &data.StatusInfo{ + CompletedEvent: true, + ErrorEvent: false, + Status: transaction.TxStatusSuccess.String(), + }) + require.Equal(t, statusInfOProc.getAllRecords()[txHash], &data.StatusInfo{ + CompletedEvent: true, + Status: "success", + }) + + statusInfOProc.addRecord(txHash, &data.StatusInfo{ + ErrorEvent: true, + Status: transaction.TxStatusFail.String(), + }) + require.Equal(t, statusInfOProc.getAllRecords()[txHash], &data.StatusInfo{ + CompletedEvent: true, + ErrorEvent: true, + Status: "fail", + }) + + statusInfOProc.addRecord(txHash, &data.StatusInfo{ + ErrorEvent: false, + CompletedEvent: false, + Status: transaction.TxStatusSuccess.String(), + }) + require.Equal(t, statusInfOProc.getAllRecords()[txHash], &data.StatusInfo{ + CompletedEvent: true, + ErrorEvent: true, + Status: "fail", + }) +} From 564904e31bc4d36a220bfaf11f95f9eabc0a5a4a Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 3 Jul 2023 14:18:35 +0300 Subject: [PATCH 099/189] more unit tests --- .../informativeLogsProcessor_test.go | 129 ++++++++++++++++++ .../elasticproc/logsevents/statusInfo_test.go | 12 +- 2 files changed, 135 insertions(+), 6 deletions(-) diff --git a/process/elasticproc/logsevents/informativeLogsProcessor_test.go b/process/elasticproc/logsevents/informativeLogsProcessor_test.go index e1ec5d1e..240732e6 100644 --- a/process/elasticproc/logsevents/informativeLogsProcessor_test.go +++ b/process/elasticproc/logsevents/informativeLogsProcessor_test.go @@ -89,5 +89,134 @@ func TestInformativeLogsProcessorSignalError(t *testing.T) { res := informativeLogsProc.processEvent(args) require.Equal(t, transaction.TxStatusFail.String(), tx.Status) + require.True(t, tx.ErrorEvent) require.Equal(t, true, res.processed) } + +func TestInformativeLogsProcessorCompletedEvent(t *testing.T) { + t.Parallel() + + tx := &data.Transaction{ + GasLimit: 200000, + GasPrice: 100000, + Data: []byte("callMe"), + } + + hexEncodedTxHash := "01020304" + txs := map[string]*data.Transaction{} + txs[hexEncodedTxHash] = tx + + event := &transaction.Event{ + Address: []byte("addr"), + Identifier: []byte(core.CompletedTxEventIdentifier), + } + args := &argsProcessEvent{ + timestamp: 1234, + event: event, + logAddress: []byte("contract"), + txs: txs, + txHashHexEncoded: hexEncodedTxHash, + } + + informativeLogsProc := newInformativeLogsProcessor() + + res := informativeLogsProc.processEvent(args) + + require.True(t, tx.CompletedEvent) + require.Equal(t, true, res.processed) +} + +func TestInformativeLogsProcessorLogsGeneratedByScrsSignalError(t *testing.T) { + t.Parallel() + + txHash := "txHash" + scrHash := "scrHash" + scr := &data.ScResult{ + OriginalTxHash: txHash, + } + scrs := make(map[string]*data.ScResult) + scrs[scrHash] = scr + + event := &transaction.Event{ + Address: []byte("addr"), + Identifier: []byte(core.SignalErrorOperation), + } + + txStatusProc := newTxHashStatusInfo() + args := &argsProcessEvent{ + timestamp: 1234, + event: event, + logAddress: []byte("contract"), + scrs: scrs, + txHashHexEncoded: scrHash, + txHashStatusInfoProc: txStatusProc, + } + + informativeLogsProc := newInformativeLogsProcessor() + res := informativeLogsProc.processEvent(args) + require.True(t, res.processed) + + require.Equal(t, &data.StatusInfo{ + Status: transaction.TxStatusFail.String(), + ErrorEvent: true, + }, txStatusProc.getAllRecords()[txHash]) +} + +func TestInformativeLogsProcessorLogsGeneratedByScrsCompletedEvent(t *testing.T) { + t.Parallel() + + txHash := "txHash" + scrHash := "scrHash" + scr := &data.ScResult{ + OriginalTxHash: txHash, + } + scrs := make(map[string]*data.ScResult) + scrs[scrHash] = scr + + event := &transaction.Event{ + Address: []byte("addr"), + Identifier: []byte(core.CompletedTxEventIdentifier), + } + + txStatusProc := newTxHashStatusInfo() + args := &argsProcessEvent{ + timestamp: 1234, + event: event, + logAddress: []byte("contract"), + scrs: scrs, + txHashHexEncoded: scrHash, + txHashStatusInfoProc: txStatusProc, + } + + informativeLogsProc := newInformativeLogsProcessor() + res := informativeLogsProc.processEvent(args) + require.True(t, res.processed) + + require.Equal(t, &data.StatusInfo{ + CompletedEvent: true, + }, txStatusProc.getAllRecords()[txHash]) +} + +func TestInformativeLogsProcessorLogsGeneratedByScrNotFoundInMap(t *testing.T) { + t.Parallel() + + scrHash := "scrHash" + + event := &transaction.Event{ + Address: []byte("addr"), + Identifier: []byte(core.CompletedTxEventIdentifier), + } + + txStatusProc := newTxHashStatusInfo() + args := &argsProcessEvent{ + timestamp: 1234, + event: event, + logAddress: []byte("contract"), + txHashHexEncoded: scrHash, + txHashStatusInfoProc: txStatusProc, + } + + informativeLogsProc := newInformativeLogsProcessor() + res := informativeLogsProc.processEvent(args) + require.True(t, res.processed) +} diff --git a/process/elasticproc/logsevents/statusInfo_test.go b/process/elasticproc/logsevents/statusInfo_test.go index f244c11c..c4545cd2 100644 --- a/process/elasticproc/logsevents/statusInfo_test.go +++ b/process/elasticproc/logsevents/statusInfo_test.go @@ -19,29 +19,29 @@ func TestStatusInfoAddRecord(t *testing.T) { ErrorEvent: false, Status: transaction.TxStatusSuccess.String(), }) - require.Equal(t, statusInfOProc.getAllRecords()[txHash], &data.StatusInfo{ + require.Equal(t, &data.StatusInfo{ CompletedEvent: true, Status: "success", - }) + }, statusInfOProc.getAllRecords()[txHash]) statusInfOProc.addRecord(txHash, &data.StatusInfo{ ErrorEvent: true, Status: transaction.TxStatusFail.String(), }) - require.Equal(t, statusInfOProc.getAllRecords()[txHash], &data.StatusInfo{ + require.Equal(t, &data.StatusInfo{ CompletedEvent: true, ErrorEvent: true, Status: "fail", - }) + }, statusInfOProc.getAllRecords()[txHash]) statusInfOProc.addRecord(txHash, &data.StatusInfo{ ErrorEvent: false, CompletedEvent: false, Status: transaction.TxStatusSuccess.String(), }) - require.Equal(t, statusInfOProc.getAllRecords()[txHash], &data.StatusInfo{ + require.Equal(t, &data.StatusInfo{ CompletedEvent: true, ErrorEvent: true, Status: "fail", - }) + }, statusInfOProc.getAllRecords()[txHash]) } From 1f87a735e774dad9988a6624dcb945f597114bd6 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 3 Jul 2023 16:08:25 +0300 Subject: [PATCH 100/189] fix integration tests --- .../accountsESDTWithTokenType_test.go | 4 +++ integrationtests/claimRewards_test.go | 1 + integrationtests/createNFTWithTags_test.go | 3 ++ .../issueTokenAndSetRoles_test.go | 6 ++++ integrationtests/issueToken_test.go | 4 +++ integrationtests/nftIssueCreateBurn_test.go | 2 ++ .../nftTransferCrossShard_test.go | 29 +++++++++++++++++++ integrationtests/scCallIntraShard_test.go | 15 ++++++++++ ...tion-after-execution-of-scr-dst-shard.json | 3 +- .../tx-complete-with-status.json | 3 +- .../tx-nft-transfer-failed-on-dst.json | 3 +- .../tx-with-status-only.json | 3 +- .../scCallIntraShard/sc-call-fail.json | 5 +++- .../logsevents/logsAndEventsProcessor.go | 2 ++ process/elasticproc/transactions/serialize.go | 26 +++++++---------- .../smartContractResultsProcessor.go | 20 ++++++------- 16 files changed, 98 insertions(+), 31 deletions(-) diff --git a/integrationtests/accountsESDTWithTokenType_test.go b/integrationtests/accountsESDTWithTokenType_test.go index 9d5833aa..bb25ce0c 100644 --- a/integrationtests/accountsESDTWithTokenType_test.go +++ b/integrationtests/accountsESDTWithTokenType_test.go @@ -43,6 +43,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { { TxHash: hex.EncodeToString([]byte("h1")), Log: &transaction.Log{ + Address: decodeAddress(address), Events: []*transaction.Event{ { Address: decodeAddress(address), @@ -104,6 +105,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { { TxHash: hex.EncodeToString([]byte("h1")), Log: &transaction.Log{ + Address: decodeAddress(address), Events: []*transaction.Event{ { Address: decodeAddress(address), @@ -176,6 +178,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) { TxHash: hex.EncodeToString([]byte("h1")), Log: &transaction.Log{ + Address: decodeAddress(address), Events: []*transaction.Event{ { Address: decodeAddress(address), @@ -215,6 +218,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) { TxHash: hex.EncodeToString([]byte("h1")), Log: &transaction.Log{ + Address: decodeAddress(address), Events: []*transaction.Event{ { Address: decodeAddress(address), diff --git a/integrationtests/claimRewards_test.go b/integrationtests/claimRewards_test.go index e3c237bf..8cf7eac1 100644 --- a/integrationtests/claimRewards_test.go +++ b/integrationtests/claimRewards_test.go @@ -109,6 +109,7 @@ func TestTransactionWithClaimRewardsGasRefund(t *testing.T) { { TxHash: hex.EncodeToString(txHash), Log: &transaction.Log{ + Address: decodeAddress(addressSender), Events: []*transaction.Event{ { Address: decodeAddress(addressSender), diff --git a/integrationtests/createNFTWithTags_test.go b/integrationtests/createNFTWithTags_test.go index fa572fe2..09a8d46a 100644 --- a/integrationtests/createNFTWithTags_test.go +++ b/integrationtests/createNFTWithTags_test.go @@ -52,6 +52,7 @@ func TestCreateNFTWithTags(t *testing.T) { { TxHash: hex.EncodeToString([]byte("h1")), Log: &transaction.Log{ + Address: decodeAddress(address1), Events: []*transaction.Event{ { Address: decodeAddress(address1), @@ -120,6 +121,7 @@ func TestCreateNFTWithTags(t *testing.T) { { TxHash: hex.EncodeToString([]byte("h1")), Log: &transaction.Log{ + Address: decodeAddress(address1), Events: []*transaction.Event{ { Address: decodeAddress(address1), @@ -169,6 +171,7 @@ func TestCreateNFTWithTags(t *testing.T) { { TxHash: hex.EncodeToString([]byte("h1")), Log: &transaction.Log{ + Address: decodeAddress(address1), Events: []*transaction.Event{ { Address: decodeAddress(address1), diff --git a/integrationtests/issueTokenAndSetRoles_test.go b/integrationtests/issueTokenAndSetRoles_test.go index 25920c50..53524d24 100644 --- a/integrationtests/issueTokenAndSetRoles_test.go +++ b/integrationtests/issueTokenAndSetRoles_test.go @@ -39,6 +39,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { { TxHash: hex.EncodeToString([]byte("h1")), Log: &transaction.Log{ + Address: decodeAddress(address1), Events: []*transaction.Event{ { Address: decodeAddress(address1), @@ -72,6 +73,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { { TxHash: hex.EncodeToString([]byte("h1")), Log: &transaction.Log{ + Address: decodeAddress(address1), Events: []*transaction.Event{ { Address: decodeAddress(address1), @@ -101,6 +103,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { { TxHash: hex.EncodeToString([]byte("h1")), Log: &transaction.Log{ + Address: decodeAddress(address1), Events: []*transaction.Event{ { Address: decodeAddress(address1), @@ -134,6 +137,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { { TxHash: hex.EncodeToString([]byte("h1")), Log: &transaction.Log{ + Address: decodeAddress(address1), Events: []*transaction.Event{ { Address: decodeAddress(address1), @@ -181,6 +185,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { { TxHash: hex.EncodeToString([]byte("h1")), Log: &transaction.Log{ + Address: decodeAddress(address1), Events: []*transaction.Event{ { Address: decodeAddress(address1), @@ -210,6 +215,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { { TxHash: hex.EncodeToString([]byte("h1")), Log: &transaction.Log{ + Address: decodeAddress(address1), Events: []*transaction.Event{ { Address: decodeAddress(address1), diff --git a/integrationtests/issueToken_test.go b/integrationtests/issueToken_test.go index d501c7d2..11cf5ded 100644 --- a/integrationtests/issueToken_test.go +++ b/integrationtests/issueToken_test.go @@ -38,6 +38,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { { TxHash: hex.EncodeToString([]byte("h1")), Log: &transaction.Log{ + Address: decodeAddress(address1), Events: []*transaction.Event{ { Address: decodeAddress(address1), @@ -70,6 +71,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { { TxHash: hex.EncodeToString([]byte("h1")), Log: &transaction.Log{ + Address: decodeAddress(address1), Events: []*transaction.Event{ { Address: decodeAddress(address1), @@ -101,6 +103,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { { TxHash: hex.EncodeToString([]byte("h1")), Log: &transaction.Log{ + Address: decodeAddress(address1), Events: []*transaction.Event{ { Address: decodeAddress(address1), @@ -128,6 +131,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { { TxHash: hex.EncodeToString([]byte("h1")), Log: &transaction.Log{ + Address: decodeAddress(address1), Events: []*transaction.Event{ { Address: decodeAddress(address1), diff --git a/integrationtests/nftIssueCreateBurn_test.go b/integrationtests/nftIssueCreateBurn_test.go index 5bd8f628..74a652e1 100644 --- a/integrationtests/nftIssueCreateBurn_test.go +++ b/integrationtests/nftIssueCreateBurn_test.go @@ -41,6 +41,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { { TxHash: hex.EncodeToString([]byte("h1")), Log: &transaction.Log{ + Address: decodeAddress(address1), Events: []*transaction.Event{ { Address: decodeAddress(address1), @@ -121,6 +122,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { { TxHash: hex.EncodeToString([]byte("h1")), Log: &transaction.Log{ + Address: decodeAddress(address1), Events: []*transaction.Event{ { Address: decodeAddress(address1), diff --git a/integrationtests/nftTransferCrossShard_test.go b/integrationtests/nftTransferCrossShard_test.go index eea7c1d9..4227924a 100644 --- a/integrationtests/nftTransferCrossShard_test.go +++ b/integrationtests/nftTransferCrossShard_test.go @@ -7,6 +7,7 @@ import ( "math/big" "testing" + "github.com/multiversx/mx-chain-core-go/core" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" @@ -271,6 +272,20 @@ func TestNFTTransferCrossShard(t *testing.T) { Data: []byte("ESDTNFTTransfer@434f4c45435449452d323663313838@01@01@08011202000122e50108011204434f4f4c1a20e0f3ecf555f63f2d101241dfc98b4614aff9284edd50b46a1c6e36b83558744d20c4132a2e516d5a7961565631786a7866446255575a503178655a7676544d3156686f61346f594752444d706d4a727a52435a324368747470733a2f2f697066732e696f2f697066732f516d5a7961565631786a7866446255575a503178655a7676544d3156686f61346f594752444d706d4a727a52435a3a41746167733a436f6f6c3b6d657461646174613a516d5869417850396e535948515954546143357358717a4d32645856334142516145355241725932777a4e686179@75736572206572726f72"), }, FeeInfo: &outport.FeeInfo{}}, }, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString(scr3WithErrHash), + Log: &transaction.Log{ + Address: decodeAddress(address1), + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.InternalVMErrorsOperation), + }, + }, + }, + }, + }, } err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, testNumOfShards)) @@ -337,6 +352,20 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t Data: []byte("ESDTNFTTransfer@434f4c4c454354494f4e2d323663313838@01@01@08011202000122e50108011204434f4f4c1a20e0f3ecf555f63f2d101241dfc98b4614aff9284edd50b46a1c6e36b83558744d20c4132a2e516d5a7961565631786a7866446255575a503178655a7676544d3156686f61346f594752444d706d4a727a52435a324368747470733a2f2f697066732e696f2f697066732f516d5a7961565631786a7866446255575a503178655a7676544d3156686f61346f594752444d706d4a727a52435a3a41746167733a436f6f6c3b6d657461646174613a516d5869417850396e535948515954546143357358717a4d32645856334142516145355241725932777a4e686179@75736572206572726f72"), }, FeeInfo: &outport.FeeInfo{}}, }, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString(scr3WithErrHash), + Log: &transaction.Log{ + Address: decodeAddress(address1), + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.SignalErrorOperation), + }, + }, + }, + }, + }, } ids := []string{hex.EncodeToString(txHash)} diff --git a/integrationtests/scCallIntraShard_test.go b/integrationtests/scCallIntraShard_test.go index 7e7492dc..3c45d6ae 100644 --- a/integrationtests/scCallIntraShard_test.go +++ b/integrationtests/scCallIntraShard_test.go @@ -7,6 +7,7 @@ import ( "math/big" "testing" + "github.com/multiversx/mx-chain-core-go/core" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" @@ -81,6 +82,20 @@ func TestTransactionWithSCCallFail(t *testing.T) { ReturnMessage: []byte("total delegation cap reached"), }, FeeInfo: &outport.FeeInfo{}}, }, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString(txHash), + Log: &transaction.Log{ + Address: decodeAddress(address1), + Events: []*transaction.Event{ + { + Address: decodeAddress(address1), + Identifier: []byte(core.SignalErrorOperation), + }, + }, + }, + }, + }, } err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) diff --git a/integrationtests/testdata/multiTransferWithScCallAndErrorSignaledBySC/transaction-after-execution-of-scr-dst-shard.json b/integrationtests/testdata/multiTransferWithScCallAndErrorSignaledBySC/transaction-after-execution-of-scr-dst-shard.json index 1956ce24..032af67c 100644 --- a/integrationtests/testdata/multiTransferWithScCallAndErrorSignaledBySC/transaction-after-execution-of-scr-dst-shard.json +++ b/integrationtests/testdata/multiTransferWithScCallAndErrorSignaledBySC/transaction-after-execution-of-scr-dst-shard.json @@ -41,5 +41,6 @@ "receiverShard": 0, "operation": "MultiESDTNFTTransfer", "status": "fail", - "searchOrder": 0 + "searchOrder": 0, + "errorEvent": true } diff --git a/integrationtests/testdata/nftTransferCrossShard/tx-complete-with-status.json b/integrationtests/testdata/nftTransferCrossShard/tx-complete-with-status.json index 0fb5adbd..62168879 100644 --- a/integrationtests/testdata/nftTransferCrossShard/tx-complete-with-status.json +++ b/integrationtests/testdata/nftTransferCrossShard/tx-complete-with-status.json @@ -35,5 +35,6 @@ "timestamp": 5040, "status": "fail", "initialPaidFee": "595490000000000", - "searchOrder": 0 + "searchOrder": 0, + "errorEvent": true } diff --git a/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-failed-on-dst.json b/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-failed-on-dst.json index af8b4ee9..70f07153 100644 --- a/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-failed-on-dst.json +++ b/integrationtests/testdata/nftTransferCrossShard/tx-nft-transfer-failed-on-dst.json @@ -35,5 +35,6 @@ "receiversShardIDs": [ 2 ], - "operation": "ESDTNFTTransfer" + "operation": "ESDTNFTTransfer", + "errorEvent": true } diff --git a/integrationtests/testdata/nftTransferCrossShard/tx-with-status-only.json b/integrationtests/testdata/nftTransferCrossShard/tx-with-status-only.json index b0bfc6b1..c9a4c62a 100644 --- a/integrationtests/testdata/nftTransferCrossShard/tx-with-status-only.json +++ b/integrationtests/testdata/nftTransferCrossShard/tx-with-status-only.json @@ -17,5 +17,6 @@ "signature": "", "timestamp": 0, "status": "fail", - "searchOrder": 0 + "searchOrder": 0, + "errorEvent": true } diff --git a/integrationtests/testdata/scCallIntraShard/sc-call-fail.json b/integrationtests/testdata/scCallIntraShard/sc-call-fail.json index dbffafcb..aaea4caf 100644 --- a/integrationtests/testdata/scCallIntraShard/sc-call-fail.json +++ b/integrationtests/testdata/scCallIntraShard/sc-call-fail.json @@ -22,5 +22,8 @@ "hasScResults": true, "operation": "transfer", "isScCall": true, - "function": "delegate" + "function": "delegate", + "errorEvent": true, + "hasLogs": true, + "hasOperations": true } diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor.go b/process/elasticproc/logsevents/logsAndEventsProcessor.go index a8c06e19..e29e8703 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor.go @@ -148,7 +148,9 @@ func (lep *logsAndEventsProcessor) processEvent(logHashHexEncoded string, logAdd timestamp: lep.logsData.timestamp, scDeploys: lep.logsData.scDeploys, txs: lep.logsData.txsMap, + scrs: lep.logsData.scrsMap, tokenRolesAndProperties: lep.logsData.tokenRolesAndProperties, + txHashStatusInfoProc: lep.logsData.txHashStatusInfoProc, selfShardID: shardID, numOfShards: numOfShards, }) diff --git a/process/elasticproc/transactions/serialize.go b/process/elasticproc/transactions/serialize.go index 3d3429bf..77746e64 100644 --- a/process/elasticproc/transactions/serialize.go +++ b/process/elasticproc/transactions/serialize.go @@ -124,27 +124,19 @@ func serializeTxHashStatus(buffSlice *data.BufferSlice, txHashStatusInfo map[str } codeToExecute := ` - if (params.statusInfo.status != "") { - ctx._source.status = params.statusInfo.status + if (!params.statusInfo.status.isEmpty()) { + ctx._source.status = params.statusInfo.status; } - if (ctx._source.containsKey('completedEvent')) { - if (!ctx._source.completedEvent) { - ctx._source.completedEvent = params.statusInfo.completedEvent - } - } else { - ctx._source.completedEvent = params.statusInfo.completedEvent + if (params.statusInfo.completedEvent) { + ctx._source.completedEvent = params.statusInfo.completedEvent; } - if (ctx._source.containsKey('errorEvent')) { - if (!ctx._source.errorEvent) { - ctx._source.errorEvent = params.statusInfo.errorEvent - } - } else { - ctx._source.errorEvent = params.statusInfo.errorEvent + if (params.statusInfo.errorEvent) { + ctx._source.errorEvent = params.statusInfo.errorEvent; } ` - serializedData := []byte(fmt.Sprintf(`{"script": {"source": "%s","lang": "painless","params": {"statusInfo": %s}},"upsert": %s }`, converters.FormatPainlessSource(codeToExecute), string(marshaledStatusInfo), string(marshaledTx))) + serializedData := []byte(fmt.Sprintf(`{"script": {"source": "%s","lang": "painless","params": {"statusInfo": %s}}, "upsert": %s }`, converters.FormatPainlessSource(codeToExecute), string(marshaledStatusInfo), string(marshaledTx))) err = buffSlice.PutData(metaData, serializedData) if err != nil { return err @@ -194,9 +186,11 @@ func prepareNFTESDTTransferOrMultiESDTTransfer(marshaledTx []byte) ([]byte, erro if ('create' == ctx.op) { ctx._source = params.tx; } else { - def status = ctx._source.status; + def status = ctx._source.status; + def errorEvent = ctx._source.errorEvent; ctx._source = params.tx; ctx._source.status = status; + ctx._source.errorEvent = errorEvent; } ` serializedData := []byte(fmt.Sprintf(`{"scripted_upsert": true, "script":{"source":"%s","lang": "painless","params":{"tx": %s}},"upsert":{}}`, diff --git a/process/elasticproc/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go index 0ba28194..3f811e5e 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -43,13 +43,13 @@ func newSmartContractResultsProcessor( func (proc *smartContractResultsProcessor) processSCRs( body *block.Body, header coreData.HeaderHandler, - txsHandler map[string]*outport.SCRInfo, + scrs map[string]*outport.SCRInfo, numOfShards uint32, ) []*indexerData.ScResult { - allSCRs := make([]*indexerData.ScResult, 0, len(txsHandler)) + allSCRs := make([]*indexerData.ScResult, 0, len(scrs)) // a copy of the SCRS map is needed because proc.processSCRsFromMiniblock would remove items from the original map - workingSCRSMap := copySCRSMap(txsHandler) + workingSCRSMap := copySCRSMap(scrs) for _, mb := range body.MiniBlocks { if mb.Type != block.SmartContractResultBlock { continue @@ -61,8 +61,8 @@ func (proc *smartContractResultsProcessor) processSCRs( } selfShardID := header.GetShardID() - for scrHash, noMBScrInfo := range workingSCRSMap { - indexerScr := proc.prepareSmartContractResult([]byte(scrHash), nil, noMBScrInfo, header, selfShardID, selfShardID, numOfShards) + for scrHashHex, noMBScrInfo := range workingSCRSMap { + indexerScr := proc.prepareSmartContractResult(scrHashHex, nil, noMBScrInfo, header, selfShardID, selfShardID, numOfShards) allSCRs = append(allSCRs, indexerScr) } @@ -93,7 +93,7 @@ func (proc *smartContractResultsProcessor) processSCRsFromMiniblock( continue } - indexerSCR := proc.prepareSmartContractResult(scrHash, mbHash, scrInfo, header, mb.SenderShardID, mb.ReceiverShardID, numOfShards) + indexerSCR := proc.prepareSmartContractResult(hex.EncodeToString(scrHash), mbHash, scrInfo, header, mb.SenderShardID, mb.ReceiverShardID, numOfShards) indexerSCRs = append(indexerSCRs, indexerSCR) delete(scrs, scrHashHex) @@ -103,7 +103,7 @@ func (proc *smartContractResultsProcessor) processSCRsFromMiniblock( } func (proc *smartContractResultsProcessor) prepareSmartContractResult( - scrHash []byte, + scrHashHex string, mbHash []byte, scrInfo *outport.SCRInfo, header coreData.HeaderHandler, @@ -140,13 +140,13 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( valueNum, err := proc.balanceConverter.ComputeESDTBalanceAsFloat(scr.Value) if err != nil { log.Warn("smartContractResultsProcessor.prepareSmartContractResult cannot compute scr value as num", - "value", scr.Value, "hash", scrHash, "error", err) + "value", scr.Value, "hash", scrHashHex, "error", err) } esdtValuesNum, err := proc.balanceConverter.ComputeSliceOfStringsAsFloat(res.ESDTValues) if err != nil { log.Warn("smartContractResultsProcessor.prepareSmartContractResult cannot compute scr esdt values as num", - "esdt values", res.ESDTValues, "hash", scrHash, "error", err) + "esdt values", res.ESDTValues, "hash", scrHashHex, "error", err) } var esdtValues []string @@ -156,7 +156,7 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( feeInfo := getFeeInfo(scrInfo) return &indexerData.ScResult{ - Hash: hex.EncodeToString(scrHash), + Hash: scrHashHex, MBHash: hexEncodedMBHash, Nonce: scr.Nonce, GasLimit: scr.GasLimit, From c08afe7102a9813785340d11a229b02b849a50f9 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 3 Jul 2023 16:11:58 +0300 Subject: [PATCH 101/189] fix unit test --- .../transactions/smartContractResultsProcessor_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/process/elasticproc/transactions/smartContractResultsProcessor_test.go b/process/elasticproc/transactions/smartContractResultsProcessor_test.go index 30d44c1b..3feb64f4 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor_test.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor_test.go @@ -60,7 +60,7 @@ func TestPrepareSmartContractResult(t *testing.T) { header := &block.Header{TimeStamp: 100} mbHash := []byte("hash") - scRes := scrsProc.prepareSmartContractResult([]byte(scHash), mbHash, scrInfo, header, 0, 1, 3) + scRes := scrsProc.prepareSmartContractResult(scHash, mbHash, scrInfo, header, 0, 1, 3) senderAddr, err := pubKeyConverter.Encode(sndAddr) require.Nil(t, err) @@ -69,7 +69,7 @@ func TestPrepareSmartContractResult(t *testing.T) { expectedTx := &data.ScResult{ Nonce: nonce, - Hash: hex.EncodeToString([]byte(scHash)), + Hash: scHash, PrevTxHash: hex.EncodeToString(txHash), MBHash: hex.EncodeToString(mbHash), Code: string(code), From bdb6ee9a50cb2de6be24419a2d25df77fcaf0228 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 4 Jul 2023 13:50:19 +0300 Subject: [PATCH 102/189] integration test --- integrationtests/scCallWithIssueEsdt_test.go | 239 ++++++++++++++++++ ...allback-executed-on-destination-shard.json | 22 ++ .../scr-with-callback-executed-on-source.json | 20 ++ ...h-issue-executed-on-destination-shard.json | 21 ++ ...r-with-issue-executed-on-source-shard.json | 21 ++ ...tion-of-callback-on-destination-shard.json | 27 ++ .../tx-after-execution-on-source-shard.json | 26 ++ ...tion-of-callback-on-destination-shard.json | 28 ++ 8 files changed, 404 insertions(+) create mode 100644 integrationtests/scCallWithIssueEsdt_test.go create mode 100644 integrationtests/testdata/scCallWithIssueEsdt/scr-with-callback-executed-on-destination-shard.json create mode 100644 integrationtests/testdata/scCallWithIssueEsdt/scr-with-callback-executed-on-source.json create mode 100644 integrationtests/testdata/scCallWithIssueEsdt/scr-with-issue-executed-on-destination-shard.json create mode 100644 integrationtests/testdata/scCallWithIssueEsdt/scr-with-issue-executed-on-source-shard.json create mode 100644 integrationtests/testdata/scCallWithIssueEsdt/tx-after-execution-of-callback-on-destination-shard.json create mode 100644 integrationtests/testdata/scCallWithIssueEsdt/tx-after-execution-on-source-shard.json create mode 100644 integrationtests/testdata/scCallWithIssueEsdt/tx-in-op-index-execution-of-callback-on-destination-shard.json diff --git a/integrationtests/scCallWithIssueEsdt_test.go b/integrationtests/scCallWithIssueEsdt_test.go new file mode 100644 index 00000000..8dea7be4 --- /dev/null +++ b/integrationtests/scCallWithIssueEsdt_test.go @@ -0,0 +1,239 @@ +//go:build integrationtests + +package integrationtests + +import ( + "encoding/hex" + "math/big" + "testing" + + "github.com/multiversx/mx-chain-core-go/core" + dataBlock "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-core-go/data/smartContractResult" + "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-core-go/data/vm" + indexerData "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/stretchr/testify/require" +) + +func TestScCallIntraShardWithIssueESDT(t *testing.T) { + setLogLevelDebug() + + esClient, err := createESClient(esURL) + require.Nil(t, err) + + esProc, err := CreateElasticProcessor(esClient) + require.Nil(t, err) + + txHash := []byte("txWithScCall") + header := &dataBlock.Header{ + Round: 50, + TimeStamp: 5040, + } + scrWithIssueHash := []byte("scrWithIssue") + body := &dataBlock.Body{ + MiniBlocks: dataBlock.MiniBlockSlice{ + { + Type: dataBlock.TxBlock, + SenderShardID: 0, + ReceiverShardID: 0, + TxHashes: [][]byte{txHash}, + }, + { + Type: dataBlock.SmartContractResultBlock, + SenderShardID: 0, + ReceiverShardID: core.MetachainShardId, + TxHashes: [][]byte{scrWithIssueHash}, + }, + }, + } + + sndAddress := "erd148m2sx48mfm8322c2kpfmgj78g5j0x7r6z6y4z8j28qk45a74nwq5pq2ts" + contractAddress := "erd1qqqqqqqqqqqqqpgqahumqen35dr9k4rmcnd70mqt5t4mt7ey4nwqwjme9g" + tx := &transaction.Transaction{ + Nonce: 46, + SndAddr: decodeAddress(sndAddress), + RcvAddr: decodeAddress(contractAddress), + GasLimit: 75_000_000, + GasPrice: 1000000000, + Data: []byte("issueToken@4D79546573744E667464@544553544E4654"), + Value: big.NewInt(50000000000000000), + } + + esdtSystemSC := "erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u" + scrWithIssueESDT := &smartContractResult.SmartContractResult{ + Nonce: 0, + SndAddr: decodeAddress(contractAddress), + RcvAddr: decodeAddress(esdtSystemSC), + OriginalTxHash: txHash, + PrevTxHash: txHash, + Data: []byte("issueNonFungible@4d79546573744e667464@544553544e4654@63616e467265657a65@74727565@63616e57697065@74727565@63616e5061757365@74727565@63616e4368616e67654f776e6572@66616c7365@63616e55706772616465@66616c7365@63616e4164645370656369616c526f6c6573@74727565@58f638"), + Value: big.NewInt(50000000000000000), + CallType: vm.AsynchronousCall, + } + txInfo := &outport.TxInfo{ + Transaction: tx, + FeeInfo: &outport.FeeInfo{ + GasUsed: 75_000_000, + Fee: big.NewInt(867810000000000), + InitialPaidFee: big.NewInt(867810000000000), + }, + } + + scrInfoWithIssue := &outport.SCRInfo{SmartContractResult: scrWithIssueESDT, FeeInfo: &outport.FeeInfo{}} + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + hex.EncodeToString(txHash): txInfo, + }, + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(scrWithIssueHash): scrInfoWithIssue, + }, + } + + // ############################ + // execute on the source shard + // ############################ + + header.ShardID = 0 + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) + require.Nil(t, err) + + ids := []string{hex.EncodeToString(txHash)} + genericResponse := &GenericResponse{} + err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse) + require.Nil(t, err) + + require.JSONEq(t, + readExpectedResult("./testdata/scCallWithIssueEsdt/tx-after-execution-on-source-shard.json"), + string(genericResponse.Docs[0].Source), + ) + + ids = []string{hex.EncodeToString(scrWithIssueHash)} + err = esClient.DoMultiGet(ids, indexerData.OperationsIndex, true, genericResponse) + require.Nil(t, err) + + require.JSONEq(t, + readExpectedResult("./testdata/scCallWithIssueEsdt/scr-with-issue-executed-on-source-shard.json"), + string(genericResponse.Docs[0].Source), + ) + + // ############################ + // execute scr on the destination shard (metachain) + // ############################ + + scrWithCallBackHash := []byte("scrWithCallback") + header.ShardID = core.MetachainShardId + body = &dataBlock.Body{ + MiniBlocks: dataBlock.MiniBlockSlice{ + { + Type: dataBlock.SmartContractResultBlock, + SenderShardID: 0, + ReceiverShardID: core.MetachainShardId, + TxHashes: [][]byte{scrWithIssueHash}, + }, + { + Type: dataBlock.SmartContractResultBlock, + SenderShardID: core.MetachainShardId, + ReceiverShardID: 0, + TxHashes: [][]byte{scrWithCallBackHash}, + }, + }, + } + + scrWithCallBack := &smartContractResult.SmartContractResult{ + Nonce: 0, + Value: big.NewInt(0), + SndAddr: decodeAddress(esdtSystemSC), + RcvAddr: decodeAddress(contractAddress), + Data: []byte("@00@544553544e46542d643964353463"), + OriginalTxHash: txHash, + PrevTxHash: scrWithIssueHash, + CallType: vm.AsynchronousCallBack, + } + scrInfoWithCallBack := &outport.SCRInfo{SmartContractResult: scrWithCallBack, FeeInfo: &outport.FeeInfo{}} + pool = &outport.TransactionPool{ + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(scrWithIssueHash): scrInfoWithIssue, + hex.EncodeToString(scrWithCallBackHash): scrInfoWithCallBack, + }, + } + + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) + require.Nil(t, err) + + ids = []string{hex.EncodeToString(scrWithIssueHash), hex.EncodeToString(scrWithCallBackHash)} + err = esClient.DoMultiGet(ids, indexerData.OperationsIndex, true, genericResponse) + require.Nil(t, err) + + require.JSONEq(t, + readExpectedResult("./testdata/scCallWithIssueEsdt/scr-with-issue-executed-on-destination-shard.json"), + string(genericResponse.Docs[0].Source), + ) + require.JSONEq(t, + readExpectedResult("./testdata/scCallWithIssueEsdt/scr-with-callback-executed-on-source.json"), + string(genericResponse.Docs[1].Source), + ) + + // ############################ + // execute scr with callback on the destination shard (0) + // ############################ + header.ShardID = 0 + body = &dataBlock.Body{ + MiniBlocks: dataBlock.MiniBlockSlice{ + { + Type: dataBlock.SmartContractResultBlock, + SenderShardID: core.MetachainShardId, + ReceiverShardID: 0, + TxHashes: [][]byte{scrWithCallBackHash}, + }, + }, + } + pool = &outport.TransactionPool{ + SmartContractResults: map[string]*outport.SCRInfo{ + hex.EncodeToString(scrWithCallBackHash): scrInfoWithCallBack, + }, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString(scrWithCallBackHash), + Log: &transaction.Log{ + Address: decodeAddress(contractAddress), + Events: []*transaction.Event{ + { + Address: decodeAddress(contractAddress), + Identifier: []byte(core.SignalErrorOperation), + }, + { + Address: decodeAddress(contractAddress), + Identifier: []byte(core.InternalVMErrorsOperation), + }, + }, + }, + }, + }, + } + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) + require.Nil(t, err) + + ids = []string{hex.EncodeToString(txHash)} + err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse) + require.Nil(t, err) + + require.JSONEq(t, + readExpectedResult("./testdata/scCallWithIssueEsdt/tx-after-execution-of-callback-on-destination-shard.json"), + string(genericResponse.Docs[0].Source), + ) + + ids = []string{hex.EncodeToString(txHash), hex.EncodeToString(scrWithCallBackHash)} + err = esClient.DoMultiGet(ids, indexerData.OperationsIndex, true, genericResponse) + require.Nil(t, err) + + require.JSONEq(t, + readExpectedResult("./testdata/scCallWithIssueEsdt/tx-in-op-index-execution-of-callback-on-destination-shard.json"), + string(genericResponse.Docs[0].Source), + ) + require.JSONEq(t, + readExpectedResult("./testdata/scCallWithIssueEsdt/scr-with-callback-executed-on-destination-shard.json"), + string(genericResponse.Docs[1].Source), + ) +} diff --git a/integrationtests/testdata/scCallWithIssueEsdt/scr-with-callback-executed-on-destination-shard.json b/integrationtests/testdata/scCallWithIssueEsdt/scr-with-callback-executed-on-destination-shard.json new file mode 100644 index 00000000..3a070a6f --- /dev/null +++ b/integrationtests/testdata/scCallWithIssueEsdt/scr-with-callback-executed-on-destination-shard.json @@ -0,0 +1,22 @@ +{ + "miniBlockHash": "49345577b344acd366143ad353904657307dc21f57c3817bf164975de3858459", + "nonce": 0, + "gasLimit": 0, + "gasPrice": 0, + "value": "0", + "valueNum": 0, + "sender": "erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u", + "receiver": "erd1qqqqqqqqqqqqqpgqahumqen35dr9k4rmcnd70mqt5t4mt7ey4nwqwjme9g", + "senderShard": 4294967295, + "receiverShard": 0, + "data": "QDAwQDU0NDU1MzU0NGU0NjU0MmQ2NDM5NjQzNTM0NjM=", + "prevTxHash": "736372576974684973737565", + "originalTxHash": "747857697468536343616c6c", + "callType": "2", + "timestamp": 5040, + "hasOperations": true, + "type": "unsigned", + "status": "success", + "operation": "transfer", + "hasLogs": true +} \ No newline at end of file diff --git a/integrationtests/testdata/scCallWithIssueEsdt/scr-with-callback-executed-on-source.json b/integrationtests/testdata/scCallWithIssueEsdt/scr-with-callback-executed-on-source.json new file mode 100644 index 00000000..219cca43 --- /dev/null +++ b/integrationtests/testdata/scCallWithIssueEsdt/scr-with-callback-executed-on-source.json @@ -0,0 +1,20 @@ +{ + "miniBlockHash": "49345577b344acd366143ad353904657307dc21f57c3817bf164975de3858459", + "nonce": 0, + "gasLimit": 0, + "gasPrice": 0, + "value": "0", + "valueNum": 0, + "sender": "erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u", + "receiver": "erd1qqqqqqqqqqqqqpgqahumqen35dr9k4rmcnd70mqt5t4mt7ey4nwqwjme9g", + "senderShard": 4294967295, + "receiverShard": 0, + "data": "QDAwQDU0NDU1MzU0NGU0NjU0MmQ2NDM5NjQzNTM0NjM=", + "prevTxHash": "736372576974684973737565", + "originalTxHash": "747857697468536343616c6c", + "callType": "2", + "timestamp": 5040, + "type": "unsigned", + "status": "pending", + "operation": "transfer" +} \ No newline at end of file diff --git a/integrationtests/testdata/scCallWithIssueEsdt/scr-with-issue-executed-on-destination-shard.json b/integrationtests/testdata/scCallWithIssueEsdt/scr-with-issue-executed-on-destination-shard.json new file mode 100644 index 00000000..6c50bad6 --- /dev/null +++ b/integrationtests/testdata/scCallWithIssueEsdt/scr-with-issue-executed-on-destination-shard.json @@ -0,0 +1,21 @@ +{ + "miniBlockHash": "5eb176de1812fdfda088c1764f07ef9f991ba6159de3096143a59185884aed4a", + "nonce": 0, + "gasLimit": 0, + "gasPrice": 0, + "value": "50000000000000000", + "valueNum": 0.05, + "sender": "erd1qqqqqqqqqqqqqpgqahumqen35dr9k4rmcnd70mqt5t4mt7ey4nwqwjme9g", + "receiver": "erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u", + "senderShard": 0, + "receiverShard": 4294967295, + "data": "aXNzdWVOb25GdW5naWJsZUA0ZDc5NTQ2NTczNzQ0ZTY2NzQ2NEA1NDQ1NTM1NDRlNDY1NEA2MzYxNmU0NjcyNjU2NTdhNjVANzQ3Mjc1NjVANjM2MTZlNTc2OTcwNjVANzQ3Mjc1NjVANjM2MTZlNTA2MTc1NzM2NUA3NDcyNzU2NUA2MzYxNmU0MzY4NjE2ZTY3NjU0Zjc3NmU2NTcyQDY2NjE2YzczNjVANjM2MTZlNTU3MDY3NzI2MTY0NjVANjY2MTZjNzM2NUA2MzYxNmU0MTY0NjQ1MzcwNjU2MzY5NjE2YzUyNmY2YzY1NzNANzQ3Mjc1NjVANThmNjM4", + "prevTxHash": "747857697468536343616c6c", + "originalTxHash": "747857697468536343616c6c", + "callType": "1", + "timestamp": 5040, + "type": "unsigned", + "status": "success", + "operation": "transfer", + "function": "issueNonFungible" +} \ No newline at end of file diff --git a/integrationtests/testdata/scCallWithIssueEsdt/scr-with-issue-executed-on-source-shard.json b/integrationtests/testdata/scCallWithIssueEsdt/scr-with-issue-executed-on-source-shard.json new file mode 100644 index 00000000..4bfb598e --- /dev/null +++ b/integrationtests/testdata/scCallWithIssueEsdt/scr-with-issue-executed-on-source-shard.json @@ -0,0 +1,21 @@ +{ + "miniBlockHash": "5eb176de1812fdfda088c1764f07ef9f991ba6159de3096143a59185884aed4a", + "nonce": 0, + "gasLimit": 0, + "gasPrice": 0, + "value": "50000000000000000", + "valueNum": 0.05, + "sender": "erd1qqqqqqqqqqqqqpgqahumqen35dr9k4rmcnd70mqt5t4mt7ey4nwqwjme9g", + "receiver": "erd1qqqqqqqqqqqqqqqpqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzllls8a5w6u", + "senderShard": 0, + "receiverShard": 4294967295, + "data": "aXNzdWVOb25GdW5naWJsZUA0ZDc5NTQ2NTczNzQ0ZTY2NzQ2NEA1NDQ1NTM1NDRlNDY1NEA2MzYxNmU0NjcyNjU2NTdhNjVANzQ3Mjc1NjVANjM2MTZlNTc2OTcwNjVANzQ3Mjc1NjVANjM2MTZlNTA2MTc1NzM2NUA3NDcyNzU2NUA2MzYxNmU0MzY4NjE2ZTY3NjU0Zjc3NmU2NTcyQDY2NjE2YzczNjVANjM2MTZlNTU3MDY3NzI2MTY0NjVANjY2MTZjNzM2NUA2MzYxNmU0MTY0NjQ1MzcwNjU2MzY5NjE2YzUyNmY2YzY1NzNANzQ3Mjc1NjVANThmNjM4", + "prevTxHash": "747857697468536343616c6c", + "originalTxHash": "747857697468536343616c6c", + "callType": "1", + "timestamp": 5040, + "type": "unsigned", + "status": "pending", + "operation": "transfer", + "function": "issueNonFungible" +} \ No newline at end of file diff --git a/integrationtests/testdata/scCallWithIssueEsdt/tx-after-execution-of-callback-on-destination-shard.json b/integrationtests/testdata/scCallWithIssueEsdt/tx-after-execution-of-callback-on-destination-shard.json new file mode 100644 index 00000000..93acfcc8 --- /dev/null +++ b/integrationtests/testdata/scCallWithIssueEsdt/tx-after-execution-of-callback-on-destination-shard.json @@ -0,0 +1,27 @@ +{ + "miniBlockHash": "ace0c639c9687eb82e5d304502668e937cf220610b5a3881ca7e210802e64ae1", + "nonce": 46, + "round": 50, + "value": "50000000000000000", + "valueNum": 0.05, + "receiver": "erd1qqqqqqqqqqqqqpgqahumqen35dr9k4rmcnd70mqt5t4mt7ey4nwqwjme9g", + "sender": "erd148m2sx48mfm8322c2kpfmgj78g5j0x7r6z6y4z8j28qk45a74nwq5pq2ts", + "receiverShard": 0, + "senderShard": 0, + "gasPrice": 1000000000, + "gasLimit": 75000000, + "gasUsed": 75000000, + "fee": "867810000000000", + "feeNum": 8.6781E-4, + "initialPaidFee": "867810000000000", + "data": "aXNzdWVUb2tlbkA0RDc5NTQ2NTczNzQ0RTY2NzQ2NEA1NDQ1NTM1NDRFNDY1NA==", + "signature": "", + "timestamp": 5040, + "status": "fail", + "searchOrder": 0, + "hasScResults": true, + "isScCall": true, + "operation": "transfer", + "function": "issueToken", + "errorEvent": true +} \ No newline at end of file diff --git a/integrationtests/testdata/scCallWithIssueEsdt/tx-after-execution-on-source-shard.json b/integrationtests/testdata/scCallWithIssueEsdt/tx-after-execution-on-source-shard.json new file mode 100644 index 00000000..c5758715 --- /dev/null +++ b/integrationtests/testdata/scCallWithIssueEsdt/tx-after-execution-on-source-shard.json @@ -0,0 +1,26 @@ +{ + "miniBlockHash": "ace0c639c9687eb82e5d304502668e937cf220610b5a3881ca7e210802e64ae1", + "nonce": 46, + "round": 50, + "value": "50000000000000000", + "valueNum": 0.05, + "receiver": "erd1qqqqqqqqqqqqqpgqahumqen35dr9k4rmcnd70mqt5t4mt7ey4nwqwjme9g", + "sender": "erd148m2sx48mfm8322c2kpfmgj78g5j0x7r6z6y4z8j28qk45a74nwq5pq2ts", + "receiverShard": 0, + "senderShard": 0, + "gasPrice": 1000000000, + "gasLimit": 75000000, + "gasUsed": 75000000, + "fee": "867810000000000", + "feeNum": 0.00086781, + "initialPaidFee": "867810000000000", + "data": "aXNzdWVUb2tlbkA0RDc5NTQ2NTczNzQ0RTY2NzQ2NEA1NDQ1NTM1NDRFNDY1NA==", + "signature": "", + "timestamp": 5040, + "status": "success", + "searchOrder": 0, + "hasScResults": true, + "isScCall": true, + "operation": "transfer", + "function": "issueToken" +} diff --git a/integrationtests/testdata/scCallWithIssueEsdt/tx-in-op-index-execution-of-callback-on-destination-shard.json b/integrationtests/testdata/scCallWithIssueEsdt/tx-in-op-index-execution-of-callback-on-destination-shard.json new file mode 100644 index 00000000..0206bc51 --- /dev/null +++ b/integrationtests/testdata/scCallWithIssueEsdt/tx-in-op-index-execution-of-callback-on-destination-shard.json @@ -0,0 +1,28 @@ +{ + "miniBlockHash": "ace0c639c9687eb82e5d304502668e937cf220610b5a3881ca7e210802e64ae1", + "nonce": 46, + "round": 50, + "value": "50000000000000000", + "valueNum": 0.05, + "receiver": "erd1qqqqqqqqqqqqqpgqahumqen35dr9k4rmcnd70mqt5t4mt7ey4nwqwjme9g", + "sender": "erd148m2sx48mfm8322c2kpfmgj78g5j0x7r6z6y4z8j28qk45a74nwq5pq2ts", + "receiverShard": 0, + "senderShard": 0, + "gasPrice": 1000000000, + "gasLimit": 75000000, + "gasUsed": 75000000, + "fee": "867810000000000", + "feeNum": 8.6781E-4, + "initialPaidFee": "867810000000000", + "data": "aXNzdWVUb2tlbkA0RDc5NTQ2NTczNzQ0RTY2NzQ2NEA1NDQ1NTM1NDRFNDY1NA==", + "signature": "", + "timestamp": 5040, + "status": "fail", + "searchOrder": 0, + "hasScResults": true, + "isScCall": true, + "operation": "transfer", + "function": "issueToken", + "errorEvent": true, + "type": "normal" +} \ No newline at end of file From 8e08b95f25805dd4783fe5b935921f3c5dced765 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 4 Jul 2023 13:50:47 +0300 Subject: [PATCH 103/189] empty lines --- .../scr-with-callback-executed-on-destination-shard.json | 2 +- .../scr-with-callback-executed-on-source.json | 2 +- .../scr-with-issue-executed-on-destination-shard.json | 2 +- .../scr-with-issue-executed-on-source-shard.json | 2 +- .../tx-after-execution-of-callback-on-destination-shard.json | 2 +- .../scCallWithIssueEsdt/tx-after-execution-on-source-shard.json | 1 + ...-in-op-index-execution-of-callback-on-destination-shard.json | 2 +- 7 files changed, 7 insertions(+), 6 deletions(-) diff --git a/integrationtests/testdata/scCallWithIssueEsdt/scr-with-callback-executed-on-destination-shard.json b/integrationtests/testdata/scCallWithIssueEsdt/scr-with-callback-executed-on-destination-shard.json index 3a070a6f..d0896ee1 100644 --- a/integrationtests/testdata/scCallWithIssueEsdt/scr-with-callback-executed-on-destination-shard.json +++ b/integrationtests/testdata/scCallWithIssueEsdt/scr-with-callback-executed-on-destination-shard.json @@ -19,4 +19,4 @@ "status": "success", "operation": "transfer", "hasLogs": true -} \ No newline at end of file +} diff --git a/integrationtests/testdata/scCallWithIssueEsdt/scr-with-callback-executed-on-source.json b/integrationtests/testdata/scCallWithIssueEsdt/scr-with-callback-executed-on-source.json index 219cca43..5bc7cbb2 100644 --- a/integrationtests/testdata/scCallWithIssueEsdt/scr-with-callback-executed-on-source.json +++ b/integrationtests/testdata/scCallWithIssueEsdt/scr-with-callback-executed-on-source.json @@ -17,4 +17,4 @@ "type": "unsigned", "status": "pending", "operation": "transfer" -} \ No newline at end of file +} diff --git a/integrationtests/testdata/scCallWithIssueEsdt/scr-with-issue-executed-on-destination-shard.json b/integrationtests/testdata/scCallWithIssueEsdt/scr-with-issue-executed-on-destination-shard.json index 6c50bad6..448dfb9e 100644 --- a/integrationtests/testdata/scCallWithIssueEsdt/scr-with-issue-executed-on-destination-shard.json +++ b/integrationtests/testdata/scCallWithIssueEsdt/scr-with-issue-executed-on-destination-shard.json @@ -18,4 +18,4 @@ "status": "success", "operation": "transfer", "function": "issueNonFungible" -} \ No newline at end of file +} diff --git a/integrationtests/testdata/scCallWithIssueEsdt/scr-with-issue-executed-on-source-shard.json b/integrationtests/testdata/scCallWithIssueEsdt/scr-with-issue-executed-on-source-shard.json index 4bfb598e..3703f353 100644 --- a/integrationtests/testdata/scCallWithIssueEsdt/scr-with-issue-executed-on-source-shard.json +++ b/integrationtests/testdata/scCallWithIssueEsdt/scr-with-issue-executed-on-source-shard.json @@ -18,4 +18,4 @@ "status": "pending", "operation": "transfer", "function": "issueNonFungible" -} \ No newline at end of file +} diff --git a/integrationtests/testdata/scCallWithIssueEsdt/tx-after-execution-of-callback-on-destination-shard.json b/integrationtests/testdata/scCallWithIssueEsdt/tx-after-execution-of-callback-on-destination-shard.json index 93acfcc8..ce08d35c 100644 --- a/integrationtests/testdata/scCallWithIssueEsdt/tx-after-execution-of-callback-on-destination-shard.json +++ b/integrationtests/testdata/scCallWithIssueEsdt/tx-after-execution-of-callback-on-destination-shard.json @@ -24,4 +24,4 @@ "operation": "transfer", "function": "issueToken", "errorEvent": true -} \ No newline at end of file +} diff --git a/integrationtests/testdata/scCallWithIssueEsdt/tx-after-execution-on-source-shard.json b/integrationtests/testdata/scCallWithIssueEsdt/tx-after-execution-on-source-shard.json index c5758715..6ea6fd40 100644 --- a/integrationtests/testdata/scCallWithIssueEsdt/tx-after-execution-on-source-shard.json +++ b/integrationtests/testdata/scCallWithIssueEsdt/tx-after-execution-on-source-shard.json @@ -24,3 +24,4 @@ "operation": "transfer", "function": "issueToken" } + diff --git a/integrationtests/testdata/scCallWithIssueEsdt/tx-in-op-index-execution-of-callback-on-destination-shard.json b/integrationtests/testdata/scCallWithIssueEsdt/tx-in-op-index-execution-of-callback-on-destination-shard.json index 0206bc51..305833e8 100644 --- a/integrationtests/testdata/scCallWithIssueEsdt/tx-in-op-index-execution-of-callback-on-destination-shard.json +++ b/integrationtests/testdata/scCallWithIssueEsdt/tx-in-op-index-execution-of-callback-on-destination-shard.json @@ -25,4 +25,4 @@ "function": "issueToken", "errorEvent": true, "type": "normal" -} \ No newline at end of file +} From 2af86f47058a54a9f1875dd9e094ed82f504afc7 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 6 Jul 2023 15:19:50 +0300 Subject: [PATCH 104/189] fixes after review --- data/logs.go | 10 ++------- go.mod | 2 +- go.sum | 2 ++ mock/dbTransactionsHandlerStub.go | 2 +- process/dataindexer/interface.go | 2 +- .../elasticproc/accounts/accountsProcessor.go | 2 +- .../elasticproc/converters/balanceToFloat.go | 6 ++--- .../converters/balanceToFloat_test.go | 14 ++++++------ process/elasticproc/elasticProcessor.go | 4 ++-- process/elasticproc/interface.go | 2 +- .../logsevents/informativeLogsProcessor.go | 4 ++-- .../informativeLogsProcessor_test.go | 11 +++++----- process/elasticproc/logsevents/interface.go | 5 +++-- process/elasticproc/logsevents/logsData.go | 2 +- process/elasticproc/logsevents/statusInfo.go | 14 ++++++------ .../elasticproc/logsevents/statusInfo_test.go | 22 +++++++++---------- .../transactions/scrsDataToTransactions.go | 2 +- process/elasticproc/transactions/serialize.go | 5 +++-- .../transactions/serialize_test.go | 7 +++--- .../smartContractResultsProcessor.go | 2 +- .../transactions/transactionDBBuilder.go | 6 ++--- 21 files changed, 63 insertions(+), 63 deletions(-) diff --git a/data/logs.go b/data/logs.go index a3baf0d5..96754daa 100644 --- a/data/logs.go +++ b/data/logs.go @@ -3,6 +3,7 @@ package data import ( "time" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokeninfo" ) @@ -24,20 +25,13 @@ type Event struct { Order int `json:"order"` } -// StatusInfo holds the fields for the transaction status -type StatusInfo struct { - CompletedEvent bool `json:"completedEvent"` - ErrorEvent bool `json:"errorEvent"` - Status string `json:"status"` -} - // PreparedLogsResults is the DTO that holds all the results after processing type PreparedLogsResults struct { Tokens TokensHandler TokensSupply TokensHandler ScDeploys map[string]*ScDeployInfo Delegators map[string]*Delegator - TxHashStatusInfo map[string]*StatusInfo + TxHashStatusInfo map[string]*outport.StatusInfo TokensInfo []*TokenInfo NFTsDataUpdates []*NFTDataUpdate TokenRolesAndProperties *tokeninfo.TokenRolesAndProperties diff --git a/go.mod b/go.mod index 0d81c4a4..dff7635c 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/multiversx/mx-chain-communication-go v1.0.3 - github.com/multiversx/mx-chain-core-go v1.2.8-0.20230630085500-96880f1bce4c + github.com/multiversx/mx-chain-core-go v1.2.9-0.20230706101124-049df93d2fac github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.1 github.com/stretchr/testify v1.8.2 diff --git a/go.sum b/go.sum index dc0ea0bb..70ab7877 100644 --- a/go.sum +++ b/go.sum @@ -478,6 +478,8 @@ github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZb github.com/multiversx/mx-chain-core-go v1.2.5/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-core-go v1.2.8-0.20230630085500-96880f1bce4c h1:UfPvZLUYZUsI+ZtxHFroyr3q+CDPgkT0n3eqKNdrJuM= github.com/multiversx/mx-chain-core-go v1.2.8-0.20230630085500-96880f1bce4c/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= +github.com/multiversx/mx-chain-core-go v1.2.9-0.20230706101124-049df93d2fac h1:hhzcq7Iv1+gM9JVTJdMVh2cd8bSEZ/GLygPraLP+i4o= +github.com/multiversx/mx-chain-core-go v1.2.9-0.20230706101124-049df93d2fac/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-crypto-go v1.2.6/go.mod h1:rOj0Rr19HTOYt9YTeym7RKxlHt91NXln3LVKjHKVmA0= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= diff --git a/mock/dbTransactionsHandlerStub.go b/mock/dbTransactionsHandlerStub.go index a810af5d..0a8c2f96 100644 --- a/mock/dbTransactionsHandlerStub.go +++ b/mock/dbTransactionsHandlerStub.go @@ -43,7 +43,7 @@ func (tps *DBTransactionProcessorStub) SerializeReceipts(recs []*data.Receipt, b } // SerializeTransactions - -func (tps *DBTransactionProcessorStub) SerializeTransactions(_ []*data.Transaction, _ map[string]*data.StatusInfo, _ uint32, _ *data.BufferSlice, _ string) error { +func (tps *DBTransactionProcessorStub) SerializeTransactions(_ []*data.Transaction, _ map[string]*outport.StatusInfo, _ uint32, _ *data.BufferSlice, _ string) error { return nil } diff --git a/process/dataindexer/interface.go b/process/dataindexer/interface.go index e72aaf78..72a9e03a 100644 --- a/process/dataindexer/interface.go +++ b/process/dataindexer/interface.go @@ -65,7 +65,7 @@ type Indexer interface { // BalanceConverter defines what a balance converter should be able to do type BalanceConverter interface { ComputeBalanceAsFloat(balance *big.Int) (float64, error) - ComputeESDTBalanceAsFloat(balance *big.Int) (float64, error) + ConvertBigValueToFloat(value *big.Int) (float64, error) ComputeSliceOfStringsAsFloat(values []string) ([]float64, error) IsInterfaceNil() bool } diff --git a/process/elasticproc/accounts/accountsProcessor.go b/process/elasticproc/accounts/accountsProcessor.go index bd0d5d16..4d90b02d 100644 --- a/process/elasticproc/accounts/accountsProcessor.go +++ b/process/elasticproc/accounts/accountsProcessor.go @@ -205,7 +205,7 @@ func (ap *accountsProcessor) PrepareAccountsMapESDT( } tokenIdentifier := converters.ComputeTokenIdentifier(accountESDT.TokenIdentifier, accountESDT.NFTNonce) - balanceNum, err := ap.balanceConverter.ComputeESDTBalanceAsFloat(balance) + balanceNum, err := ap.balanceConverter.ConvertBigValueToFloat(balance) if err != nil { log.Warn("accountsProcessor.PrepareAccountsMapESDT: cannot compute esdt balance as num", "balance", balance, "address", address, "error", err, "token", tokenIdentifier) diff --git a/process/elasticproc/converters/balanceToFloat.go b/process/elasticproc/converters/balanceToFloat.go index de69e450..e3ae8d6c 100644 --- a/process/elasticproc/converters/balanceToFloat.go +++ b/process/elasticproc/converters/balanceToFloat.go @@ -46,8 +46,8 @@ func (bc *balanceConverter) ComputeBalanceAsFloat(balance *big.Int) (float64, er return bc.computeBalanceAsFloat(balance, bc.balancePrecision) } -// ComputeESDTBalanceAsFloat will compute ESDT balance as float -func (bc *balanceConverter) ComputeESDTBalanceAsFloat(balance *big.Int) (float64, error) { +// ConvertBigValueToFloat will convert big value to float +func (bc *balanceConverter) ConvertBigValueToFloat(balance *big.Int) (float64, error) { return bc.computeBalanceAsFloat(balance, bc.balancePrecisionESDT) } @@ -61,7 +61,7 @@ func (bc *balanceConverter) ComputeSliceOfStringsAsFloat(values []string) ([]flo return nil, errCastStringToBigInt } - valueNum, err := bc.ComputeESDTBalanceAsFloat(valueBig) + valueNum, err := bc.ConvertBigValueToFloat(valueBig) if err != nil { return nil, err } diff --git a/process/elasticproc/converters/balanceToFloat_test.go b/process/elasticproc/converters/balanceToFloat_test.go index 8deb3375..96935dde 100644 --- a/process/elasticproc/converters/balanceToFloat_test.go +++ b/process/elasticproc/converters/balanceToFloat_test.go @@ -59,19 +59,19 @@ func TestComputeBalanceToFloat18Decimals(t *testing.T) { ap, _ := NewBalanceConverter(18) require.NotNil(t, ap) - valueNum, _ := ap.ComputeESDTBalanceAsFloat(big.NewInt(1)) + valueNum, _ := ap.ConvertBigValueToFloat(big.NewInt(1)) require.Equal(t, 1e-18, valueNum) - valueNum, _ = ap.ComputeESDTBalanceAsFloat(big.NewInt(10)) + valueNum, _ = ap.ConvertBigValueToFloat(big.NewInt(10)) require.Equal(t, 1e-17, valueNum) - valueNum, _ = ap.ComputeESDTBalanceAsFloat(big.NewInt(100)) + valueNum, _ = ap.ConvertBigValueToFloat(big.NewInt(100)) require.Equal(t, 1e-16, valueNum) - valueNum, _ = ap.ComputeESDTBalanceAsFloat(big.NewInt(1000)) + valueNum, _ = ap.ConvertBigValueToFloat(big.NewInt(1000)) require.Equal(t, 1e-15, valueNum) - valueNum, _ = ap.ComputeESDTBalanceAsFloat(big.NewInt(0)) + valueNum, _ = ap.ConvertBigValueToFloat(big.NewInt(0)) require.Equal(t, float64(0), valueNum) } @@ -83,14 +83,14 @@ func TestComputeBalanceToFloatInf(t *testing.T) { str := "erd1ahmy0yjhjg87n755yv99nzla22zzwfud55sa69gk3anyxyyucq9q2hgxwwerd1ahmy0yjhjg87n755yv99nzla22zzwfud55sa69gk3anyxyyucq9q2hgxwwerd1ahmy0yjhjg87n755yv99nzla22zzwfud55sa69gk3anyxyyucq9q2hgxwwerd1ahmy0yjhjg87n755yv99nzla22zzwfud55sa69gk3anyxyyucq9q2hgxww" bigValue := big.NewInt(0).SetBytes([]byte(str)) - valueNum, err := ap.ComputeESDTBalanceAsFloat(bigValue) + valueNum, err := ap.ConvertBigValueToFloat(bigValue) require.Equal(t, errValueTooBig, err) require.Equal(t, float64(0), valueNum) hexValueStr := "2642378914478872274757363306845016200438452904128227930177150600998175785079732885392662259024767727006622197340762976891962082611710440131598510606436851189901116516523843401702254087190199876126823217692111058487892984414016231313689031989" decoded, _ := hex.DecodeString(hexValueStr) bigValue = big.NewInt(0).SetBytes(decoded) - valueNum, err = ap.ComputeESDTBalanceAsFloat(bigValue) + valueNum, err = ap.ConvertBigValueToFloat(bigValue) require.Equal(t, errValueTooBig, err) require.Equal(t, float64(0), valueNum) } diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index d70fff04..efcf13a5 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -539,7 +539,7 @@ func (ei *elasticProcessor) indexScDeploys(deployData map[string]*data.ScDeployI return ei.logsAndEventsProc.SerializeSCDeploys(deployData, buffSlice, elasticIndexer.SCDeploysIndex) } -func (ei *elasticProcessor) indexTransactions(txs []*data.Transaction, txHashStatusInfo map[string]*data.StatusInfo, header coreData.HeaderHandler, bytesBuff *data.BufferSlice) error { +func (ei *elasticProcessor) indexTransactions(txs []*data.Transaction, txHashStatusInfo map[string]*outport.StatusInfo, header coreData.HeaderHandler, bytesBuff *data.BufferSlice) error { if !ei.isIndexEnabled(elasticIndexer.TransactionsIndex) { return nil } @@ -549,7 +549,7 @@ func (ei *elasticProcessor) indexTransactions(txs []*data.Transaction, txHashSta func (ei *elasticProcessor) prepareAndIndexOperations( txs []*data.Transaction, - txHashStatusInfo map[string]*data.StatusInfo, + txHashStatusInfo map[string]*outport.StatusInfo, header coreData.HeaderHandler, scrs []*data.ScResult, buffSlice *data.BufferSlice, diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index f7fcef17..be7e9166 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -66,7 +66,7 @@ type DBTransactionsHandler interface { GetHexEncodedHashesForRemove(header coreData.HeaderHandler, body *block.Body) ([]string, []string) SerializeReceipts(receipts []*data.Receipt, buffSlice *data.BufferSlice, index string) error - SerializeTransactions(transactions []*data.Transaction, txHashStatusInfo map[string]*data.StatusInfo, selfShardID uint32, buffSlice *data.BufferSlice, index string) error + SerializeTransactions(transactions []*data.Transaction, txHashStatusInfo map[string]*outport.StatusInfo, selfShardID uint32, buffSlice *data.BufferSlice, index string) error SerializeTransactionsFeeData(txHashRefund map[string]*data.FeeData, buffSlice *data.BufferSlice, index string) error SerializeScResults(scResults []*data.ScResult, buffSlice *data.BufferSlice, index string) error } diff --git a/process/elasticproc/logsevents/informativeLogsProcessor.go b/process/elasticproc/logsevents/informativeLogsProcessor.go index d9725dc2..3706f708 100644 --- a/process/elasticproc/logsevents/informativeLogsProcessor.go +++ b/process/elasticproc/logsevents/informativeLogsProcessor.go @@ -2,8 +2,8 @@ package logsevents import ( "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/transaction" - "github.com/multiversx/mx-chain-es-indexer-go/data" ) type informativeLogsProcessor struct { @@ -62,7 +62,7 @@ func processEventNoTx(args *argsProcessEvent) argOutputProcessEvent { } } - record := &data.StatusInfo{} + record := &outport.StatusInfo{} switch string(args.event.GetIdentifier()) { case core.CompletedTxEventIdentifier: { diff --git a/process/elasticproc/logsevents/informativeLogsProcessor_test.go b/process/elasticproc/logsevents/informativeLogsProcessor_test.go index 240732e6..649bd96f 100644 --- a/process/elasticproc/logsevents/informativeLogsProcessor_test.go +++ b/process/elasticproc/logsevents/informativeLogsProcessor_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/stretchr/testify/require" @@ -142,7 +143,7 @@ func TestInformativeLogsProcessorLogsGeneratedByScrsSignalError(t *testing.T) { Identifier: []byte(core.SignalErrorOperation), } - txStatusProc := newTxHashStatusInfo() + txStatusProc := newTxHashStatusInfoProcessor() args := &argsProcessEvent{ timestamp: 1234, event: event, @@ -156,7 +157,7 @@ func TestInformativeLogsProcessorLogsGeneratedByScrsSignalError(t *testing.T) { res := informativeLogsProc.processEvent(args) require.True(t, res.processed) - require.Equal(t, &data.StatusInfo{ + require.Equal(t, &outport.StatusInfo{ Status: transaction.TxStatusFail.String(), ErrorEvent: true, }, txStatusProc.getAllRecords()[txHash]) @@ -178,7 +179,7 @@ func TestInformativeLogsProcessorLogsGeneratedByScrsCompletedEvent(t *testing.T) Identifier: []byte(core.CompletedTxEventIdentifier), } - txStatusProc := newTxHashStatusInfo() + txStatusProc := newTxHashStatusInfoProcessor() args := &argsProcessEvent{ timestamp: 1234, event: event, @@ -192,7 +193,7 @@ func TestInformativeLogsProcessorLogsGeneratedByScrsCompletedEvent(t *testing.T) res := informativeLogsProc.processEvent(args) require.True(t, res.processed) - require.Equal(t, &data.StatusInfo{ + require.Equal(t, &outport.StatusInfo{ CompletedEvent: true, }, txStatusProc.getAllRecords()[txHash]) } @@ -207,7 +208,7 @@ func TestInformativeLogsProcessorLogsGeneratedByScrNotFoundInMap(t *testing.T) { Identifier: []byte(core.CompletedTxEventIdentifier), } - txStatusProc := newTxHashStatusInfo() + txStatusProc := newTxHashStatusInfoProcessor() args := &argsProcessEvent{ timestamp: 1234, event: event, diff --git a/process/elasticproc/logsevents/interface.go b/process/elasticproc/logsevents/interface.go index b3d4d9ba..e65f5271 100644 --- a/process/elasticproc/logsevents/interface.go +++ b/process/elasticproc/logsevents/interface.go @@ -2,6 +2,7 @@ package logsevents import ( coreData "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokeninfo" ) @@ -34,6 +35,6 @@ type eventsProcessor interface { } type txHashStatusInfoHandler interface { - addRecord(hash string, statusInfo *data.StatusInfo) - getAllRecords() map[string]*data.StatusInfo + addRecord(hash string, statusInfo *outport.StatusInfo) + getAllRecords() map[string]*outport.StatusInfo } diff --git a/process/elasticproc/logsevents/logsData.go b/process/elasticproc/logsevents/logsData.go index 2d0aab4e..ee12e55c 100644 --- a/process/elasticproc/logsevents/logsData.go +++ b/process/elasticproc/logsevents/logsData.go @@ -37,7 +37,7 @@ func newLogsData( ld.delegators = make(map[string]*data.Delegator) ld.nftsDataUpdates = make([]*data.NFTDataUpdate, 0) ld.tokenRolesAndProperties = tokeninfo.NewTokenRolesAndProperties() - ld.txHashStatusInfoProc = newTxHashStatusInfo() + ld.txHashStatusInfoProc = newTxHashStatusInfoProcessor() return ld } diff --git a/process/elasticproc/logsevents/statusInfo.go b/process/elasticproc/logsevents/statusInfo.go index 2ad2a5df..c08d04b5 100644 --- a/process/elasticproc/logsevents/statusInfo.go +++ b/process/elasticproc/logsevents/statusInfo.go @@ -1,23 +1,23 @@ package logsevents import ( + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/transaction" - "github.com/multiversx/mx-chain-es-indexer-go/data" ) type txHashStatusInfoProc struct { - hashStatusInfo map[string]*data.StatusInfo + hashStatusInfo map[string]*outport.StatusInfo } -// NewTxHashStatusInfo will create a new instance of TxHashStatusInfo -func newTxHashStatusInfo() *txHashStatusInfoProc { +// newTxHashStatusInfoProcessor will create a new instance of txHashStatusInfoProc +func newTxHashStatusInfoProcessor() *txHashStatusInfoProc { return &txHashStatusInfoProc{ - hashStatusInfo: make(map[string]*data.StatusInfo), + hashStatusInfo: make(map[string]*outport.StatusInfo), } } // AddRecord will add a new record for the given hash -func (ths *txHashStatusInfoProc) addRecord(hash string, statusInfo *data.StatusInfo) { +func (ths *txHashStatusInfoProc) addRecord(hash string, statusInfo *outport.StatusInfo) { statusInfoFromMap, found := ths.hashStatusInfo[hash] if !found { ths.hashStatusInfo[hash] = statusInfo @@ -33,6 +33,6 @@ func (ths *txHashStatusInfoProc) addRecord(hash string, statusInfo *data.StatusI } // GetAllRecords will return all the records -func (ths *txHashStatusInfoProc) getAllRecords() map[string]*data.StatusInfo { +func (ths *txHashStatusInfoProc) getAllRecords() map[string]*outport.StatusInfo { return ths.hashStatusInfo } diff --git a/process/elasticproc/logsevents/statusInfo_test.go b/process/elasticproc/logsevents/statusInfo_test.go index c4545cd2..280c8570 100644 --- a/process/elasticproc/logsevents/statusInfo_test.go +++ b/process/elasticproc/logsevents/statusInfo_test.go @@ -3,45 +3,45 @@ package logsevents import ( "testing" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/transaction" - "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/stretchr/testify/require" ) func TestStatusInfoAddRecord(t *testing.T) { t.Parallel() - statusInfOProc := newTxHashStatusInfo() + statusInfoProc := newTxHashStatusInfoProcessor() txHash := "txHash1" - statusInfOProc.addRecord(txHash, &data.StatusInfo{ + statusInfoProc.addRecord(txHash, &outport.StatusInfo{ CompletedEvent: true, ErrorEvent: false, Status: transaction.TxStatusSuccess.String(), }) - require.Equal(t, &data.StatusInfo{ + require.Equal(t, &outport.StatusInfo{ CompletedEvent: true, Status: "success", - }, statusInfOProc.getAllRecords()[txHash]) + }, statusInfoProc.getAllRecords()[txHash]) - statusInfOProc.addRecord(txHash, &data.StatusInfo{ + statusInfoProc.addRecord(txHash, &outport.StatusInfo{ ErrorEvent: true, Status: transaction.TxStatusFail.String(), }) - require.Equal(t, &data.StatusInfo{ + require.Equal(t, &outport.StatusInfo{ CompletedEvent: true, ErrorEvent: true, Status: "fail", - }, statusInfOProc.getAllRecords()[txHash]) + }, statusInfoProc.getAllRecords()[txHash]) - statusInfOProc.addRecord(txHash, &data.StatusInfo{ + statusInfoProc.addRecord(txHash, &outport.StatusInfo{ ErrorEvent: false, CompletedEvent: false, Status: transaction.TxStatusSuccess.String(), }) - require.Equal(t, &data.StatusInfo{ + require.Equal(t, &outport.StatusInfo{ CompletedEvent: true, ErrorEvent: true, Status: "fail", - }, statusInfOProc.getAllRecords()[txHash]) + }, statusInfoProc.getAllRecords()[txHash]) } diff --git a/process/elasticproc/transactions/scrsDataToTransactions.go b/process/elasticproc/transactions/scrsDataToTransactions.go index 08383872..f18aa741 100644 --- a/process/elasticproc/transactions/scrsDataToTransactions.go +++ b/process/elasticproc/transactions/scrsDataToTransactions.go @@ -62,7 +62,7 @@ func (st *scrsDataToTransactions) processSCRsWithoutTx(scrs []*data.ScResult) ma var err error initialTxFeeBig, ok := big.NewInt(0).SetString(scr.InitialTxFee, 10) if ok { - feeNum, err = st.balanceConverter.ComputeESDTBalanceAsFloat(initialTxFeeBig) + feeNum, err = st.balanceConverter.ConvertBigValueToFloat(initialTxFeeBig) } if err != nil { log.Warn("scrsDataToTransactions.processSCRsWithoutTx: cannot compute fee as num", diff --git a/process/elasticproc/transactions/serialize.go b/process/elasticproc/transactions/serialize.go index 77746e64..2cab3fc9 100644 --- a/process/elasticproc/transactions/serialize.go +++ b/process/elasticproc/transactions/serialize.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" ) @@ -80,7 +81,7 @@ func (tdp *txsDatabaseProcessor) SerializeTransactionsFeeData(txHashRefund map[s // SerializeTransactions will serialize the transactions in a way that Elasticsearch expects a bulk request func (tdp *txsDatabaseProcessor) SerializeTransactions( transactions []*data.Transaction, - txHashStatusInfo map[string]*data.StatusInfo, + txHashStatusInfo map[string]*outport.StatusInfo, selfShardID uint32, buffSlice *data.BufferSlice, index string, @@ -105,7 +106,7 @@ func (tdp *txsDatabaseProcessor) SerializeTransactions( return nil } -func serializeTxHashStatus(buffSlice *data.BufferSlice, txHashStatusInfo map[string]*data.StatusInfo, index string) error { +func serializeTxHashStatus(buffSlice *data.BufferSlice, txHashStatusInfo map[string]*outport.StatusInfo, index string) error { for txHash, statusInfo := range txHashStatusInfo { metaData := []byte(fmt.Sprintf(`{"update":{ "_index":"%s","_id":"%s"}}%s`, index, txHash, "\n")) diff --git a/process/elasticproc/transactions/serialize_test.go b/process/elasticproc/transactions/serialize_test.go index bccad6ca..1565ee2d 100644 --- a/process/elasticproc/transactions/serialize_test.go +++ b/process/elasticproc/transactions/serialize_test.go @@ -3,6 +3,7 @@ package transactions import ( "testing" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/stretchr/testify/require" ) @@ -81,7 +82,7 @@ func TestSerializeTransactionsIntraShardTx(t *testing.T) { err := (&txsDatabaseProcessor{}).SerializeTransactions([]*data.Transaction{{ Hash: "txHash", SmartContractResults: []*data.ScResult{{}}, - }}, map[string]*data.StatusInfo{}, 0, buffSlice, "transactions") + }}, map[string]*outport.StatusInfo{}, 0, buffSlice, "transactions") require.Nil(t, err) expectedBuff := `{ "index" : { "_index":"transactions", "_id" : "txHash" } } @@ -100,7 +101,7 @@ func TestSerializeTransactionCrossShardTxSource(t *testing.T) { ReceiverShard: 1, SmartContractResults: []*data.ScResult{{}}, Version: 1, - }}, map[string]*data.StatusInfo{}, 0, buffSlice, "transactions") + }}, map[string]*outport.StatusInfo{}, 0, buffSlice, "transactions") require.Nil(t, err) expectedBuff := `{"update":{ "_index":"transactions", "_id":"txHash"}} @@ -119,7 +120,7 @@ func TestSerializeTransactionsCrossShardTxDestination(t *testing.T) { ReceiverShard: 0, SmartContractResults: []*data.ScResult{{}}, Version: 1, - }}, map[string]*data.StatusInfo{}, 0, buffSlice, "transactions") + }}, map[string]*outport.StatusInfo{}, 0, buffSlice, "transactions") require.Nil(t, err) expectedBuff := `{ "index" : { "_index":"transactions", "_id" : "txHash" } } diff --git a/process/elasticproc/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go index 3f811e5e..3d91deb1 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -137,7 +137,7 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( receiverAddr := proc.pubKeyConverter.SilentEncode(scr.RcvAddr, log) receiversAddr, _ := proc.pubKeyConverter.EncodeSlice(res.Receivers) - valueNum, err := proc.balanceConverter.ComputeESDTBalanceAsFloat(scr.Value) + valueNum, err := proc.balanceConverter.ConvertBigValueToFloat(scr.Value) if err != nil { log.Warn("smartContractResultsProcessor.prepareSmartContractResult cannot compute scr value as num", "value", scr.Value, "hash", scrHashHex, "error", err) diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index 7a4a5232..30c3fd92 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -58,14 +58,14 @@ func (dtb *dbTransactionBuilder) prepareTransaction( receiverShardID = sharding.ComputeShardID(tx.RcvAddr, numOfShards) } - valueNum, err := dtb.balanceConverter.ComputeESDTBalanceAsFloat(tx.Value) + valueNum, err := dtb.balanceConverter.ConvertBigValueToFloat(tx.Value) if err != nil { log.Warn("dbTransactionBuilder.prepareTransaction: cannot compute value as num", "value", tx.Value, "hash", txHash, "error", err) } feeInfo := getFeeInfo(txInfo) - feeNum, err := dtb.balanceConverter.ComputeESDTBalanceAsFloat(feeInfo.Fee) + feeNum, err := dtb.balanceConverter.ConvertBigValueToFloat(feeInfo.Fee) if err != nil { log.Warn("dbTransactionBuilder.prepareTransaction: cannot compute transaction fee as num", "fee", feeInfo.Fee, "hash", txHash, "error", err) @@ -133,7 +133,7 @@ func (dtb *dbTransactionBuilder) prepareRewardTransaction( header coreData.HeaderHandler, txStatus string, ) *data.Transaction { - valueNum, err := dtb.balanceConverter.ComputeESDTBalanceAsFloat(rTx.Value) + valueNum, err := dtb.balanceConverter.ConvertBigValueToFloat(rTx.Value) if err != nil { log.Warn("dbTransactionBuilder.prepareRewardTransaction cannot compute value as num", "value", rTx.Value, "hash", txHash, "error", err) From 8f2155ac9f43b05edd96f59e6b22c2a7c836acbc Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 6 Jul 2023 15:20:04 +0300 Subject: [PATCH 105/189] go mod tidy --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 70ab7877..207e277f 100644 --- a/go.sum +++ b/go.sum @@ -476,8 +476,6 @@ github.com/multiversx/mx-chain-communication-go v1.0.3/go.mod h1:7oTI77XfWmRWwVE github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.5/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= -github.com/multiversx/mx-chain-core-go v1.2.8-0.20230630085500-96880f1bce4c h1:UfPvZLUYZUsI+ZtxHFroyr3q+CDPgkT0n3eqKNdrJuM= -github.com/multiversx/mx-chain-core-go v1.2.8-0.20230630085500-96880f1bce4c/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-core-go v1.2.9-0.20230706101124-049df93d2fac h1:hhzcq7Iv1+gM9JVTJdMVh2cd8bSEZ/GLygPraLP+i4o= github.com/multiversx/mx-chain-core-go v1.2.9-0.20230706101124-049df93d2fac/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-crypto-go v1.2.6/go.mod h1:rOj0Rr19HTOYt9YTeym7RKxlHt91NXln3LVKjHKVmA0= From 8b1bd58d90c99e6639b2e1e0f82ce01c07017c90 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 7 Jul 2023 11:28:46 +0300 Subject: [PATCH 106/189] sc deploy integration test --- integrationtests/scDeploy_test.go | 106 ++++++++++++++++++ .../testdata/scDeploy/deploy.json | 6 + .../testdata/scDeploy/tx-sc-deploy.json | 24 ++++ integrationtests/utils.go | 2 +- 4 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 integrationtests/scDeploy_test.go create mode 100644 integrationtests/testdata/scDeploy/deploy.json create mode 100644 integrationtests/testdata/scDeploy/tx-sc-deploy.json diff --git a/integrationtests/scDeploy_test.go b/integrationtests/scDeploy_test.go new file mode 100644 index 00000000..2ed3f944 --- /dev/null +++ b/integrationtests/scDeploy_test.go @@ -0,0 +1,106 @@ +//go:build integrationtests + +package integrationtests + +import ( + "encoding/hex" + "math/big" + "testing" + + "github.com/multiversx/mx-chain-core-go/core" + dataBlock "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-core-go/data/transaction" + indexerData "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/stretchr/testify/require" +) + +func TestTransactionWithSCDeploy(t *testing.T) { + setLogLevelDebug() + + esClient, err := createESClient(esURL) + require.Nil(t, err) + + esProc, err := CreateElasticProcessor(esClient) + require.Nil(t, err) + + txHash := []byte("scDeployHash") + header := &dataBlock.Header{ + Round: 50, + TimeStamp: 5040, + ShardID: 2, + } + body := &dataBlock.Body{ + MiniBlocks: dataBlock.MiniBlockSlice{ + { + Type: dataBlock.TxBlock, + SenderShardID: 2, + ReceiverShardID: 2, + TxHashes: [][]byte{txHash}, + }, + }, + } + sndAddress := "erd12m3x8jp6dl027pj5f2nw6ght2cyhhjfrs86cdwsa8xn83r375qfqrwpdx0" + tx := &transaction.Transaction{ + Nonce: 1, + SndAddr: decodeAddress(sndAddress), + RcvAddr: decodeAddress("erd1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq6gq4hu"), + GasLimit: 1000000000, + GasPrice: 2000000, + Data: []byte("0061736d01000000010d036000006000017f60027f7f00023e0303656e760f6765744e756d417267756d656e7473000103656e760b7369676e616c4572726f72000203656e760e636865636b4e6f5061796d656e74000003030200000503010003060f027f00419980080b7f0041a080080b073705066d656d6f7279020004696e697400030863616c6c4261636b00040a5f5f646174615f656e6403000b5f5f686561705f6261736503010a180212001002100004404180800841191001000b0b0300010b0b210100418080080b1977726f6e67206e756d626572206f6620617267756d656e7473@0500@0502"), + Value: big.NewInt(0), + } + + txInfo := &outport.TxInfo{ + Transaction: tx, + FeeInfo: &outport.FeeInfo{ + GasUsed: 1130820, + Fee: big.NewInt(764698200000000), + InitialPaidFee: big.NewInt(773390000000000), + }, + ExecutionOrder: 0, + } + + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + hex.EncodeToString(txHash): txInfo, + }, + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h1")), + Log: &transaction.Log{ + Address: decodeAddress(sndAddress), + Events: []*transaction.Event{ + { + Address: decodeAddress(sndAddress), + Identifier: []byte(core.SCDeployIdentifier), + Topics: [][]byte{decodeAddress("erd1qqqqqqqqqqqqqpgq4t2tqxpst9a6qttpak8cz8wvz6a0nses5qfqel6rhy"), decodeAddress("erd12m3x8jp6dl027pj5f2nw6ght2cyhhjfrs86cdwsa8xn83r375qfqrwpdx0")}, + }, + nil, + }, + }, + }, + }, + } + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) + require.Nil(t, err) + + ids := []string{hex.EncodeToString(txHash)} + genericResponse := &GenericResponse{} + err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse) + require.Nil(t, err) + + require.JSONEq(t, + readExpectedResult("./testdata/scDeploy/tx-sc-deploy.json"), + string(genericResponse.Docs[0].Source), + ) + + ids = []string{"erd1qqqqqqqqqqqqqpgq4t2tqxpst9a6qttpak8cz8wvz6a0nses5qfqel6rhy"} + err = esClient.DoMultiGet(ids, indexerData.SCDeploysIndex, true, genericResponse) + require.Nil(t, err) + + require.JSONEq(t, + readExpectedResult("./testdata/scDeploy/deploy.json"), + string(genericResponse.Docs[0].Source), + ) +} diff --git a/integrationtests/testdata/scDeploy/deploy.json b/integrationtests/testdata/scDeploy/deploy.json new file mode 100644 index 00000000..6cf1d915 --- /dev/null +++ b/integrationtests/testdata/scDeploy/deploy.json @@ -0,0 +1,6 @@ +{ + "deployTxHash": "6831", + "deployer": "erd12m3x8jp6dl027pj5f2nw6ght2cyhhjfrs86cdwsa8xn83r375qfqrwpdx0", + "timestamp": 5040, + "upgrades": [] +} diff --git a/integrationtests/testdata/scDeploy/tx-sc-deploy.json b/integrationtests/testdata/scDeploy/tx-sc-deploy.json new file mode 100644 index 00000000..03a79f1b --- /dev/null +++ b/integrationtests/testdata/scDeploy/tx-sc-deploy.json @@ -0,0 +1,24 @@ +{ + "miniBlockHash": "1703039e791823d0e722a0d3df76c87786fb5074d2b333c98a870335b418a97d", + "nonce": 1, + "round": 50, + "value": "0", + "valueNum": 0, + "receiver": "erd1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq6gq4hu", + "sender": "erd12m3x8jp6dl027pj5f2nw6ght2cyhhjfrs86cdwsa8xn83r375qfqrwpdx0", + "receiverShard": 2, + "senderShard": 2, + "gasPrice": 2000000, + "gasLimit": 1000000000, + "gasUsed": 1130820, + "fee": "764698200000000", + "feeNum": 0.0007646982, + "initialPaidFee": "773390000000000", + "data": "MDA2MTczNmQwMTAwMDAwMDAxMGQwMzYwMDAwMDYwMDAwMTdmNjAwMjdmN2YwMDAyM2UwMzAzNjU2ZTc2MGY2NzY1NzQ0ZTc1NmQ0MTcyNjc3NTZkNjU2ZTc0NzMwMDAxMDM2NTZlNzYwYjczNjk2NzZlNjE2YzQ1NzI3MjZmNzIwMDAyMDM2NTZlNzYwZTYzNjg2NTYzNmI0ZTZmNTA2MTc5NmQ2NTZlNzQwMDAwMDMwMzAyMDAwMDA1MDMwMTAwMDMwNjBmMDI3ZjAwNDE5OTgwMDgwYjdmMDA0MWEwODAwODBiMDczNzA1MDY2ZDY1NmQ2ZjcyNzkwMjAwMDQ2OTZlNjk3NDAwMDMwODYzNjE2YzZjNDI2MTYzNmIwMDA0MGE1ZjVmNjQ2MTc0NjE1ZjY1NmU2NDAzMDAwYjVmNWY2ODY1NjE3MDVmNjI2MTczNjUwMzAxMGExODAyMTIwMDEwMDIxMDAwMDQ0MDQxODA4MDA4NDExOTEwMDEwMDBiMGIwMzAwMDEwYjBiMjEwMTAwNDE4MDgwMDgwYjE5Nzc3MjZmNmU2NzIwNmU3NTZkNjI2NTcyMjA2ZjY2MjA2MTcyNjc3NTZkNjU2ZTc0NzNAMDUwMEAwNTAy", + "signature": "", + "timestamp": 5040, + "status": "success", + "searchOrder": 0, + "isScCall": true, + "operation": "scDeploy" +} diff --git a/integrationtests/utils.go b/integrationtests/utils.go index d8d8ddad..d76b82c7 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -58,7 +58,7 @@ func CreateElasticProcessor( DBClient: esClient, EnabledIndexes: []string{dataindexer.TransactionsIndex, dataindexer.LogsIndex, dataindexer.AccountsESDTIndex, dataindexer.ScResultsIndex, dataindexer.ReceiptsIndex, dataindexer.BlockIndex, dataindexer.AccountsIndex, dataindexer.TokensIndex, dataindexer.TagsIndex, - dataindexer.OperationsIndex, dataindexer.DelegatorsIndex, dataindexer.ESDTsIndex}, + dataindexer.OperationsIndex, dataindexer.DelegatorsIndex, dataindexer.ESDTsIndex, dataindexer.SCDeploysIndex}, Denomination: 18, } From 37c9bc953e7793dcf29abecb458b38deb79640c6 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 7 Jul 2023 12:46:34 +0300 Subject: [PATCH 107/189] fix comments --- process/elasticproc/logsevents/statusInfo.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/process/elasticproc/logsevents/statusInfo.go b/process/elasticproc/logsevents/statusInfo.go index c08d04b5..80d89316 100644 --- a/process/elasticproc/logsevents/statusInfo.go +++ b/process/elasticproc/logsevents/statusInfo.go @@ -16,7 +16,7 @@ func newTxHashStatusInfoProcessor() *txHashStatusInfoProc { } } -// AddRecord will add a new record for the given hash +// addRecord will add a new record for the given hash func (ths *txHashStatusInfoProc) addRecord(hash string, statusInfo *outport.StatusInfo) { statusInfoFromMap, found := ths.hashStatusInfo[hash] if !found { @@ -32,7 +32,7 @@ func (ths *txHashStatusInfoProc) addRecord(hash string, statusInfo *outport.Stat statusInfoFromMap.CompletedEvent = statusInfoFromMap.CompletedEvent || statusInfo.CompletedEvent } -// GetAllRecords will return all the records +// getAllRecords will return all the records func (ths *txHashStatusInfoProc) getAllRecords() map[string]*outport.StatusInfo { return ths.hashStatusInfo } From 1d269c578f42173d119e34fe018d529dfbabb228 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 11 Jul 2023 12:06:41 +0300 Subject: [PATCH 108/189] proper tag --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index dff7635c..b89e9e5e 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/multiversx/mx-chain-communication-go v1.0.3 - github.com/multiversx/mx-chain-core-go v1.2.9-0.20230706101124-049df93d2fac + github.com/multiversx/mx-chain-core-go v1.2.9 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.1 github.com/stretchr/testify v1.8.2 diff --git a/go.sum b/go.sum index 207e277f..d98700c6 100644 --- a/go.sum +++ b/go.sum @@ -476,8 +476,8 @@ github.com/multiversx/mx-chain-communication-go v1.0.3/go.mod h1:7oTI77XfWmRWwVE github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.5/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= -github.com/multiversx/mx-chain-core-go v1.2.9-0.20230706101124-049df93d2fac h1:hhzcq7Iv1+gM9JVTJdMVh2cd8bSEZ/GLygPraLP+i4o= -github.com/multiversx/mx-chain-core-go v1.2.9-0.20230706101124-049df93d2fac/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= +github.com/multiversx/mx-chain-core-go v1.2.9 h1:uXrmzhKxEe5sztUTs1am3ILrxuBwNQQOESOKBlyrlLE= +github.com/multiversx/mx-chain-core-go v1.2.9/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-crypto-go v1.2.6/go.mod h1:rOj0Rr19HTOYt9YTeym7RKxlHt91NXln3LVKjHKVmA0= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= From c3d7b71bf19fbada782475f7b8a2cdfc1e2483e6 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 12 Jul 2023 15:21:19 +0300 Subject: [PATCH 109/189] integrate new intra-shard miniblocks --- go.mod | 2 +- go.sum | 4 +- process/elasticproc/elasticProcessor.go | 3 +- process/elasticproc/interface.go | 2 +- process/elasticproc/transactions/checkers.go | 6 +- .../elasticproc/transactions/checkers_test.go | 17 +- .../smartContractResultsProcessor.go | 4 +- .../transactions/transactionsProcessor.go | 14 +- .../transactionsProcessor_test.go | 168 ++++++++---------- 9 files changed, 99 insertions(+), 121 deletions(-) diff --git a/go.mod b/go.mod index b89e9e5e..f3665175 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/multiversx/mx-chain-communication-go v1.0.3 - github.com/multiversx/mx-chain-core-go v1.2.9 + github.com/multiversx/mx-chain-core-go v1.2.10-0.20230712104444-e818eaa01c7e github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.1 github.com/stretchr/testify v1.8.2 diff --git a/go.sum b/go.sum index d98700c6..c7a2f55c 100644 --- a/go.sum +++ b/go.sum @@ -476,8 +476,8 @@ github.com/multiversx/mx-chain-communication-go v1.0.3/go.mod h1:7oTI77XfWmRWwVE github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.5/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= -github.com/multiversx/mx-chain-core-go v1.2.9 h1:uXrmzhKxEe5sztUTs1am3ILrxuBwNQQOESOKBlyrlLE= -github.com/multiversx/mx-chain-core-go v1.2.9/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= +github.com/multiversx/mx-chain-core-go v1.2.10-0.20230712104444-e818eaa01c7e h1:ZITU8iHolDswJGMToGHIO7Wc8K6p6k6rsaxV/Z8UO3E= +github.com/multiversx/mx-chain-core-go v1.2.10-0.20230712104444-e818eaa01c7e/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-crypto-go v1.2.6/go.mod h1:rOj0Rr19HTOYt9YTeym7RKxlHt91NXln3LVKjHKVmA0= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index efcf13a5..c9da2975 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -410,7 +410,8 @@ func (ei *elasticProcessor) miniblocksInDBMap(mbs []*data.Miniblock) (map[string func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader) error { headerTimestamp := obh.Header.GetTimeStamp() - preparedResults := ei.transactionsProc.PrepareTransactionsForDatabase(obh.BlockData.Body, obh.Header, obh.TransactionPool, ei.isImportDB(), obh.NumberOfShards) + miniBlocks := append(obh.BlockData.Body.MiniBlocks, obh.BlockData.IntraShardMiniBlocks...) + preparedResults := ei.transactionsProc.PrepareTransactionsForDatabase(miniBlocks, obh.Header, obh.TransactionPool, ei.isImportDB(), obh.NumberOfShards) logsData := ei.logsAndEventsProc.ExtractDataFromLogs(obh.TransactionPool.Logs, preparedResults, headerTimestamp, obh.Header.GetShardID(), obh.NumberOfShards) buffers := data.NewBufferSlice(ei.bulkRequestMaxSize) diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index be7e9166..8f2479d2 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -57,7 +57,7 @@ type DBBlockHandler interface { // DBTransactionsHandler defines the actions that a transactions handler should do type DBTransactionsHandler interface { PrepareTransactionsForDatabase( - body *block.Body, + miniBlocks []*block.MiniBlock, header coreData.HeaderHandler, pool *outport.TransactionPool, isImportDB bool, diff --git a/process/elasticproc/transactions/checkers.go b/process/elasticproc/transactions/checkers.go index 041cdfb8..7d9fcaf7 100644 --- a/process/elasticproc/transactions/checkers.go +++ b/process/elasticproc/transactions/checkers.go @@ -9,7 +9,6 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" coreData "github.com/multiversx/mx-chain-core-go/data" - "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-es-indexer-go/data" elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" @@ -51,13 +50,10 @@ func areESDTValuesOK(values []string) bool { } func checkPrepareTransactionForDatabaseArguments( - body *block.Body, header coreData.HeaderHandler, pool *outport.TransactionPool, ) error { - if body == nil { - return elasticIndexer.ErrNilBlockBody - } + if check.IfNil(header) { return elasticIndexer.ErrNilHeaderHandler } diff --git a/process/elasticproc/transactions/checkers_test.go b/process/elasticproc/transactions/checkers_test.go index 1377be2d..06ebe9e9 100644 --- a/process/elasticproc/transactions/checkers_test.go +++ b/process/elasticproc/transactions/checkers_test.go @@ -72,27 +72,20 @@ func TestCheckTxsProcessorArg(t *testing.T) { tests := []struct { name string - args func() (body *block.Body, header coreData.HeaderHandler, pool *outport.TransactionPool) + args func() (header coreData.HeaderHandler, pool *outport.TransactionPool) exErr error }{ - { - name: "NilBlockBody", - args: func() (body *block.Body, header coreData.HeaderHandler, pool *outport.TransactionPool) { - return nil, &block.Header{}, &outport.TransactionPool{} - }, - exErr: elasticIndexer.ErrNilBlockBody, - }, { name: "NilHeaderHandler", - args: func() (body *block.Body, header coreData.HeaderHandler, pool *outport.TransactionPool) { - return &block.Body{}, nil, &outport.TransactionPool{} + args: func() (header coreData.HeaderHandler, pool *outport.TransactionPool) { + return nil, &outport.TransactionPool{} }, exErr: elasticIndexer.ErrNilHeaderHandler, }, { name: "NilPool", - args: func() (body *block.Body, header coreData.HeaderHandler, pool *outport.TransactionPool) { - return &block.Body{}, &block.Header{}, nil + args: func() (header coreData.HeaderHandler, pool *outport.TransactionPool) { + return &block.Header{}, nil }, exErr: elasticIndexer.ErrNilPool, }, diff --git a/process/elasticproc/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go index 3d91deb1..e5f6a72e 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -41,7 +41,7 @@ func newSmartContractResultsProcessor( } func (proc *smartContractResultsProcessor) processSCRs( - body *block.Body, + miniBlocks []*block.MiniBlock, header coreData.HeaderHandler, scrs map[string]*outport.SCRInfo, numOfShards uint32, @@ -50,7 +50,7 @@ func (proc *smartContractResultsProcessor) processSCRs( // a copy of the SCRS map is needed because proc.processSCRsFromMiniblock would remove items from the original map workingSCRSMap := copySCRSMap(scrs) - for _, mb := range body.MiniBlocks { + for _, mb := range miniBlocks { if mb.Type != block.SmartContractResultBlock { continue } diff --git a/process/elasticproc/transactions/transactionsProcessor.go b/process/elasticproc/transactions/transactionsProcessor.go index 3c7494cf..86e59b38 100644 --- a/process/elasticproc/transactions/transactionsProcessor.go +++ b/process/elasticproc/transactions/transactionsProcessor.go @@ -66,13 +66,13 @@ func NewTransactionsProcessor(args *ArgsTransactionProcessor) (*txsDatabaseProce // PrepareTransactionsForDatabase will prepare transactions for database func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( - body *block.Body, + miniBlocks []*block.MiniBlock, header coreData.HeaderHandler, pool *outport.TransactionPool, isImportDB bool, numOfShards uint32, ) *data.PreparedResults { - err := checkPrepareTransactionForDatabaseArguments(body, header, pool) + err := checkPrepareTransactionForDatabaseArguments(header, pool) if err != nil { log.Warn("checkPrepareTransactionForDatabaseArguments", "error", err) @@ -86,7 +86,7 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( normalTxs := make(map[string]*data.Transaction) rewardsTxs := make(map[string]*data.Transaction) - for mbIndex, mb := range body.MiniBlocks { + for mbIndex, mb := range miniBlocks { switch mb.Type { case block.TxBlock: if shouldIgnoreProcessedMBScheduled(header, mbIndex) { @@ -120,7 +120,7 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( normalTxs = tdp.setTransactionSearchOrder(normalTxs) dbReceipts := tdp.txsGrouper.groupReceipts(header, pool.Receipts) - dbSCResults := tdp.scrsProc.processSCRs(body, header, pool.SmartContractResults, numOfShards) + dbSCResults := tdp.scrsProc.processSCRs(miniBlocks, header, pool.SmartContractResults, numOfShards) srcsNoTxInCurrentShard := tdp.scrsDataToTxs.attachSCRsToTransactionsAndReturnSCRsWithoutTx(normalTxs, dbSCResults) tdp.scrsDataToTxs.processTransactionsAfterSCRsWereAttached(normalTxs) @@ -184,12 +184,12 @@ func isCrossShardAtSourceNormalTx(selfShardID uint32, miniblock *block.MiniBlock } func shouldIgnoreProcessedMBScheduled(header coreData.HeaderHandler, mbIndex int) bool { - miniblockHeaders := header.GetMiniBlockHeaderHandlers() - if len(miniblockHeaders) <= mbIndex { + miniBlockHeaders := header.GetMiniBlockHeaderHandlers() + if len(miniBlockHeaders) <= mbIndex { return false } - processingType := miniblockHeaders[mbIndex].GetProcessingType() + processingType := miniBlockHeaders[mbIndex].GetProcessingType() return processingType == int32(block.Processed) } diff --git a/process/elasticproc/transactions/transactionsProcessor_test.go b/process/elasticproc/transactions/transactionsProcessor_test.go index f4eb6b66..1348e427 100644 --- a/process/elasticproc/transactions/transactionsProcessor_test.go +++ b/process/elasticproc/transactions/transactionsProcessor_test.go @@ -151,38 +151,37 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { }, } - body := &block.Body{ - MiniBlocks: []*block.MiniBlock{ - { - TxHashes: [][]byte{txHash1, txHash2, txHash3}, - Type: block.TxBlock, - }, - { - TxHashes: [][]byte{txHash4}, - Type: block.TxBlock, - }, - { - TxHashes: [][]byte{scHash1, scHash2}, - Type: block.SmartContractResultBlock, - }, - { - TxHashes: [][]byte{scHash3}, - Type: block.SmartContractResultBlock, - }, - { - TxHashes: [][]byte{recHash1, recHash2}, - Type: block.ReceiptBlock, - }, - { - TxHashes: [][]byte{rTx1Hash, rTx2Hash}, - Type: block.RewardsBlock, - }, - { - TxHashes: [][]byte{txHash5}, - Type: block.InvalidBlock, - }, + mbs := []*block.MiniBlock{ + { + TxHashes: [][]byte{txHash1, txHash2, txHash3}, + Type: block.TxBlock, + }, + { + TxHashes: [][]byte{txHash4}, + Type: block.TxBlock, + }, + { + TxHashes: [][]byte{scHash1, scHash2}, + Type: block.SmartContractResultBlock, + }, + { + TxHashes: [][]byte{scHash3}, + Type: block.SmartContractResultBlock, + }, + { + TxHashes: [][]byte{recHash1, recHash2}, + Type: block.ReceiptBlock, + }, + { + TxHashes: [][]byte{rTx1Hash, rTx2Hash}, + Type: block.RewardsBlock, + }, + { + TxHashes: [][]byte{txHash5}, + Type: block.InvalidBlock, }, } + header := &block.Header{} pool := &outport.TransactionPool{ @@ -212,7 +211,7 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { txDbProc, _ := NewTransactionsProcessor(createMockArgsTxsDBProc()) - results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false, 3) + results := txDbProc.PrepareTransactionsForDatabase(mbs, header, pool, false, 3) assert.Equal(t, 7, len(results.Transactions)) } @@ -243,16 +242,14 @@ func TestRelayedTransactions(t *testing.T) { GasLimit: 1, }, FeeInfo: &outport.FeeInfo{}} - body := &block.Body{ - MiniBlocks: []*block.MiniBlock{ - { - TxHashes: [][]byte{txHash1}, - Type: block.TxBlock, - }, - { - TxHashes: [][]byte{scHash1, scHash2}, - Type: block.SmartContractResultBlock, - }, + mbs := []*block.MiniBlock{ + { + TxHashes: [][]byte{txHash1}, + Type: block.TxBlock, + }, + { + TxHashes: [][]byte{scHash1, scHash2}, + Type: block.SmartContractResultBlock, }, } @@ -270,7 +267,7 @@ func TestRelayedTransactions(t *testing.T) { txDbProc, _ := NewTransactionsProcessor(createMockArgsTxsDBProc()) - results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false, 3) + results := txDbProc.PrepareTransactionsForDatabase(mbs, header, pool, false, 3) assert.Equal(t, 1, len(results.Transactions)) assert.Equal(t, 2, len(results.Transactions[0].SmartContractResults)) assert.Equal(t, transaction.TxStatusSuccess.String(), results.Transactions[0].Status) @@ -357,19 +354,16 @@ func TestCheckGasUsedInvalidTransaction(t *testing.T) { TxHash: txHash1, } - body := &block.Body{ - MiniBlocks: []*block.MiniBlock{ - { - TxHashes: [][]byte{txHash1}, - Type: block.InvalidBlock, - }, - { - TxHashes: [][]byte{recHash1}, - Type: block.ReceiptBlock, - }, + mbs := []*block.MiniBlock{ + { + TxHashes: [][]byte{txHash1}, + Type: block.InvalidBlock, + }, + { + TxHashes: [][]byte{recHash1}, + Type: block.ReceiptBlock, }, } - header := &block.Header{} pool := &outport.TransactionPool{ @@ -381,7 +375,7 @@ func TestCheckGasUsedInvalidTransaction(t *testing.T) { }, } - results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false, 3) + results := txDbProc.PrepareTransactionsForDatabase(mbs, header, pool, false, 3) require.Len(t, results.Transactions, 1) require.Equal(t, tx1.Transaction.GetGasLimit(), results.Transactions[0].GasUsed) } @@ -489,16 +483,14 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseInvalidTxWithSCR(t * FeeInfo: &outport.FeeInfo{}, } - body := &block.Body{ - MiniBlocks: []*block.MiniBlock{ - { - TxHashes: [][]byte{txHash1}, - Type: block.InvalidBlock, - }, - { - TxHashes: [][]byte{scResHash1}, - Type: block.SmartContractResultBlock, - }, + mbs := []*block.MiniBlock{ + { + TxHashes: [][]byte{txHash1}, + Type: block.InvalidBlock, + }, + { + TxHashes: [][]byte{scResHash1}, + Type: block.SmartContractResultBlock, }, } @@ -513,7 +505,7 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseInvalidTxWithSCR(t * }, } - results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false, 3) + results := txDbProc.PrepareTransactionsForDatabase(mbs, header, pool, false, 3) require.NotNil(t, results) require.Len(t, results.Transactions, 1) require.Len(t, results.ScResults, 1) @@ -548,16 +540,14 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseESDTNFTTransfer(t *t }, FeeInfo: &outport.FeeInfo{}} - body := &block.Body{ - MiniBlocks: []*block.MiniBlock{ - { - TxHashes: [][]byte{txHash1}, - Type: block.TxBlock, - }, - { - TxHashes: [][]byte{scResHash1}, - Type: block.SmartContractResultBlock, - }, + mbs := []*block.MiniBlock{ + { + TxHashes: [][]byte{txHash1}, + Type: block.TxBlock, + }, + { + TxHashes: [][]byte{scResHash1}, + Type: block.SmartContractResultBlock, }, } @@ -572,7 +562,7 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseESDTNFTTransfer(t *t }, } - results := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false, 3) + results := txDbProc.PrepareTransactionsForDatabase(mbs, header, pool, false, 3) require.NotNil(t, results) require.Len(t, results.Transactions, 1) require.Len(t, results.ScResults, 1) @@ -597,18 +587,16 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { } // transaction success - body := &block.Body{ - MiniBlocks: []*block.MiniBlock{ - { - TxHashes: [][]byte{[]byte("t1")}, - Type: block.TxBlock, - SenderShardID: 0, - ReceiverShardID: core.MetachainShardId, - }, - { - TxHashes: [][]byte{[]byte("scr1"), []byte("scr2")}, - Type: block.SmartContractResultBlock, - }, + mbs := []*block.MiniBlock{ + { + TxHashes: [][]byte{[]byte("t1")}, + Type: block.TxBlock, + SenderShardID: 0, + ReceiverShardID: core.MetachainShardId, + }, + { + TxHashes: [][]byte{[]byte("scr1"), []byte("scr2")}, + Type: block.SmartContractResultBlock, }, } header := &block.Header{ @@ -638,7 +626,7 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { }, } - res := txDbProc.PrepareTransactionsForDatabase(body, header, pool, false, 3) + res := txDbProc.PrepareTransactionsForDatabase(mbs, header, pool, false, 3) require.Equal(t, "success", res.Transactions[0].Status) require.Equal(t, 2, len(res.ScResults)) @@ -661,7 +649,7 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { }, } - res = txDbProc.PrepareTransactionsForDatabase(body, header, pool, false, 3) + res = txDbProc.PrepareTransactionsForDatabase(mbs, header, pool, false, 3) require.Equal(t, "success", res.Transactions[0].Status) require.Equal(t, 1, len(res.ScResults)) } From 67373356b886e38460c59967bc380af61d75c877 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 12 Jul 2023 15:40:54 +0300 Subject: [PATCH 110/189] fix tests --- mock/dbTransactionsHandlerStub.go | 6 +++--- process/elasticproc/elasticProcessor_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mock/dbTransactionsHandlerStub.go b/mock/dbTransactionsHandlerStub.go index 0a8c2f96..c4aff654 100644 --- a/mock/dbTransactionsHandlerStub.go +++ b/mock/dbTransactionsHandlerStub.go @@ -9,7 +9,7 @@ import ( // DBTransactionProcessorStub - type DBTransactionProcessorStub struct { - PrepareTransactionsForDatabaseCalled func(body *block.Body, header coreData.HeaderHandler, pool *outport.TransactionPool) *data.PreparedResults + PrepareTransactionsForDatabaseCalled func(mbs []*block.MiniBlock, header coreData.HeaderHandler, pool *outport.TransactionPool) *data.PreparedResults SerializeReceiptsCalled func(recs []*data.Receipt, buffSlice *data.BufferSlice, index string) error SerializeScResultsCalled func(scrs []*data.ScResult, buffSlice *data.BufferSlice, index string) error } @@ -20,9 +20,9 @@ func (tps *DBTransactionProcessorStub) SerializeTransactionsFeeData(_ map[string } // PrepareTransactionsForDatabase - -func (tps *DBTransactionProcessorStub) PrepareTransactionsForDatabase(body *block.Body, header coreData.HeaderHandler, pool *outport.TransactionPool, _ bool, _ uint32) *data.PreparedResults { +func (tps *DBTransactionProcessorStub) PrepareTransactionsForDatabase(mbs []*block.MiniBlock, header coreData.HeaderHandler, pool *outport.TransactionPool, _ bool, _ uint32) *data.PreparedResults { if tps.PrepareTransactionsForDatabaseCalled != nil { - return tps.PrepareTransactionsForDatabaseCalled(body, header, pool) + return tps.PrepareTransactionsForDatabaseCalled(mbs, header, pool) } return nil diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index 5c6385c5..a50e6efc 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -614,7 +614,7 @@ func TestElasticProcessor_SaveTransactionNoDataShouldNotDoRequest(t *testing.T) called := false arguments := createMockElasticProcessorArgs() arguments.TransactionsProc = &mock.DBTransactionProcessorStub{ - PrepareTransactionsForDatabaseCalled: func(body *dataBlock.Body, header coreData.HeaderHandler, pool *outport.TransactionPool) *data.PreparedResults { + PrepareTransactionsForDatabaseCalled: func(mbs []*dataBlock.MiniBlock, header coreData.HeaderHandler, pool *outport.TransactionPool) *data.PreparedResults { return &data.PreparedResults{ Transactions: nil, ScResults: nil, From aa453b6c7e3e1436d9cfc6194163603d717e16ac Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 13 Jul 2023 12:03:29 +0300 Subject: [PATCH 111/189] index intra miniblocks --- mock/elasticProcessorStub.go | 6 +++--- process/dataindexer/dataIndexer.go | 3 ++- process/dataindexer/dataIndexer_test.go | 2 +- process/dataindexer/interface.go | 2 +- process/elasticproc/elasticProcessor.go | 4 ++-- process/elasticproc/elasticProcessor_test.go | 2 +- process/elasticproc/interface.go | 2 +- .../miniblocks/miniblocksProcessor.go | 19 ++++++++++++------- .../miniblocks/miniblocksProcessor_test.go | 4 ++-- 9 files changed, 25 insertions(+), 19 deletions(-) diff --git a/mock/elasticProcessorStub.go b/mock/elasticProcessorStub.go index 7c0cd091..3d01b732 100644 --- a/mock/elasticProcessorStub.go +++ b/mock/elasticProcessorStub.go @@ -12,7 +12,7 @@ type ElasticProcessorStub struct { RemoveHeaderCalled func(header coreData.HeaderHandler) error RemoveMiniblocksCalled func(header coreData.HeaderHandler, body *block.Body) error RemoveTransactionsCalled func(header coreData.HeaderHandler, body *block.Body) error - SaveMiniblocksCalled func(header coreData.HeaderHandler, body *block.Body) error + SaveMiniblocksCalled func(header coreData.HeaderHandler, miniBlocks []*block.MiniBlock) error SaveTransactionsCalled func(outportBlockWithHeader *outport.OutportBlockWithHeader) error SaveValidatorsRatingCalled func(validatorsRating *outport.ValidatorsRating) error SaveRoundsInfoCalled func(infos *outport.RoundsInfo) error @@ -63,9 +63,9 @@ func (eim *ElasticProcessorStub) RemoveTransactions(header coreData.HeaderHandle } // SaveMiniblocks - -func (eim *ElasticProcessorStub) SaveMiniblocks(header coreData.HeaderHandler, body *block.Body) error { +func (eim *ElasticProcessorStub) SaveMiniblocks(header coreData.HeaderHandler, miniBlocks []*block.MiniBlock) error { if eim.SaveMiniblocksCalled != nil { - return eim.SaveMiniblocksCalled(header, body) + return eim.SaveMiniblocksCalled(header, miniBlocks) } return nil } diff --git a/process/dataindexer/dataIndexer.go b/process/dataindexer/dataIndexer.go index 6e571b08..9f75e343 100644 --- a/process/dataindexer/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -114,7 +114,8 @@ func (di *dataIndexer) saveBlockData(outportBlock *outport.OutportBlock, header return nil } - err = di.elasticProcessor.SaveMiniblocks(header, outportBlock.BlockData.Body) + miniBlocks := append(outportBlock.BlockData.Body.MiniBlocks, outportBlock.BlockData.IntraShardMiniBlocks...) + err = di.elasticProcessor.SaveMiniblocks(header, miniBlocks) if err != nil { return fmt.Errorf("%w when saving miniblocks, block hash %s, nonce %d", err, hex.EncodeToString(headerHash), headerNonce) diff --git a/process/dataindexer/dataIndexer_test.go b/process/dataindexer/dataIndexer_test.go index 9cc77a5f..88970177 100644 --- a/process/dataindexer/dataIndexer_test.go +++ b/process/dataindexer/dataIndexer_test.go @@ -62,7 +62,7 @@ func TestDataIndexer_SaveBlock(t *testing.T) { countMap[0]++ return nil }, - SaveMiniblocksCalled: func(header coreData.HeaderHandler, body *dataBlock.Body) error { + SaveMiniblocksCalled: func(header coreData.HeaderHandler, miniBlocks []*dataBlock.MiniBlock) error { countMap[1]++ return nil }, diff --git a/process/dataindexer/interface.go b/process/dataindexer/interface.go index 72a9e03a..cb99ab00 100644 --- a/process/dataindexer/interface.go +++ b/process/dataindexer/interface.go @@ -17,7 +17,7 @@ type ElasticProcessor interface { RemoveMiniblocks(header coreData.HeaderHandler, body *block.Body) error RemoveTransactions(header coreData.HeaderHandler, body *block.Body) error RemoveAccountsESDT(headerTimestamp uint64, shardID uint32) error - SaveMiniblocks(header coreData.HeaderHandler, body *block.Body) error + SaveMiniblocks(header coreData.HeaderHandler, miniBlocks []*block.MiniBlock) error SaveTransactions(outportBlockWithHeader *outport.OutportBlockWithHeader) error SaveValidatorsRating(ratingData *outport.ValidatorsRating) error SaveRoundsInfo(rounds *outport.RoundsInfo) error diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index c9da2975..7170b4f8 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -376,12 +376,12 @@ func (ei *elasticProcessor) RemoveAccountsESDT(headerTimestamp uint64, shardID u } // SaveMiniblocks will prepare and save information about miniblocks in elasticsearch server -func (ei *elasticProcessor) SaveMiniblocks(header coreData.HeaderHandler, body *block.Body) error { +func (ei *elasticProcessor) SaveMiniblocks(header coreData.HeaderHandler, miniBlocks []*block.MiniBlock) error { if !ei.isIndexEnabled(elasticIndexer.MiniblocksIndex) { return nil } - mbs := ei.miniblocksProc.PrepareDBMiniblocks(header, body) + mbs := ei.miniblocksProc.PrepareDBMiniblocks(header, miniBlocks) if len(mbs) == 0 { return nil } diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index a50e6efc..7a533f60 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -455,7 +455,7 @@ func TestElasticProcessor_SaveMiniblocks(t *testing.T) { body := &dataBlock.Body{MiniBlocks: dataBlock.MiniBlockSlice{ {SenderShardID: 0, ReceiverShardID: 1}, }} - err := elasticProc.SaveMiniblocks(header, body) + err := elasticProc.SaveMiniblocks(header, body.MiniBlocks) require.Equal(t, localErr, err) } diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index 8f2479d2..dcb8ef93 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -73,7 +73,7 @@ type DBTransactionsHandler interface { // DBMiniblocksHandler defines the actions that a miniblocks handler should do type DBMiniblocksHandler interface { - PrepareDBMiniblocks(header coreData.HeaderHandler, body *block.Body) []*data.Miniblock + PrepareDBMiniblocks(header coreData.HeaderHandler, miniBlocks []*block.MiniBlock) []*data.Miniblock GetMiniblocksHashesHexEncoded(header coreData.HeaderHandler, body *block.Body) []string SerializeBulkMiniBlocks(bulkMbs []*data.Miniblock, mbsInDB map[string]bool, buffSlice *data.BufferSlice, index string, shardID uint32) diff --git a/process/elasticproc/miniblocks/miniblocksProcessor.go b/process/elasticproc/miniblocks/miniblocksProcessor.go index fd8067b9..b7acefe7 100644 --- a/process/elasticproc/miniblocks/miniblocksProcessor.go +++ b/process/elasticproc/miniblocks/miniblocksProcessor.go @@ -40,8 +40,8 @@ func NewMiniblocksProcessor( }, nil } -// PrepareDBMiniblocks will prepare miniblocks from body -func (mp *miniblocksProcessor) PrepareDBMiniblocks(header coreData.HeaderHandler, body *block.Body) []*data.Miniblock { +// PrepareDBMiniblocks will prepare miniblocks +func (mp *miniblocksProcessor) PrepareDBMiniblocks(header coreData.HeaderHandler, miniBlocks []*block.MiniBlock) []*data.Miniblock { headerHash, err := mp.calculateHash(header) if err != nil { log.Warn("indexer: could not calculate header hash", "error", err) @@ -49,14 +49,14 @@ func (mp *miniblocksProcessor) PrepareDBMiniblocks(header coreData.HeaderHandler } dbMiniblocks := make([]*data.Miniblock, 0) - for mbIndex, miniblock := range body.MiniBlocks { - dbMiniblock, errPrepareMiniblock := mp.prepareMiniblockForDB(mbIndex, miniblock, header, headerHash) - if errPrepareMiniblock != nil { - log.Warn("miniblocksProcessor.PrepareDBMiniblocks cannot prepare miniblock", "error", errPrepareMiniblock) + for mbIndex, miniBlock := range miniBlocks { + dbMiniBlock, errPrepareMiniBlock := mp.prepareMiniblockForDB(mbIndex, miniBlock, header, headerHash) + if errPrepareMiniBlock != nil { + log.Warn("miniblocksProcessor.PrepareDBMiniBlocks cannot prepare miniblock", "error", errPrepareMiniBlock) continue } - dbMiniblocks = append(dbMiniblocks, dbMiniblock) + dbMiniblocks = append(dbMiniblocks, dbMiniBlock) } return dbMiniblocks @@ -121,6 +121,11 @@ func (mp *miniblocksProcessor) setFieldsMBIntraShardAndCrossFromMe( case constructionState == int32(block.Final) && processingType == block.Processed.String(): dbMiniblock.ReceiverBlockHash = headerHash dbMiniblock.ProcessingTypeOnDestination = processingType + case constructionState == 0 && processingType == "": + dbMiniblock.SenderBlockHash = headerHash + dbMiniblock.ReceiverBlockHash = headerHash + dbMiniblock.ProcessingTypeOnSource = block.Normal.String() + dbMiniblock.ProcessingTypeOnDestination = block.Normal.String() } } diff --git a/process/elasticproc/miniblocks/miniblocksProcessor_test.go b/process/elasticproc/miniblocks/miniblocksProcessor_test.go index c5b80ccb..5fea0248 100644 --- a/process/elasticproc/miniblocks/miniblocksProcessor_test.go +++ b/process/elasticproc/miniblocks/miniblocksProcessor_test.go @@ -64,7 +64,7 @@ func TestMiniblocksProcessor_PrepareDBMiniblocks(t *testing.T) { }, } - miniblocks := mp.PrepareDBMiniblocks(header, body) + miniblocks := mp.PrepareDBMiniblocks(header, body.MiniBlocks) require.Len(t, miniblocks, 3) } @@ -103,7 +103,7 @@ func TestMiniblocksProcessor_PrepareScheduledMB(t *testing.T) { }, } - miniblocks := mp.PrepareDBMiniblocks(header, body) + miniblocks := mp.PrepareDBMiniblocks(header, body.MiniBlocks) require.Len(t, miniblocks, 2) require.Equal(t, dataBlock.Scheduled.String(), miniblocks[1].ProcessingTypeOnSource) } From 5c8bab4abfb7f2aaf606986c210379f9aec50866 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 13 Jul 2023 12:09:11 +0300 Subject: [PATCH 112/189] extend test --- .../miniblocks/miniblocksProcessor_test.go | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/process/elasticproc/miniblocks/miniblocksProcessor_test.go b/process/elasticproc/miniblocks/miniblocksProcessor_test.go index 5fea0248..f9cf687b 100644 --- a/process/elasticproc/miniblocks/miniblocksProcessor_test.go +++ b/process/elasticproc/miniblocks/miniblocksProcessor_test.go @@ -6,6 +6,7 @@ import ( dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/stretchr/testify/require" @@ -90,22 +91,35 @@ func TestMiniblocksProcessor_PrepareScheduledMB(t *testing.T) { }, }, } - body := &dataBlock.Body{ - MiniBlocks: []*dataBlock.MiniBlock{ - { - SenderShardID: 0, - ReceiverShardID: 1, - }, - { - SenderShardID: 0, - ReceiverShardID: 1, - }, + miniBlocks := []*dataBlock.MiniBlock{ + { + SenderShardID: 0, + ReceiverShardID: 1, + }, + { + SenderShardID: 0, + ReceiverShardID: 1, + }, + { + SenderShardID: 0, + ReceiverShardID: 0, }, } - miniblocks := mp.PrepareDBMiniblocks(header, body.MiniBlocks) - require.Len(t, miniblocks, 2) + miniblocks := mp.PrepareDBMiniblocks(header, miniBlocks) + require.Len(t, miniblocks, 3) require.Equal(t, dataBlock.Scheduled.String(), miniblocks[1].ProcessingTypeOnSource) + + require.Equal(t, &data.Miniblock{ + Hash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + SenderShardID: 0, + ReceiverShardID: 0, + SenderBlockHash: "84b80cbf612d067201b5260b4c6f90fa7b5c11e898fa9c1c65f4c75e61e41619", + ReceiverBlockHash: "84b80cbf612d067201b5260b4c6f90fa7b5c11e898fa9c1c65f4c75e61e41619", + ProcessingTypeOnSource: dataBlock.Normal.String(), + ProcessingTypeOnDestination: dataBlock.Normal.String(), + Type: dataBlock.TxBlock.String(), + }, miniblocks[2]) } func TestMiniblocksProcessor_GetMiniblocksHashesHexEncoded(t *testing.T) { From 993494180fcab2b02faba7337aa4323276b18c86 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 14 Jul 2023 11:47:28 +0300 Subject: [PATCH 113/189] web server for metrics --- api/gin/httpServer.go | 59 ++++++++++++++ api/gin/interface.go | 8 ++ api/gin/webServer.go | 122 +++++++++++++++++++++++++++++ api/gin/writters.go | 25 ++++++ api/groups/baseGroup.go | 69 ++++++++++++++++ api/groups/statusGroup.go | 81 +++++++++++++++++++ api/shared/interface.go | 29 +++++++ api/shared/shared.go | 35 +++++++++ cmd/elasticindexer/config/api.toml | 9 +++ cmd/elasticindexer/flags.go | 14 +++- cmd/elasticindexer/main.go | 34 ++++++++ config/config.go | 17 ++++ core/errors.go | 9 +++ core/interface.go | 13 +++ facade/facade.go | 35 +++++++++ factory/webServerFactory.go | 21 +++++ go.mod | 33 ++++++-- go.sum | 114 +++++++++++++++++++++++++-- metrics/statusMetrics.go | 59 ++++++++++++++ 19 files changed, 772 insertions(+), 14 deletions(-) create mode 100644 api/gin/httpServer.go create mode 100644 api/gin/interface.go create mode 100644 api/gin/webServer.go create mode 100644 api/gin/writters.go create mode 100644 api/groups/baseGroup.go create mode 100644 api/groups/statusGroup.go create mode 100644 api/shared/interface.go create mode 100644 api/shared/shared.go create mode 100644 cmd/elasticindexer/config/api.toml create mode 100644 core/errors.go create mode 100644 core/interface.go create mode 100644 facade/facade.go create mode 100644 factory/webServerFactory.go create mode 100644 metrics/statusMetrics.go diff --git a/api/gin/httpServer.go b/api/gin/httpServer.go new file mode 100644 index 00000000..18ed0bf8 --- /dev/null +++ b/api/gin/httpServer.go @@ -0,0 +1,59 @@ +package gin + +import ( + "context" + "errors" + "net/http" + "time" + + logger "github.com/multiversx/mx-chain-logger-go" +) + +var log = logger.GetOrCreate("api") + +// ErrNilHttpServer signals that a nil http server has been provided +var ErrNilHttpServer = errors.New("nil http server") + +type httpServer struct { + server server +} + +// NewHttpServer returns a new instance of httpServer +func NewHttpServer(server server) (*httpServer, error) { + if server == nil { + return nil, ErrNilHttpServer + } + + return &httpServer{ + server: server, + }, nil +} + +// Start will handle the starting of the gin web server. This call is blocking, and it should be +// called on a go routine (different from the main one) +func (h *httpServer) Start() { + err := h.server.ListenAndServe() + if err == nil { + return + } + + if err == http.ErrServerClosed { + log.Debug("ListenAndServe - webserver closed") + return + } + + log.Error("could not start webserver", "error", err.Error()) +} + +// Close will handle the stopping of the gin web server +func (h *httpServer) Close() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + return h.server.Shutdown(ctx) +} + +// IsInterfaceNil returns true if there is no value under the interface +func (h *httpServer) IsInterfaceNil() bool { + return h == nil +} diff --git a/api/gin/interface.go b/api/gin/interface.go new file mode 100644 index 00000000..6773fe65 --- /dev/null +++ b/api/gin/interface.go @@ -0,0 +1,8 @@ +package gin + +import "context" + +type server interface { + ListenAndServe() error + Shutdown(ctx context.Context) error +} diff --git a/api/gin/webServer.go b/api/gin/webServer.go new file mode 100644 index 00000000..36e2dd62 --- /dev/null +++ b/api/gin/webServer.go @@ -0,0 +1,122 @@ +package gin + +import ( + "fmt" + "net/http" + "sync" + + "github.com/gin-contrib/cors" + "github.com/gin-gonic/gin" + "github.com/multiversx/mx-chain-es-indexer-go/api/groups" + "github.com/multiversx/mx-chain-es-indexer-go/api/shared" + "github.com/multiversx/mx-chain-es-indexer-go/config" +) + +const ( + webServerOffString = "off" +) + +type ArgsWebServer struct { + Facade shared.FacadeHandler + ApiConfig config.ApiRoutesConfig +} + +type webServer struct { + sync.RWMutex + facade shared.FacadeHandler + apiConfig config.ApiRoutesConfig + groups map[string]shared.GroupHandler + httpServer shared.HttpServerCloser +} + +// NewWebServer will create a new instance of the webServer +func NewWebServer(args ArgsWebServer) (*webServer, error) { + return &webServer{ + facade: args.Facade, + apiConfig: args.ApiConfig, + }, nil +} + +// StartHttpServer will start the http server +func (ws *webServer) StartHttpServer() error { + ws.Lock() + defer ws.Unlock() + + apiInterface := ws.apiConfig.RestApiInterface + if apiInterface == webServerOffString { + log.Debug("web server is turned off") + return nil + } + + var engine *gin.Engine + gin.DefaultWriter = &ginWriter{} + gin.DefaultErrorWriter = &ginErrorWriter{} + gin.DisableConsoleColor() + gin.SetMode(gin.ReleaseMode) + + engine = gin.Default() + cfg := cors.DefaultConfig() + cfg.AllowAllOrigins = true + cfg.AddAllowHeaders("Authorization") + engine.Use(cors.New(cfg)) + + err := ws.createGroups() + if err != nil { + return err + } + + s := &http.Server{Addr: apiInterface, Handler: engine} + log.Debug("creating gin web sever", "interface", apiInterface) + ws.httpServer, err = NewHttpServer(s) + if err != nil { + return err + } + + log.Debug("starting web server") + go ws.httpServer.Start() + + return nil +} + +func (ws *webServer) createGroups() error { + groupsMap := make(map[string]shared.GroupHandler) + + statusGroup, err := groups.NewStatusGroup(ws.facade) + if err != nil { + return err + } + groupsMap["status"] = statusGroup + + ws.groups = groupsMap + + return nil +} + +func (ws *webServer) registerRoutes(ginRouter *gin.Engine) { + for groupName, groupHandler := range ws.groups { + log.Debug("registering gin API group", "group name", groupName) + ginGroup := ginRouter.Group(fmt.Sprintf("/%s", groupName)) + groupHandler.RegisterRoutes(ginGroup, ws.apiConfig) + } +} + +// Close will handle the closing of inner components +func (ws *webServer) Close() error { + var err error + ws.Lock() + if ws.httpServer != nil { + err = ws.httpServer.Close() + } + ws.Unlock() + + if err != nil { + err = fmt.Errorf("%w while closing the http server in gin/webServer", err) + } + + return err +} + +// IsInterfaceNil returns true if there is no value under the interface +func (ws *webServer) IsInterfaceNil() bool { + return ws == nil +} diff --git a/api/gin/writters.go b/api/gin/writters.go new file mode 100644 index 00000000..ea73cceb --- /dev/null +++ b/api/gin/writters.go @@ -0,0 +1,25 @@ +package gin + +import "bytes" + +type ginWriter struct { +} + +// Write will output the message using mx-chain-logger-go +func (gv *ginWriter) Write(p []byte) (n int, err error) { + trimmed := bytes.TrimSpace(p) + log.Trace("gin server", "message", string(trimmed)) + + return len(p), nil +} + +type ginErrorWriter struct { +} + +// Write will output the error using mx-chain-logger-go +func (gev *ginErrorWriter) Write(p []byte) (n int, err error) { + trimmed := bytes.TrimSpace(p) + log.Trace("gin server", "error", string(trimmed)) + + return len(p), nil +} diff --git a/api/groups/baseGroup.go b/api/groups/baseGroup.go new file mode 100644 index 00000000..a5b03180 --- /dev/null +++ b/api/groups/baseGroup.go @@ -0,0 +1,69 @@ +package groups + +import ( + "strings" + + "github.com/gin-gonic/gin" + "github.com/multiversx/mx-chain-es-indexer-go/api/shared" + "github.com/multiversx/mx-chain-es-indexer-go/config" + logger "github.com/multiversx/mx-chain-logger-go" +) + +var log = logger.GetOrCreate("api/groups") + +type endpointProperties struct { + isOpen bool +} + +type baseGroup struct { + endpoints []*shared.EndpointHandlerData +} + +// GetEndpoints returns all the providers specific to the group +func (bg *baseGroup) GetEndpoints() []*shared.EndpointHandlerData { + return bg.endpoints +} + +// RegisterRoutes will register all the providers to the given web server +func (bg *baseGroup) RegisterRoutes( + ws *gin.RouterGroup, + apiConfig config.ApiRoutesConfig, +) { + for _, handlerData := range bg.endpoints { + properties := getEndpointProperties(ws, handlerData.Path, apiConfig) + + if !properties.isOpen { + log.Debug("endpoint is closed", "path", handlerData.Path) + continue + } + + ws.Handle(handlerData.Method, handlerData.Path, handlerData.Handler) + } +} + +func getEndpointProperties(ws *gin.RouterGroup, path string, apiConfig config.ApiRoutesConfig) endpointProperties { + basePath := ws.BasePath() + + // ws.BasePath will return paths like /group or /v1.0/group, so we need the last token after splitting by / + splitPath := strings.Split(basePath, "/") + basePath = splitPath[len(splitPath)-1] + + group, ok := apiConfig.APIPackages[basePath] + if !ok { + return endpointProperties{ + isOpen: false, + } + } + + for _, route := range group.Routes { + if route.Name == path { + return endpointProperties{ + isOpen: route.Open, + } + } + } + + return endpointProperties{ + isOpen: false, + } +} diff --git a/api/groups/statusGroup.go b/api/groups/statusGroup.go new file mode 100644 index 00000000..36215f05 --- /dev/null +++ b/api/groups/statusGroup.go @@ -0,0 +1,81 @@ +package groups + +import ( + "fmt" + "net/http" + "sync" + + "github.com/gin-gonic/gin" + "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-es-indexer-go/api/shared" + "github.com/multiversx/mx-chain-es-indexer-go/core" +) + +const ( + metricsPath = "/metrics" + prometheusMetricsPath = "/prometheus-metrics" +) + +type statusGroup struct { + *baseGroup + facade shared.FacadeHandler + mutFacade sync.RWMutex +} + +// NewStatusGroup returns a new instance of status group +func NewStatusGroup(facade shared.FacadeHandler) (*statusGroup, error) { + if check.IfNil(facade) { + return nil, fmt.Errorf("%w for status group", core.ErrNilFacadeHandler) + } + + sg := &statusGroup{ + facade: facade, + baseGroup: &baseGroup{}, + } + + endpoints := []*shared.EndpointHandlerData{ + { + Path: metricsPath, + Handler: sg.getMetrics, + Method: http.MethodGet, + }, + { + Path: prometheusMetricsPath, + Handler: sg.getPrometheusMetrics, + Method: http.MethodGet, + }, + } + sg.endpoints = endpoints + + return sg, nil +} + +// getMetrics will expose endpoints statistics in json format +func (sg *statusGroup) getMetrics(c *gin.Context) { + metricsResults := sg.facade.GetMetrics() + + returnStatus(c, gin.H{"metrics": metricsResults}, http.StatusOK, "", "successful") +} + +// getPrometheusMetrics will expose proxy metrics in prometheus format +func (sg *statusGroup) getPrometheusMetrics(c *gin.Context) { + metricsResults := sg.facade.GetMetricsForPrometheus() + + c.String(http.StatusOK, metricsResults) +} + +// IsInterfaceNil returns true if there is no value under the interface +func (sg *statusGroup) IsInterfaceNil() bool { + return sg == nil +} + +func returnStatus(c *gin.Context, data interface{}, httpStatus int, err string, code string) { + c.JSON( + httpStatus, + shared.GenericAPIResponse{ + Data: data, + Error: err, + Code: code, + }, + ) +} diff --git a/api/shared/interface.go b/api/shared/interface.go new file mode 100644 index 00000000..8ff5fe96 --- /dev/null +++ b/api/shared/interface.go @@ -0,0 +1,29 @@ +package shared + +import ( + "github.com/gin-gonic/gin" + "github.com/multiversx/mx-chain-es-indexer-go/config" +) + +// GroupHandler defines the actions needed to be performed by a gin API group +type GroupHandler interface { + RegisterRoutes( + ws *gin.RouterGroup, + apiConfig config.ApiRoutesConfig, + ) + IsInterfaceNil() bool +} + +// FacadeHandler defines all the methods that a facade should implement +type FacadeHandler interface { + GetMetrics() map[string]interface{} + GetMetricsForPrometheus() string + IsInterfaceNil() bool +} + +// HttpServerCloser defines the basic actions of starting and closing that a web server should be able to do +type HttpServerCloser interface { + Start() + Close() error + IsInterfaceNil() bool +} diff --git a/api/shared/shared.go b/api/shared/shared.go new file mode 100644 index 00000000..e8cf9356 --- /dev/null +++ b/api/shared/shared.go @@ -0,0 +1,35 @@ +package shared + +import "github.com/gin-gonic/gin" + +// MiddlewarePosition is the type that specifies the position of a middleware relative to the base endpoint handler +type MiddlewarePosition bool + +const ( + // Before indicates that the middleware should be used before the base endpoint handler + Before MiddlewarePosition = true + + // After indicates that the middleware should be used after the base endpoint handler + After MiddlewarePosition = false +) + +// AdditionalMiddleware holds the data needed for adding a middleware to an API endpoint +type AdditionalMiddleware struct { + Middleware gin.HandlerFunc + Position MiddlewarePosition +} + +// EndpointHandlerData holds the items needed for creating a new gin HTTP endpoint +type EndpointHandlerData struct { + Path string + Method string + Handler gin.HandlerFunc + AdditionalMiddlewares []AdditionalMiddleware +} + +// GenericAPIResponse defines the structure of all responses on API endpoints +type GenericAPIResponse struct { + Data interface{} `json:"data"` + Error string `json:"error"` + Code string `json:"code"` +} diff --git a/cmd/elasticindexer/config/api.toml b/cmd/elasticindexer/config/api.toml new file mode 100644 index 00000000..ececfc1c --- /dev/null +++ b/cmd/elasticindexer/config/api.toml @@ -0,0 +1,9 @@ +rest-api-interface = ":8080" + +[api-packages] + +[api-packages.status] + routes = [ + { name = "/metrics", open = true }, + { name = "/prometheus-metrics", open = true }, + ] \ No newline at end of file diff --git a/cmd/elasticindexer/flags.go b/cmd/elasticindexer/flags.go index c8e2421d..a8c6ed86 100644 --- a/cmd/elasticindexer/flags.go +++ b/cmd/elasticindexer/flags.go @@ -5,6 +5,10 @@ import ( "github.com/urfave/cli" ) +const ( + filePathPlaceholder = "[path]" +) + var ( configurationFile = cli.StringFlag{ Name: "config", @@ -14,10 +18,18 @@ var ( // configurationPreferencesFile defines a flag for the path to the preferences toml configuration file configurationPreferencesFile = cli.StringFlag{ Name: "config-preferences", - Usage: "The [path] for the preferences configuration file. This TOML file contains " + + Usage: "The `" + filePathPlaceholder + "` for the preferences configuration file. This TOML file contains " + "preferences configurations, such as the node display name or the shard to start in when starting as observer", Value: "./config/prefs.toml", } + // configurationApiFile defines a flag for the path to the api routes toml configuration file + configurationApiFile = cli.StringFlag{ + Name: "config-api", + Usage: "The `" + filePathPlaceholder + "` for the api configuration file. This TOML file contains " + + "all available routes for Rest API and options to enable or disable them.", + Value: "config/api.toml", + } + logLevel = cli.StringFlag{ Name: "log-level", Usage: "This flag specifies the logger `level(s)`. It can contain multiple comma-separated value. For example" + diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index 5b8b9026..9794b65f 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -13,6 +13,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-es-indexer-go/config" "github.com/multiversx/mx-chain-es-indexer-go/factory" + "github.com/multiversx/mx-chain-es-indexer-go/metrics" "github.com/multiversx/mx-chain-es-indexer-go/process/wsindexer" logger "github.com/multiversx/mx-chain-logger-go" "github.com/multiversx/mx-chain-logger-go/file" @@ -46,6 +47,7 @@ func main() { app.Flags = []cli.Flag{ configurationFile, configurationPreferencesFile, + configurationApiFile, logLevel, logSaveFile, disableAnsiColor, @@ -89,6 +91,22 @@ func startIndexer(ctx *cli.Context) error { return fmt.Errorf("%w while creating the indexer", err) } + apiConfig, err := loadApiConfig(ctx.GlobalString(configurationApiFile.Name)) + if err != nil { + return fmt.Errorf("%w while loading the api config file", err) + } + + statusMetric := metrics.NewStatusMetrics() + webServer, err := factory.CreateWebServer(apiConfig, statusMetric) + if err != nil { + return fmt.Errorf("%w while creating the web server", err) + } + + err = webServer.StartHttpServer() + if err != nil { + return fmt.Errorf("%w while starting the web server", err) + } + interrupt := make(chan os.Signal, 1) signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM) @@ -104,6 +122,11 @@ func startIndexer(ctx *cli.Context) error { log.Error("cannot close ws indexer", "error", err) } + err = webServer.Close() + if err != nil { + log.Error("cannot close web server", "error", err) + } + if !check.IfNilReflect(fileLogging) { err = fileLogging.Close() log.LogIfError(err) @@ -146,6 +169,17 @@ func loadClusterConfig(filepath string) (config.ClusterConfig, error) { return cfg, err } +// loadApiConfig returns a ApiRoutesConfig by reading the config file provided +func loadApiConfig(filepath string) (config.ApiRoutesConfig, error) { + cfg := config.ApiRoutesConfig{} + err := core.LoadTomlFile(&cfg, filepath) + if err != nil { + return config.ApiRoutesConfig{}, err + } + + return cfg, nil +} + func initializeLogger(ctx *cli.Context, cfg config.Config) (closing.Closer, error) { logLevelFlagValue := ctx.GlobalString(logLevel.Name) err := logger.SetLogLevel(logLevelFlagValue) diff --git a/config/config.go b/config/config.go index e07b00b2..dc1ab2cb 100644 --- a/config/config.go +++ b/config/config.go @@ -53,3 +53,20 @@ type ClusterConfig struct { } `toml:"elastic-cluster"` } `toml:"config"` } + +// ApiRoutesConfig holds the configuration related to Rest API routes +type ApiRoutesConfig struct { + RestApiInterface string `toml:"rest-api-interface"` + APIPackages map[string]APIPackageConfig `toml:"api-packages"` +} + +// APIPackageConfig holds the configuration for the routes of each package +type APIPackageConfig struct { + Routes []RouteConfig `toml:"routes"` +} + +// RouteConfig holds the configuration for a single route +type RouteConfig struct { + Name string `toml:"name"` + Open bool `toml:"open"` +} diff --git a/core/errors.go b/core/errors.go new file mode 100644 index 00000000..45e554d7 --- /dev/null +++ b/core/errors.go @@ -0,0 +1,9 @@ +package core + +import "errors" + +// ErrNilMetricsHandler signals that a nil metrics handler has been provided +var ErrNilMetricsHandler = errors.New("nil metrics handler") + +// ErrNilFacadeHandler signal that a nil facade handler has been provided +var ErrNilFacadeHandler = errors.New("nil facade handler") diff --git a/core/interface.go b/core/interface.go new file mode 100644 index 00000000..997b6523 --- /dev/null +++ b/core/interface.go @@ -0,0 +1,13 @@ +package core + +// StatusMetricsHandler defines the behavior of a component that handles status metrics +type StatusMetricsHandler interface { + GetMetrics() map[string]interface{} + GetMetricsForPrometheus() string + IsInterfaceNil() bool +} + +type WebServerHandler interface { + StartHttpServer() error + Close() error +} diff --git a/facade/facade.go b/facade/facade.go new file mode 100644 index 00000000..37fd8af4 --- /dev/null +++ b/facade/facade.go @@ -0,0 +1,35 @@ +package facade + +import ( + "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-es-indexer-go/core" +) + +type metricsFacade struct { + statusMetrics core.StatusMetricsHandler +} + +func NewMetricsFacade(statusMetrics core.StatusMetricsHandler) (*metricsFacade, error) { + if check.IfNil(statusMetrics) { + return nil, core.ErrNilMetricsHandler + } + + return &metricsFacade{ + statusMetrics: statusMetrics, + }, nil +} + +// GetMetrics will return metrics in json format +func (mf *metricsFacade) GetMetrics() map[string]interface{} { + return mf.statusMetrics.GetMetrics() +} + +// GetMetricsForPrometheus will return metrics in prometheus format +func (mf *metricsFacade) GetMetricsForPrometheus() string { + return mf.statusMetrics.GetMetricsForPrometheus() +} + +// IsInterfaceNil returns true if there is no value under the interface +func (mf *metricsFacade) IsInterfaceNil() bool { + return mf == nil +} diff --git a/factory/webServerFactory.go b/factory/webServerFactory.go new file mode 100644 index 00000000..47f6c90e --- /dev/null +++ b/factory/webServerFactory.go @@ -0,0 +1,21 @@ +package factory + +import ( + "github.com/multiversx/mx-chain-es-indexer-go/api/gin" + "github.com/multiversx/mx-chain-es-indexer-go/config" + "github.com/multiversx/mx-chain-es-indexer-go/core" + "github.com/multiversx/mx-chain-es-indexer-go/facade" +) + +func CreateWebServer(apiConfig config.ApiRoutesConfig, statusMetricsHandler core.StatusMetricsHandler) (core.WebServerHandler, error) { + metricsFacade, err := facade.NewMetricsFacade(statusMetricsHandler) + if err != nil { + return nil, err + } + + args := gin.ArgsWebServer{ + Facade: metricsFacade, + ApiConfig: apiConfig, + } + return gin.NewWebServer(args) +} diff --git a/go.mod b/go.mod index b89e9e5e..c3c14011 100644 --- a/go.mod +++ b/go.mod @@ -4,35 +4,56 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 + github.com/gin-contrib/cors v1.4.0 + github.com/gin-gonic/gin v1.9.0 github.com/multiversx/mx-chain-communication-go v1.0.3 github.com/multiversx/mx-chain-core-go v1.2.9 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.1 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.4 github.com/tidwall/gjson v1.14.0 github.com/urfave/cli v1.22.10 ) require ( github.com/btcsuite/btcd/btcutil v1.1.3 // indirect + github.com/bytedance/sonic v1.9.1 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/denisbrodbeck/machineid v1.0.1 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.14.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.8 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/leodido/go-urn v1.2.4 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/pelletier/go-toml v1.9.3 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.0.1 // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect - golang.org/x/crypto v0.3.0 // indirect - golang.org/x/sys v0.2.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.11 // indirect + golang.org/x/arch v0.3.0 // indirect + golang.org/x/crypto v0.9.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d98700c6..b5541e4a 100644 --- a/go.sum +++ b/go.sum @@ -83,11 +83,18 @@ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= +github.com/bytedance/sonic v1.8.0/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= +github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= +github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -109,6 +116,7 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -142,7 +150,16 @@ github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g= +github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= +github.com/gin-gonic/gin v1.9.0 h1:OjyFBKICoexlu99ctXNR2gg+c5pKrKMuyjgARg9qeY8= +github.com/gin-gonic/gin v1.9.0/go.mod h1:W1Me9+hsUSyj3CePGrd1/QrKJMSJ1Tu/0hFEH89961k= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= @@ -160,8 +177,25 @@ github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KE github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= +github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s= +github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= +github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -199,8 +233,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -312,6 +347,7 @@ github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlT github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -328,6 +364,8 @@ github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= @@ -335,12 +373,17 @@ github.com/koron/go-ssdp v0.0.3/go.mod h1:b2MxI6yh02pKrsyNoQUsk4+YNikaGhe4894J+Q github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= @@ -407,6 +450,9 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= @@ -426,9 +472,11 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -511,6 +559,11 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhM github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -542,6 +595,9 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= @@ -603,8 +659,11 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= @@ -614,7 +673,15 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go v1.2.7 h1:qYhyWUUd6WbiM+C6JZAUkIJt/1WrjzNHY9+KCIjVqTo= +github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/ugorji/go/codec v1.2.9/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= +github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.10 h1:p8Fspmz3iTctJstry1PYS3HVdllxnEzTEsgIgtxTrCk= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -662,6 +729,9 @@ go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= +golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -682,10 +752,14 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -719,6 +793,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -773,6 +848,12 @@ golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -796,6 +877,7 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -858,6 +940,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -867,12 +950,20 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -880,7 +971,13 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -941,6 +1038,7 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1034,8 +1132,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1071,6 +1170,7 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= diff --git a/metrics/statusMetrics.go b/metrics/statusMetrics.go new file mode 100644 index 00000000..4e1b71a6 --- /dev/null +++ b/metrics/statusMetrics.go @@ -0,0 +1,59 @@ +package metrics + +import ( + "strings" + "sync" +) + +const ( + numWSConnections = "num_ws_connections" + numIndexingErrors = "num_indexing_errors" +) + +type statusMetrics struct { + metrics map[string]interface{} + mutEndpointsMetrics sync.RWMutex +} + +// NewStatusMetrics will return an instance of the statusMetrics +func NewStatusMetrics() *statusMetrics { + return &statusMetrics{ + metrics: make(map[string]interface{}), + } +} + +// GetMetrics returns the metrics map +func (sm *statusMetrics) GetMetrics() map[string]interface{} { + sm.mutEndpointsMetrics.RLock() + defer sm.mutEndpointsMetrics.RUnlock() + + return sm.getAll() +} + +// GetMetricsForPrometheus returns the metrics in a prometheus format +func (sm *statusMetrics) GetMetricsForPrometheus() string { + sm.mutEndpointsMetrics.RLock() + defer sm.mutEndpointsMetrics.RUnlock() + + //metricsMap := sm.getAll() + + stringBuilder := strings.Builder{} + + // TODO populate with metrics + + return stringBuilder.String() +} + +func (sm *statusMetrics) getAll() map[string]interface{} { + newMap := make(map[string]interface{}) + for key, value := range sm.metrics { + newMap[key] = value + } + + return newMap +} + +// IsInterfaceNil returns true if there is no value under the interface +func (sm *statusMetrics) IsInterfaceNil() bool { + return sm == nil +} From 276bf8d35eb98590caa35ccc000c709e077be7bf Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 14 Jul 2023 12:03:55 +0300 Subject: [PATCH 114/189] fix linter --- api/gin/webServer.go | 2 ++ api/groups/statusGroup.go | 4 +--- metrics/statusMetrics.go | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api/gin/webServer.go b/api/gin/webServer.go index 36e2dd62..23218b7f 100644 --- a/api/gin/webServer.go +++ b/api/gin/webServer.go @@ -65,6 +65,8 @@ func (ws *webServer) StartHttpServer() error { return err } + ws.registerRoutes(engine) + s := &http.Server{Addr: apiInterface, Handler: engine} log.Debug("creating gin web sever", "interface", apiInterface) ws.httpServer, err = NewHttpServer(s) diff --git a/api/groups/statusGroup.go b/api/groups/statusGroup.go index 36215f05..4f9e15f0 100644 --- a/api/groups/statusGroup.go +++ b/api/groups/statusGroup.go @@ -3,7 +3,6 @@ package groups import ( "fmt" "net/http" - "sync" "github.com/gin-gonic/gin" "github.com/multiversx/mx-chain-core-go/core/check" @@ -18,8 +17,7 @@ const ( type statusGroup struct { *baseGroup - facade shared.FacadeHandler - mutFacade sync.RWMutex + facade shared.FacadeHandler } // NewStatusGroup returns a new instance of status group diff --git a/metrics/statusMetrics.go b/metrics/statusMetrics.go index 4e1b71a6..4fbdadfd 100644 --- a/metrics/statusMetrics.go +++ b/metrics/statusMetrics.go @@ -5,10 +5,10 @@ import ( "sync" ) -const ( - numWSConnections = "num_ws_connections" - numIndexingErrors = "num_indexing_errors" -) +//const ( +// numWSConnections = "num_ws_connections" +// numIndexingErrors = "num_indexing_errors" +//) type statusMetrics struct { metrics map[string]interface{} From a630bdc7efb2df05ce6af5e7780c22e0c39e3cfc Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 14 Jul 2023 14:23:30 +0300 Subject: [PATCH 115/189] fixes after first review --- api/gin/webServer.go | 3 ++- core/interface.go | 1 + facade/facade.go | 1 + factory/webServerFactory.go | 1 + metrics/statusMetrics.go | 4 ++-- 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/api/gin/webServer.go b/api/gin/webServer.go index 23218b7f..469736bd 100644 --- a/api/gin/webServer.go +++ b/api/gin/webServer.go @@ -16,6 +16,7 @@ const ( webServerOffString = "off" ) +// ArgsWebServer holds the arguments needed for a webServer type ArgsWebServer struct { Facade shared.FacadeHandler ApiConfig config.ApiRoutesConfig @@ -115,7 +116,7 @@ func (ws *webServer) Close() error { err = fmt.Errorf("%w while closing the http server in gin/webServer", err) } - return err + return nil } // IsInterfaceNil returns true if there is no value under the interface diff --git a/core/interface.go b/core/interface.go index 997b6523..ff5cda27 100644 --- a/core/interface.go +++ b/core/interface.go @@ -7,6 +7,7 @@ type StatusMetricsHandler interface { IsInterfaceNil() bool } +// WebServerHandler defines the behavior of a component that handles the web server type WebServerHandler interface { StartHttpServer() error Close() error diff --git a/facade/facade.go b/facade/facade.go index 37fd8af4..1394235b 100644 --- a/facade/facade.go +++ b/facade/facade.go @@ -9,6 +9,7 @@ type metricsFacade struct { statusMetrics core.StatusMetricsHandler } +// NewMetricsFacade will create a new instance of metricsFacade func NewMetricsFacade(statusMetrics core.StatusMetricsHandler) (*metricsFacade, error) { if check.IfNil(statusMetrics) { return nil, core.ErrNilMetricsHandler diff --git a/factory/webServerFactory.go b/factory/webServerFactory.go index 47f6c90e..14830649 100644 --- a/factory/webServerFactory.go +++ b/factory/webServerFactory.go @@ -7,6 +7,7 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/facade" ) +// CreateWebServer will create a new instance of core.WebServerHandler func CreateWebServer(apiConfig config.ApiRoutesConfig, statusMetricsHandler core.StatusMetricsHandler) (core.WebServerHandler, error) { metricsFacade, err := facade.NewMetricsFacade(statusMetricsHandler) if err != nil { diff --git a/metrics/statusMetrics.go b/metrics/statusMetrics.go index 4fbdadfd..ca90cc28 100644 --- a/metrics/statusMetrics.go +++ b/metrics/statusMetrics.go @@ -27,7 +27,7 @@ func (sm *statusMetrics) GetMetrics() map[string]interface{} { sm.mutEndpointsMetrics.RLock() defer sm.mutEndpointsMetrics.RUnlock() - return sm.getAll() + return sm.getAllUnprotected() } // GetMetricsForPrometheus returns the metrics in a prometheus format @@ -44,7 +44,7 @@ func (sm *statusMetrics) GetMetricsForPrometheus() string { return stringBuilder.String() } -func (sm *statusMetrics) getAll() map[string]interface{} { +func (sm *statusMetrics) getAllUnprotected() map[string]interface{} { newMap := make(map[string]interface{}) for key, value := range sm.metrics { newMap[key] = value From c430ec7cf6ae9a512fb975b155c969b295f13821 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 14 Jul 2023 14:35:46 +0300 Subject: [PATCH 116/189] small fix --- api/gin/webServer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/gin/webServer.go b/api/gin/webServer.go index 469736bd..656ad5f4 100644 --- a/api/gin/webServer.go +++ b/api/gin/webServer.go @@ -113,7 +113,7 @@ func (ws *webServer) Close() error { ws.Unlock() if err != nil { - err = fmt.Errorf("%w while closing the http server in gin/webServer", err) + return fmt.Errorf("%w while closing the http server in gin/webServer", err) } return nil From 0298953c3e239b0ebcb5e1664e7105a1c2a4beb8 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 17 Jul 2023 11:45:11 +0300 Subject: [PATCH 117/189] fixes after second review --- api/gin/httpServer.go | 2 +- cmd/elasticindexer/config/api.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/gin/httpServer.go b/api/gin/httpServer.go index 18ed0bf8..1f7d935f 100644 --- a/api/gin/httpServer.go +++ b/api/gin/httpServer.go @@ -9,7 +9,7 @@ import ( logger "github.com/multiversx/mx-chain-logger-go" ) -var log = logger.GetOrCreate("api") +var log = logger.GetOrCreate("api/gin") // ErrNilHttpServer signals that a nil http server has been provided var ErrNilHttpServer = errors.New("nil http server") diff --git a/cmd/elasticindexer/config/api.toml b/cmd/elasticindexer/config/api.toml index ececfc1c..dd41e8f5 100644 --- a/cmd/elasticindexer/config/api.toml +++ b/cmd/elasticindexer/config/api.toml @@ -5,5 +5,5 @@ rest-api-interface = ":8080" [api-packages.status] routes = [ { name = "/metrics", open = true }, - { name = "/prometheus-metrics", open = true }, - ] \ No newline at end of file + { name = "/prometheus-metrics", open = true } + ] From 045ebb851a8e3f3662047831bf61f7bf43b375a3 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 19 Jul 2023 10:41:45 +0300 Subject: [PATCH 118/189] small refactor --- process/elasticproc/miniblocks/miniblocksProcessor.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/process/elasticproc/miniblocks/miniblocksProcessor.go b/process/elasticproc/miniblocks/miniblocksProcessor.go index b7acefe7..05234be5 100644 --- a/process/elasticproc/miniblocks/miniblocksProcessor.go +++ b/process/elasticproc/miniblocks/miniblocksProcessor.go @@ -121,18 +121,13 @@ func (mp *miniblocksProcessor) setFieldsMBIntraShardAndCrossFromMe( case constructionState == int32(block.Final) && processingType == block.Processed.String(): dbMiniblock.ReceiverBlockHash = headerHash dbMiniblock.ProcessingTypeOnDestination = processingType - case constructionState == 0 && processingType == "": - dbMiniblock.SenderBlockHash = headerHash - dbMiniblock.ReceiverBlockHash = headerHash - dbMiniblock.ProcessingTypeOnSource = block.Normal.String() - dbMiniblock.ProcessingTypeOnDestination = block.Normal.String() } } func (mp *miniblocksProcessor) computeProcessingTypeAndConstructionState(mbIndex int, header coreData.HeaderHandler) (string, int32) { miniblockHeaders := header.GetMiniBlockHeaderHandlers() if len(miniblockHeaders) <= mbIndex { - return "", 0 + return block.Normal.String(), int32(block.Final) } processingType := miniblockHeaders[mbIndex].GetProcessingType() From ead6adb5ba8f916d7ae3c09eb1dc15ff64fccc93 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 19 Jul 2023 13:51:51 +0300 Subject: [PATCH 119/189] add indexing metrics --- api/shared/interface.go | 3 ++- cmd/elasticindexer/main.go | 6 ++--- core/interface.go | 5 +++- core/requests.go | 11 ++++++++ facade/facade.go | 2 +- factory/wsIndexerFactory.go | 10 +++++-- go.mod | 2 +- go.sum | 4 +-- metrics/statusMetrics.go | 50 +++++++++++++++++++++++----------- process/wsindexer/indexer.go | 52 +++++++++++++++++++++++++++++------- scripts/observers/.env | 2 +- scripts/observers/config.py | 17 ++++++++++++ scripts/observers/utils.py | 3 +++ 13 files changed, 131 insertions(+), 36 deletions(-) create mode 100644 core/requests.go diff --git a/api/shared/interface.go b/api/shared/interface.go index 8ff5fe96..3b730258 100644 --- a/api/shared/interface.go +++ b/api/shared/interface.go @@ -3,6 +3,7 @@ package shared import ( "github.com/gin-gonic/gin" "github.com/multiversx/mx-chain-es-indexer-go/config" + "github.com/multiversx/mx-chain-es-indexer-go/core" ) // GroupHandler defines the actions needed to be performed by a gin API group @@ -16,7 +17,7 @@ type GroupHandler interface { // FacadeHandler defines all the methods that a facade should implement type FacadeHandler interface { - GetMetrics() map[string]interface{} + GetMetrics() map[string]*core.IndexTopicMetricsResponse GetMetricsForPrometheus() string IsInterfaceNil() bool } diff --git a/cmd/elasticindexer/main.go b/cmd/elasticindexer/main.go index 9794b65f..cea8460b 100644 --- a/cmd/elasticindexer/main.go +++ b/cmd/elasticindexer/main.go @@ -86,7 +86,8 @@ func startIndexer(ctx *cli.Context) error { } importDBMode := ctx.GlobalBool(importDB.Name) - wsHost, err := factory.CreateWsIndexer(cfg, clusterCfg, importDBMode) + statusMetrics := metrics.NewStatusMetrics() + wsHost, err := factory.CreateWsIndexer(cfg, clusterCfg, importDBMode, statusMetrics) if err != nil { return fmt.Errorf("%w while creating the indexer", err) } @@ -96,8 +97,7 @@ func startIndexer(ctx *cli.Context) error { return fmt.Errorf("%w while loading the api config file", err) } - statusMetric := metrics.NewStatusMetrics() - webServer, err := factory.CreateWebServer(apiConfig, statusMetric) + webServer, err := factory.CreateWebServer(apiConfig, statusMetrics) if err != nil { return fmt.Errorf("%w while creating the web server", err) } diff --git a/core/interface.go b/core/interface.go index ff5cda27..527c09ea 100644 --- a/core/interface.go +++ b/core/interface.go @@ -1,8 +1,11 @@ package core +import "time" + // StatusMetricsHandler defines the behavior of a component that handles status metrics type StatusMetricsHandler interface { - GetMetrics() map[string]interface{} + AddIndexingData(topic string, shardID uint32, duration time.Duration, gotError bool) + GetMetrics() map[string]*IndexTopicMetricsResponse GetMetricsForPrometheus() string IsInterfaceNil() bool } diff --git a/core/requests.go b/core/requests.go new file mode 100644 index 00000000..51b376b2 --- /dev/null +++ b/core/requests.go @@ -0,0 +1,11 @@ +package core + +import "time" + +// IndexTopicMetricsResponse defines the response for status metrics endpoint +type IndexTopicMetricsResponse struct { + NumIndexingOperations uint64 `json:"num_indexing_operations"` + NumTotalErrors uint64 `json:"num_total_errors"` + LastIndexingTime time.Duration `json:"last_indexing_time"` + TotalIndexingTime time.Duration `json:"total_indexing_time"` +} diff --git a/facade/facade.go b/facade/facade.go index 1394235b..ecca0571 100644 --- a/facade/facade.go +++ b/facade/facade.go @@ -21,7 +21,7 @@ func NewMetricsFacade(statusMetrics core.StatusMetricsHandler) (*metricsFacade, } // GetMetrics will return metrics in json format -func (mf *metricsFacade) GetMetrics() map[string]interface{} { +func (mf *metricsFacade) GetMetrics() map[string]*core.IndexTopicMetricsResponse { return mf.statusMetrics.GetMetrics() } diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index 0d4a3278..a71c2319 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -8,6 +8,7 @@ import ( "github.com/multiversx/mx-chain-core-go/marshal" factoryMarshaller "github.com/multiversx/mx-chain-core-go/marshal/factory" "github.com/multiversx/mx-chain-es-indexer-go/config" + "github.com/multiversx/mx-chain-es-indexer-go/core" "github.com/multiversx/mx-chain-es-indexer-go/process/factory" "github.com/multiversx/mx-chain-es-indexer-go/process/wsindexer" logger "github.com/multiversx/mx-chain-logger-go" @@ -16,7 +17,7 @@ import ( var log = logger.GetOrCreate("elasticindexer") // CreateWsIndexer will create a new instance of wsindexer.WSClient -func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig, importDB bool) (wsindexer.WSClient, error) { +func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig, importDB bool, statusMetrics core.StatusMetricsHandler) (wsindexer.WSClient, error) { wsMarshaller, err := factoryMarshaller.NewMarshalizer(clusterCfg.Config.WebSocket.DataMarshallerType) if err != nil { return nil, err @@ -27,7 +28,12 @@ func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig, importD return nil, err } - indexer, err := wsindexer.NewIndexer(wsMarshaller, dataIndexer) + args := wsindexer.ArgsIndexer{ + Marshaller: wsMarshaller, + DataIndexer: dataIndexer, + StatusMetrics: statusMetrics, + } + indexer, err := wsindexer.NewIndexer(args) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index c3c14011..f42911a5 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.0 github.com/multiversx/mx-chain-communication-go v1.0.3 - github.com/multiversx/mx-chain-core-go v1.2.9 + github.com/multiversx/mx-chain-core-go v1.2.10-0.20230719093633-30f63c3c1d7e github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.1 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index b5541e4a..0cc2bad3 100644 --- a/go.sum +++ b/go.sum @@ -524,8 +524,8 @@ github.com/multiversx/mx-chain-communication-go v1.0.3/go.mod h1:7oTI77XfWmRWwVE github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.5/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= -github.com/multiversx/mx-chain-core-go v1.2.9 h1:uXrmzhKxEe5sztUTs1am3ILrxuBwNQQOESOKBlyrlLE= -github.com/multiversx/mx-chain-core-go v1.2.9/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= +github.com/multiversx/mx-chain-core-go v1.2.10-0.20230719093633-30f63c3c1d7e h1:8bGfoQS8ViL/12gImiq3ywiaXAEsvNkh51d2xUnnZdE= +github.com/multiversx/mx-chain-core-go v1.2.10-0.20230719093633-30f63c3c1d7e/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-crypto-go v1.2.6/go.mod h1:rOj0Rr19HTOYt9YTeym7RKxlHt91NXln3LVKjHKVmA0= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= diff --git a/metrics/statusMetrics.go b/metrics/statusMetrics.go index ca90cc28..a75c128e 100644 --- a/metrics/statusMetrics.go +++ b/metrics/statusMetrics.go @@ -1,39 +1,59 @@ package metrics import ( + "fmt" "strings" "sync" -) + "time" -//const ( -// numWSConnections = "num_ws_connections" -// numIndexingErrors = "num_indexing_errors" -//) + "github.com/multiversx/mx-chain-es-indexer-go/core" +) type statusMetrics struct { - metrics map[string]interface{} - mutEndpointsMetrics sync.RWMutex + metrics map[string]*core.IndexTopicMetricsResponse + mut sync.RWMutex } // NewStatusMetrics will return an instance of the statusMetrics func NewStatusMetrics() *statusMetrics { return &statusMetrics{ - metrics: make(map[string]interface{}), + metrics: make(map[string]*core.IndexTopicMetricsResponse), + } +} + +// AddIndexingData will add the indexing data for the give topic +func (sm *statusMetrics) AddIndexingData(topic string, shardID uint32, duration time.Duration, gotError bool) { + sm.mut.Lock() + defer sm.mut.Unlock() + + key := fmt.Sprintf("%s_%d", topic, shardID) + + _, found := sm.metrics[key] + if !found { + sm.metrics[key] = &core.IndexTopicMetricsResponse{} + } + + sm.metrics[key].NumIndexingOperations++ + sm.metrics[key].TotalIndexingTime += duration + sm.metrics[key].LastIndexingTime = duration + + if gotError == true { + sm.metrics[key].NumTotalErrors++ } } // GetMetrics returns the metrics map -func (sm *statusMetrics) GetMetrics() map[string]interface{} { - sm.mutEndpointsMetrics.RLock() - defer sm.mutEndpointsMetrics.RUnlock() +func (sm *statusMetrics) GetMetrics() map[string]*core.IndexTopicMetricsResponse { + sm.mut.RLock() + defer sm.mut.RUnlock() return sm.getAllUnprotected() } // GetMetricsForPrometheus returns the metrics in a prometheus format func (sm *statusMetrics) GetMetricsForPrometheus() string { - sm.mutEndpointsMetrics.RLock() - defer sm.mutEndpointsMetrics.RUnlock() + sm.mut.RLock() + defer sm.mut.RUnlock() //metricsMap := sm.getAll() @@ -44,8 +64,8 @@ func (sm *statusMetrics) GetMetricsForPrometheus() string { return stringBuilder.String() } -func (sm *statusMetrics) getAllUnprotected() map[string]interface{} { - newMap := make(map[string]interface{}) +func (sm *statusMetrics) getAllUnprotected() map[string]*core.IndexTopicMetricsResponse { + newMap := make(map[string]*core.IndexTopicMetricsResponse) for key, value := range sm.metrics { newMap[key] = value } diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index 5dd50ab8..9551f33f 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -2,10 +2,12 @@ package wsindexer import ( "errors" + "time" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-es-indexer-go/core" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" logger "github.com/multiversx/mx-chain-logger-go" ) @@ -15,24 +17,35 @@ var ( errNilDataIndexer = errors.New("nil data indexer") ) +type ArgsIndexer struct { + Marshaller marshal.Marshalizer + DataIndexer DataIndexer + StatusMetrics core.StatusMetricsHandler +} + type indexer struct { - marshaller marshal.Marshalizer - di DataIndexer - actions map[string]func(marshalledData []byte) error + marshaller marshal.Marshalizer + di DataIndexer + statusMetrics core.StatusMetricsHandler + actions map[string]func(marshalledData []byte) error } // NewIndexer will create a new instance of *indexer -func NewIndexer(marshaller marshal.Marshalizer, dataIndexer DataIndexer) (*indexer, error) { - if check.IfNil(marshaller) { +func NewIndexer(args ArgsIndexer) (*indexer, error) { + if check.IfNil(args.Marshaller) { return nil, dataindexer.ErrNilMarshalizer } - if check.IfNil(dataIndexer) { + if check.IfNil(args.DataIndexer) { return nil, errNilDataIndexer } + if check.IfNil(args.StatusMetrics) { + return nil, core.ErrNilMetricsHandler + } payloadIndexer := &indexer{ - marshaller: marshaller, - di: dataIndexer, + marshaller: args.Marshaller, + di: args.DataIndexer, + statusMetrics: args.StatusMetrics, } payloadIndexer.initActionsMap() @@ -61,7 +74,18 @@ func (i *indexer) ProcessPayload(payload []byte, topic string) error { return nil } - return payloadTypeAction(payload) + shardID, err := i.getShardID(payload) + if err != nil { + log.Warn("indexer.ProcessPayload: cannot get shardID from payload", "error", err) + } + + start := time.Now() + err = payloadTypeAction(payload) + duration := time.Since(start) + + i.statusMetrics.AddIndexingData(topic, shardID, duration, err != nil) + + return err } func (i *indexer) saveBlock(marshalledData []byte) error { @@ -147,3 +171,13 @@ func (i *indexer) Close() error { func (i *indexer) IsInterfaceNil() bool { return i == nil } + +func (i *indexer) getShardID(payload []byte) (uint32, error) { + shard := &outport.Shard{} + err := i.marshaller.Unmarshal(shard, payload) + if err != nil { + return 0, err + } + + return shard.ShardID, nil +} diff --git a/scripts/observers/.env b/scripts/observers/.env index f888c87d..da050862 100644 --- a/scripts/observers/.env +++ b/scripts/observers/.env @@ -2,7 +2,7 @@ MX_CHAIN_DEPLOY_GO_URL=https://github.com/multiversx/mx-chain-deploy-go MX_CHAIN_PROXY_URL=https://github.com/multiversx/mx-chain-proxy-go NODE_GO_URL="https://github.com/multiversx/mx-chain-go" -NODE_GO_BRANCH="rc/v1.6.0" +NODE_GO_BRANCH="indexer-with-metrics" WORKING_DIRECTORY="IndexerObservers" OBSERVER_DIR_PREFIX="observer_shard_" diff --git a/scripts/observers/config.py b/scripts/observers/config.py index 0477054f..e52bd98d 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -31,6 +31,23 @@ def update_toml_indexer(path, shard_id): toml.dump(prefs_data, f) f.close() + # api.toml + path_api = path / "api.toml" + api_data = toml.load(str(path_api)) + + api_port = API_PORT_BASE + shard_id + api_meta_port = API_META_PORT + if is_indexer_server: + api_port = API_PORT_BASE + api_meta_port = API_PORT_BASE + if shard_id != METACHAIN: + api_data['rest-api-interface'] = f":{api_port}" + else: + api_data['rest-api-interface'] = f":{api_meta_port}" + f = open(path_api, 'w') + toml.dump(api_data, f) + f.close() + def update_toml_node(path, shard_id): # prefs.toml diff --git a/scripts/observers/utils.py b/scripts/observers/utils.py index 8de9661e..13ce6088 100644 --- a/scripts/observers/utils.py +++ b/scripts/observers/utils.py @@ -8,6 +8,9 @@ WS_METACHAIN_PORT = WS_PORT_BASE + 50 MAX_NUM_OF_SHARDS = 3 +API_PORT_BASE = 8081 +API_META_PORT = API_PORT_BASE + 50 + def get_working_dir(): working_dir_var = os.getenv('WORKING_DIRECTORY') From b5c03d4f10ae0e378ac6c24a3cb54bf2fdf058eb Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 19 Jul 2023 14:17:51 +0300 Subject: [PATCH 120/189] new go mod --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f42911a5..a0bee4f8 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.0 github.com/multiversx/mx-chain-communication-go v1.0.3 - github.com/multiversx/mx-chain-core-go v1.2.10-0.20230719093633-30f63c3c1d7e + github.com/multiversx/mx-chain-core-go v1.2.10-0.20230719111700-ba37de156180 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.1 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index 0cc2bad3..6ac7bf22 100644 --- a/go.sum +++ b/go.sum @@ -524,8 +524,8 @@ github.com/multiversx/mx-chain-communication-go v1.0.3/go.mod h1:7oTI77XfWmRWwVE github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.5/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= -github.com/multiversx/mx-chain-core-go v1.2.10-0.20230719093633-30f63c3c1d7e h1:8bGfoQS8ViL/12gImiq3ywiaXAEsvNkh51d2xUnnZdE= -github.com/multiversx/mx-chain-core-go v1.2.10-0.20230719093633-30f63c3c1d7e/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= +github.com/multiversx/mx-chain-core-go v1.2.10-0.20230719111700-ba37de156180 h1:SZkaCv6L/3EBEELkAr+F/HCCrYgia/OLVgFHFQSggJw= +github.com/multiversx/mx-chain-core-go v1.2.10-0.20230719111700-ba37de156180/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-crypto-go v1.2.6/go.mod h1:rOj0Rr19HTOYt9YTeym7RKxlHt91NXln3LVKjHKVmA0= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= From 9d3dd3120f37cd6146a4fa13cc22ea480d155b31 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 19 Jul 2023 15:05:54 +0300 Subject: [PATCH 121/189] fix linter --- metrics/statusMetrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/statusMetrics.go b/metrics/statusMetrics.go index a75c128e..971a6542 100644 --- a/metrics/statusMetrics.go +++ b/metrics/statusMetrics.go @@ -37,7 +37,7 @@ func (sm *statusMetrics) AddIndexingData(topic string, shardID uint32, duration sm.metrics[key].TotalIndexingTime += duration sm.metrics[key].LastIndexingTime = duration - if gotError == true { + if gotError { sm.metrics[key].NumTotalErrors++ } } From ed223f1b7ea8b29c63279abde9fa80eb5859bfe5 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 19 Jul 2023 17:21:34 +0300 Subject: [PATCH 122/189] request metrics and refactoring --- api/shared/interface.go | 4 +-- client/elasticClient.go | 12 +++++++++ client/transport/transport.go | 45 +++++++++++++++++++++++++++++++ core/interface.go | 9 ++++--- core/request/requests.go | 19 +++++++++++++ core/requests.go | 11 -------- facade/facade.go | 3 ++- factory/wsIndexerFactory.go | 11 ++++++-- metrics/dtos.go | 11 ++++++++ metrics/statusMetrics.go | 33 +++++++++++------------ process/factory/indexerFactory.go | 36 +++++++++++++++++++------ process/wsindexer/indexer.go | 10 ++++++- 12 files changed, 158 insertions(+), 46 deletions(-) create mode 100644 client/transport/transport.go create mode 100644 core/request/requests.go delete mode 100644 core/requests.go create mode 100644 metrics/dtos.go diff --git a/api/shared/interface.go b/api/shared/interface.go index 3b730258..98aea754 100644 --- a/api/shared/interface.go +++ b/api/shared/interface.go @@ -3,7 +3,7 @@ package shared import ( "github.com/gin-gonic/gin" "github.com/multiversx/mx-chain-es-indexer-go/config" - "github.com/multiversx/mx-chain-es-indexer-go/core" + "github.com/multiversx/mx-chain-es-indexer-go/core/request" ) // GroupHandler defines the actions needed to be performed by a gin API group @@ -17,7 +17,7 @@ type GroupHandler interface { // FacadeHandler defines all the methods that a facade should implement type FacadeHandler interface { - GetMetrics() map[string]*core.IndexTopicMetricsResponse + GetMetrics() map[string]*request.MetricsResponse GetMetricsForPrometheus() string IsInterfaceNil() bool } diff --git a/client/elasticClient.go b/client/elasticClient.go index dc56be6e..032bc4ff 100644 --- a/client/elasticClient.go +++ b/client/elasticClient.go @@ -10,6 +10,7 @@ import ( "github.com/elastic/go-elasticsearch/v7" "github.com/elastic/go-elasticsearch/v7/esapi" + "github.com/multiversx/mx-chain-es-indexer-go/core/request" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" logger "github.com/multiversx/mx-chain-logger-go" @@ -108,10 +109,15 @@ func (ec *elasticClient) DoBulkRequest(buff *bytes.Buffer, index string) error { reader := bytes.NewReader(buff.Bytes()) options := make([]func(*esapi.BulkRequest), 0) + topicValue := fmt.Sprintf("%s_common", request.BulkTopic) if index != "" { + topicValue = fmt.Sprintf("%s_%s", request.BulkTopic, index) options = append(options, ec.client.Bulk.WithIndex(index)) } + valueCtx := context.WithValue(context.Background(), request.ContextKey, topicValue) + options = append(options, ec.client.Bulk.WithContext(valueCtx)) + res, err := ec.client.Bulk( reader, options..., @@ -133,9 +139,11 @@ func (ec *elasticClient) DoMultiGet(ids []string, index string, withSource bool, return err } + valueCtx := context.WithValue(context.Background(), request.ContextKey, request.GetTopic) res, err := ec.client.Mget( &body, ec.client.Mget.WithIndex(index), + ec.client.Mget.WithContext(valueCtx), ) if err != nil { log.Warn("elasticClient.DoMultiGet", @@ -159,11 +167,13 @@ func (ec *elasticClient) DoQueryRemove(index string, body *bytes.Buffer) error { log.Warn("elasticClient.doRefresh", "cannot do refresh", err.Error()) } + valueCtx := context.WithValue(context.Background(), request.ContextKey, request.RemoveTopic) res, err := ec.client.DeleteByQuery( []string{index}, body, ec.client.DeleteByQuery.WithIgnoreUnavailable(true), ec.client.DeleteByQuery.WithConflicts(esConflictsPolicy), + ec.client.DeleteByQuery.WithContext(valueCtx), ) if err != nil { @@ -332,9 +342,11 @@ func (ec *elasticClient) createAlias(alias string, index string) error { func (ec *elasticClient) UpdateByQuery(index string, buff *bytes.Buffer) error { reader := bytes.NewReader(buff.Bytes()) + valueCtx := context.WithValue(context.Background(), request.ContextKey, request.UpdateTopic) res, err := ec.client.UpdateByQuery( []string{index}, ec.client.UpdateByQuery.WithBody(reader), + ec.client.UpdateByQuery.WithContext(valueCtx), ) if err != nil { return err diff --git a/client/transport/transport.go b/client/transport/transport.go new file mode 100644 index 00000000..d70aefc5 --- /dev/null +++ b/client/transport/transport.go @@ -0,0 +1,45 @@ +package transport + +import ( + "fmt" + "net/http" + "time" + + "github.com/multiversx/mx-chain-es-indexer-go/core" + "github.com/multiversx/mx-chain-es-indexer-go/core/request" + "github.com/multiversx/mx-chain-es-indexer-go/metrics" +) + +type metricsTransport struct { + statusMetrics core.StatusMetricsHandler + transport http.RoundTripper +} + +func NewMetricsTransport(statusMetrics core.StatusMetricsHandler) (*metricsTransport, error) { + return &metricsTransport{ + statusMetrics: statusMetrics, + transport: http.DefaultTransport, + }, nil +} + +func (m *metricsTransport) RoundTrip(req *http.Request) (*http.Response, error) { + startTime := time.Now() + size := req.ContentLength + resp, err := m.transport.RoundTrip(req) + duration := time.Since(startTime) + + valueFromCtx := req.Context().Value(request.ContextKey) + if valueFromCtx == nil { + return resp, err + } + topic := fmt.Sprintf("%s", valueFromCtx) + + m.statusMetrics.AddIndexingData(metrics.ArgsAddIndexingData{ + GotError: err != nil, + MessageLen: uint64(size), + Topic: topic, + Duration: duration, + }) + + return resp, err +} diff --git a/core/interface.go b/core/interface.go index 527c09ea..b233db88 100644 --- a/core/interface.go +++ b/core/interface.go @@ -1,11 +1,14 @@ package core -import "time" +import ( + "github.com/multiversx/mx-chain-es-indexer-go/core/request" + "github.com/multiversx/mx-chain-es-indexer-go/metrics" +) // StatusMetricsHandler defines the behavior of a component that handles status metrics type StatusMetricsHandler interface { - AddIndexingData(topic string, shardID uint32, duration time.Duration, gotError bool) - GetMetrics() map[string]*IndexTopicMetricsResponse + AddIndexingData(args metrics.ArgsAddIndexingData) + GetMetrics() map[string]*request.MetricsResponse GetMetricsForPrometheus() string IsInterfaceNil() bool } diff --git a/core/request/requests.go b/core/request/requests.go new file mode 100644 index 00000000..ca4e6b7f --- /dev/null +++ b/core/request/requests.go @@ -0,0 +1,19 @@ +package request + +import "time" + +const ( + ContextKey = "key" + RemoveTopic = "req_remove" + GetTopic = "req_get" + BulkTopic = "req_bulk" + UpdateTopic = "req_update" +) + +// MetricsResponse defines the response for status metrics endpoint +type MetricsResponse struct { + OperationsCount uint64 `json:"operations_count"` + ErrorsCount uint64 `json:"errors_count"` + TotalIndexingTime time.Duration `json:"total_time"` + TotalData uint64 `json:"total_data"` +} diff --git a/core/requests.go b/core/requests.go deleted file mode 100644 index 51b376b2..00000000 --- a/core/requests.go +++ /dev/null @@ -1,11 +0,0 @@ -package core - -import "time" - -// IndexTopicMetricsResponse defines the response for status metrics endpoint -type IndexTopicMetricsResponse struct { - NumIndexingOperations uint64 `json:"num_indexing_operations"` - NumTotalErrors uint64 `json:"num_total_errors"` - LastIndexingTime time.Duration `json:"last_indexing_time"` - TotalIndexingTime time.Duration `json:"total_indexing_time"` -} diff --git a/facade/facade.go b/facade/facade.go index ecca0571..b11a4331 100644 --- a/facade/facade.go +++ b/facade/facade.go @@ -3,6 +3,7 @@ package facade import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-es-indexer-go/core" + "github.com/multiversx/mx-chain-es-indexer-go/core/request" ) type metricsFacade struct { @@ -21,7 +22,7 @@ func NewMetricsFacade(statusMetrics core.StatusMetricsHandler) (*metricsFacade, } // GetMetrics will return metrics in json format -func (mf *metricsFacade) GetMetrics() map[string]*core.IndexTopicMetricsResponse { +func (mf *metricsFacade) GetMetrics() map[string]*request.MetricsResponse { return mf.statusMetrics.GetMetrics() } diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index a71c2319..eaae1fdc 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -23,7 +23,7 @@ func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig, importD return nil, err } - dataIndexer, err := createDataIndexer(cfg, clusterCfg, wsMarshaller, importDB) + dataIndexer, err := createDataIndexer(cfg, clusterCfg, wsMarshaller, importDB, statusMetrics) if err != nil { return nil, err } @@ -51,7 +51,13 @@ func CreateWsIndexer(cfg config.Config, clusterCfg config.ClusterConfig, importD return host, nil } -func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig, wsMarshaller marshal.Marshalizer, importDB bool) (wsindexer.DataIndexer, error) { +func createDataIndexer( + cfg config.Config, + clusterCfg config.ClusterConfig, + wsMarshaller marshal.Marshalizer, + importDB bool, + statusMetrics core.StatusMetricsHandler, +) (wsindexer.DataIndexer, error) { marshaller, err := factoryMarshaller.NewMarshalizer(cfg.Config.Marshaller.Type) if err != nil { return nil, err @@ -83,6 +89,7 @@ func createDataIndexer(cfg config.Config, clusterCfg config.ClusterConfig, wsMar ValidatorPubkeyConverter: validatorPubkeyConverter, HeaderMarshaller: wsMarshaller, ImportDB: importDB, + StatusMetrics: statusMetrics, }) } diff --git a/metrics/dtos.go b/metrics/dtos.go new file mode 100644 index 00000000..52e950d6 --- /dev/null +++ b/metrics/dtos.go @@ -0,0 +1,11 @@ +package metrics + +import "time" + +// ArgsAddIndexingData holds all the data needed for indexing metrics +type ArgsAddIndexingData struct { + GotError bool + MessageLen uint64 + Topic string + Duration time.Duration +} diff --git a/metrics/statusMetrics.go b/metrics/statusMetrics.go index 971a6542..e2a3ee1f 100644 --- a/metrics/statusMetrics.go +++ b/metrics/statusMetrics.go @@ -1,49 +1,46 @@ package metrics import ( - "fmt" "strings" "sync" - "time" - "github.com/multiversx/mx-chain-es-indexer-go/core" + "github.com/multiversx/mx-chain-es-indexer-go/core/request" ) type statusMetrics struct { - metrics map[string]*core.IndexTopicMetricsResponse + metrics map[string]*request.MetricsResponse mut sync.RWMutex } // NewStatusMetrics will return an instance of the statusMetrics func NewStatusMetrics() *statusMetrics { return &statusMetrics{ - metrics: make(map[string]*core.IndexTopicMetricsResponse), + metrics: make(map[string]*request.MetricsResponse), } } // AddIndexingData will add the indexing data for the give topic -func (sm *statusMetrics) AddIndexingData(topic string, shardID uint32, duration time.Duration, gotError bool) { +func (sm *statusMetrics) AddIndexingData(args ArgsAddIndexingData) { sm.mut.Lock() defer sm.mut.Unlock() - key := fmt.Sprintf("%s_%d", topic, shardID) - - _, found := sm.metrics[key] + topic := args.Topic + _, found := sm.metrics[topic] if !found { - sm.metrics[key] = &core.IndexTopicMetricsResponse{} + sm.metrics[topic] = &request.MetricsResponse{} } - sm.metrics[key].NumIndexingOperations++ - sm.metrics[key].TotalIndexingTime += duration - sm.metrics[key].LastIndexingTime = duration + sm.metrics[topic].OperationsCount++ + sm.metrics[topic].TotalIndexingTime += args.Duration + sm.metrics[topic].TotalData += args.MessageLen - if gotError { - sm.metrics[key].NumTotalErrors++ + if args.GotError { + sm.metrics[topic].ErrorsCount++ } } // GetMetrics returns the metrics map -func (sm *statusMetrics) GetMetrics() map[string]*core.IndexTopicMetricsResponse { +func (sm *statusMetrics) GetMetrics() map[string]*request.MetricsResponse { sm.mut.RLock() defer sm.mut.RUnlock() @@ -64,8 +61,8 @@ func (sm *statusMetrics) GetMetricsForPrometheus() string { return stringBuilder.String() } -func (sm *statusMetrics) getAllUnprotected() map[string]*core.IndexTopicMetricsResponse { - newMap := make(map[string]*core.IndexTopicMetricsResponse) +func (sm *statusMetrics) getAllUnprotected() map[string]*request.MetricsResponse { + newMap := make(map[string]*request.MetricsResponse) for key, value := range sm.metrics { newMap[key] = value } diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index 2782b51d..5e298d53 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -14,7 +14,10 @@ import ( "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-es-indexer-go/client" "github.com/multiversx/mx-chain-es-indexer-go/client/logging" + "github.com/multiversx/mx-chain-es-indexer-go/client/transport" + indexerCore "github.com/multiversx/mx-chain-es-indexer-go/core" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/factory" logger "github.com/multiversx/mx-chain-logger-go" ) @@ -39,6 +42,7 @@ type ArgsIndexerFactory struct { Hasher hashing.Hasher AddressPubkeyConverter core.PubkeyConverter ValidatorPubkeyConverter core.PubkeyConverter + StatusMetrics indexerCore.StatusMetricsHandler } // NewIndexer will create a new instance of Indexer @@ -75,14 +79,7 @@ func retryBackOff(attempt int) time.Duration { } func createElasticProcessor(args ArgsIndexerFactory) (dataindexer.ElasticProcessor, error) { - databaseClient, err := client.NewElasticClient(elasticsearch.Config{ - Addresses: []string{args.Url}, - Username: args.UserName, - Password: args.Password, - Logger: &logging.CustomLogger{}, - RetryOnStatus: []int{http.StatusConflict}, - RetryBackoff: retryBackOff, - }) + databaseClient, err := createElasticClient(args) if err != nil { return nil, err } @@ -103,6 +100,29 @@ func createElasticProcessor(args ArgsIndexerFactory) (dataindexer.ElasticProcess return factory.CreateElasticProcessor(argsElasticProcFac) } +func createElasticClient(args ArgsIndexerFactory) (elasticproc.DatabaseClientHandler, error) { + argsEsClient := elasticsearch.Config{ + Addresses: []string{args.Url}, + Username: args.UserName, + Password: args.Password, + Logger: &logging.CustomLogger{}, + RetryOnStatus: []int{http.StatusConflict}, + RetryBackoff: retryBackOff, + } + + if args.StatusMetrics == nil { + return client.NewElasticClient(argsEsClient) + } + + transportMetrics, err := transport.NewMetricsTransport(args.StatusMetrics) + if err != nil { + return nil, err + } + argsEsClient.Transport = transportMetrics + + return client.NewElasticClient(argsEsClient) +} + func checkDataIndexerParams(arguments ArgsIndexerFactory) error { if check.IfNil(arguments.AddressPubkeyConverter) { return fmt.Errorf("%w when setting AddressPubkeyConverter in indexer", dataindexer.ErrNilPubkeyConverter) diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index 9551f33f..b72d22b4 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -2,12 +2,14 @@ package wsindexer import ( "errors" + "fmt" "time" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-es-indexer-go/core" + "github.com/multiversx/mx-chain-es-indexer-go/metrics" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" logger "github.com/multiversx/mx-chain-logger-go" ) @@ -83,7 +85,13 @@ func (i *indexer) ProcessPayload(payload []byte, topic string) error { err = payloadTypeAction(payload) duration := time.Since(start) - i.statusMetrics.AddIndexingData(topic, shardID, duration, err != nil) + topicKey := fmt.Sprintf("%s_%d", topic, shardID) + i.statusMetrics.AddIndexingData(metrics.ArgsAddIndexingData{ + GotError: err != nil, + MessageLen: uint64(len(payload)), + Topic: topicKey, + Duration: duration, + }) return err } From 2ce50c0b8eb8250672035d992ede7d2363671f62 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 20 Jul 2023 13:31:16 +0300 Subject: [PATCH 123/189] prometheus metrics --- client/elasticClient.go | 23 +++----- client/elasticClientScroll.go | 5 +- client/elasticClientScroll_test.go | 3 +- client/elasticClient_test.go | 5 +- core/request/requests.go | 31 +++++++++- go.mod | 7 +++ go.sum | 7 +++ metrics/prometheusMetrics.go | 42 ++++++++++++++ metrics/statusMetrics.go | 17 ++++-- mock/databaseWriterStub.go | 15 ++--- process/elasticproc/elasticProcessor.go | 77 +++++++++++++++---------- process/elasticproc/interface.go | 15 ++--- process/elasticproc/updateTokenType.go | 18 +++--- 13 files changed, 191 insertions(+), 74 deletions(-) create mode 100644 metrics/prometheusMetrics.go diff --git a/client/elasticClient.go b/client/elasticClient.go index 032bc4ff..f5c39728 100644 --- a/client/elasticClient.go +++ b/client/elasticClient.go @@ -95,8 +95,8 @@ func (ec *elasticClient) CheckAndCreateAlias(alias string, indexName string) err } // DoRequest will do a request to elastic server -func (ec *elasticClient) DoRequest(req *esapi.IndexRequest) error { - res, err := req.Do(context.Background(), ec.client) +func (ec *elasticClient) DoRequest(req *esapi.IndexRequest, ctx context.Context) error { + res, err := req.Do(ctx, ec.client) if err != nil { return err } @@ -105,18 +105,15 @@ func (ec *elasticClient) DoRequest(req *esapi.IndexRequest) error { } // DoBulkRequest will do a bulk of request to elastic server -func (ec *elasticClient) DoBulkRequest(buff *bytes.Buffer, index string) error { +func (ec *elasticClient) DoBulkRequest(buff *bytes.Buffer, index string, ctx context.Context) error { reader := bytes.NewReader(buff.Bytes()) options := make([]func(*esapi.BulkRequest), 0) - topicValue := fmt.Sprintf("%s_common", request.BulkTopic) if index != "" { - topicValue = fmt.Sprintf("%s_%s", request.BulkTopic, index) options = append(options, ec.client.Bulk.WithIndex(index)) } - valueCtx := context.WithValue(context.Background(), request.ContextKey, topicValue) - options = append(options, ec.client.Bulk.WithContext(valueCtx)) + options = append(options, ec.client.Bulk.WithContext(ctx)) res, err := ec.client.Bulk( reader, @@ -132,18 +129,17 @@ func (ec *elasticClient) DoBulkRequest(buff *bytes.Buffer, index string) error { } // DoMultiGet wil do a multi get request to Elasticsearch server -func (ec *elasticClient) DoMultiGet(ids []string, index string, withSource bool, resBody interface{}) error { +func (ec *elasticClient) DoMultiGet(ids []string, index string, withSource bool, resBody interface{}, ctx context.Context) error { obj := getDocumentsByIDsQuery(ids, withSource) body, err := encode(obj) if err != nil { return err } - valueCtx := context.WithValue(context.Background(), request.ContextKey, request.GetTopic) res, err := ec.client.Mget( &body, ec.client.Mget.WithIndex(index), - ec.client.Mget.WithContext(valueCtx), + ec.client.Mget.WithContext(ctx), ) if err != nil { log.Warn("elasticClient.DoMultiGet", @@ -162,18 +158,17 @@ func (ec *elasticClient) DoMultiGet(ids []string, index string, withSource bool, } // DoQueryRemove will do a query remove to elasticsearch server -func (ec *elasticClient) DoQueryRemove(index string, body *bytes.Buffer) error { +func (ec *elasticClient) DoQueryRemove(index string, body *bytes.Buffer, ctx context.Context) error { if err := ec.doRefresh(index); err != nil { log.Warn("elasticClient.doRefresh", "cannot do refresh", err.Error()) } - valueCtx := context.WithValue(context.Background(), request.ContextKey, request.RemoveTopic) res, err := ec.client.DeleteByQuery( []string{index}, body, ec.client.DeleteByQuery.WithIgnoreUnavailable(true), ec.client.DeleteByQuery.WithConflicts(esConflictsPolicy), - ec.client.DeleteByQuery.WithContext(valueCtx), + ec.client.DeleteByQuery.WithContext(ctx), ) if err != nil { @@ -339,7 +334,7 @@ func (ec *elasticClient) createAlias(alias string, index string) error { } // UpdateByQuery will update all the documents that match the provided query from the provided index -func (ec *elasticClient) UpdateByQuery(index string, buff *bytes.Buffer) error { +func (ec *elasticClient) UpdateByQuery(index string, buff *bytes.Buffer, ctx context.Context) error { reader := bytes.NewReader(buff.Bytes()) valueCtx := context.WithValue(context.Background(), request.ContextKey, request.UpdateTopic) diff --git a/client/elasticClientScroll.go b/client/elasticClientScroll.go index 1bda8a88..8044e7a0 100644 --- a/client/elasticClientScroll.go +++ b/client/elasticClientScroll.go @@ -14,10 +14,11 @@ import ( ) // DoCountRequest will get the number of elements that correspond with the provided query -func (ec *elasticClient) DoCountRequest(index string, body []byte) (uint64, error) { +func (ec *elasticClient) DoCountRequest(index string, body []byte, ctx context.Context) (uint64, error) { res, err := ec.client.Count( ec.client.Count.WithIndex(index), ec.client.Count.WithBody(bytes.NewBuffer(body)), + ec.client.Count.WithContext(ctx), ) if err != nil { return 0, err @@ -42,6 +43,7 @@ func (ec *elasticClient) DoScrollRequest( body []byte, withSource bool, handlerFunc func(responseBytes []byte) error, + ctx context.Context, ) error { ec.countScroll++ res, err := ec.client.Search( @@ -51,6 +53,7 @@ func (ec *elasticClient) DoScrollRequest( ec.client.Search.WithIndex(index), ec.client.Search.WithBody(bytes.NewBuffer(body)), ec.client.Search.WithSource(strconv.FormatBool(withSource)), + ec.client.Search.WithContext(ctx), ) if err != nil { return err diff --git a/client/elasticClientScroll_test.go b/client/elasticClientScroll_test.go index 013ef83f..9be41496 100644 --- a/client/elasticClientScroll_test.go +++ b/client/elasticClientScroll_test.go @@ -1,6 +1,7 @@ package client import ( + "context" "io/ioutil" "net/http" "net/http/httptest" @@ -32,7 +33,7 @@ func TestElasticClient_DoCountRequest(t *testing.T) { Logger: &logging.CustomLogger{}, }) - count, err := esClient.DoCountRequest("tokens", []byte(`{}`)) + count, err := esClient.DoCountRequest("tokens", []byte(`{}`), context.Background()) require.Nil(t, err) require.Equal(t, uint64(112671), count) } diff --git a/client/elasticClient_test.go b/client/elasticClient_test.go index 67dbb494..dbe7039e 100644 --- a/client/elasticClient_test.go +++ b/client/elasticClient_test.go @@ -1,6 +1,7 @@ package client import ( + "context" "io/ioutil" "net/http" "net/http/httptest" @@ -63,12 +64,12 @@ func TestElasticClient_DoMultiGet(t *testing.T) { ids := []string{"id"} res := &data.ResponseTokens{} - err := esClient.DoMultiGet(ids, "tokens", true, res) + err := esClient.DoMultiGet(ids, "tokens", true, res, context.Background()) require.Nil(t, err) require.Len(t, res.Docs, 3) resMap := make(objectsMap) - err = esClient.DoMultiGet(ids, "tokens", true, &resMap) + err = esClient.DoMultiGet(ids, "tokens", true, &resMap, context.Background()) require.Nil(t, err) _, ok := resMap["docs"] diff --git a/core/request/requests.go b/core/request/requests.go index ca4e6b7f..22d2daa8 100644 --- a/core/request/requests.go +++ b/core/request/requests.go @@ -1,13 +1,21 @@ package request -import "time" +import ( + "fmt" + "strconv" + "strings" + "time" +) const ( + separator = "_" + ContextKey = "key" RemoveTopic = "req_remove" GetTopic = "req_get" BulkTopic = "req_bulk" UpdateTopic = "req_update" + ScrollTopic = "req_scroll" ) // MetricsResponse defines the response for status metrics endpoint @@ -17,3 +25,24 @@ type MetricsResponse struct { TotalIndexingTime time.Duration `json:"total_time"` TotalData uint64 `json:"total_data"` } + +// ExtendTopicWithShardID will concatenate topic with shardID +func ExtendTopicWithShardID(topic string, shardID uint32) string { + return topic + separator + fmt.Sprintf("%d", shardID) +} + +// SplitTopicAndShardID will extract shard id from the provided topic +func SplitTopicAndShardID(topicWithShardID string) (string, uint32) { + split := strings.Split(topicWithShardID, separator) + if len(split) < 2 { + return topicWithShardID, 0 + } + + shardIDStr := split[len(split)-1] + shardID, err := strconv.ParseUint(shardIDStr, 10, 32) + if err != nil { + return topicWithShardID, 0 + } + + return strings.Join(split[:len(split)-1], separator), uint32(shardID) +} diff --git a/go.mod b/go.mod index a0bee4f8..f16466c9 100644 --- a/go.mod +++ b/go.mod @@ -10,14 +10,17 @@ require ( github.com/multiversx/mx-chain-core-go v1.2.10-0.20230719111700-ba37de156180 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.1 + github.com/prometheus/client_golang v1.12.1 github.com/stretchr/testify v1.8.4 github.com/tidwall/gjson v1.14.0 github.com/urfave/cli v1.22.10 ) require ( + github.com/beorn7/perks v1.0.1 // indirect github.com/btcsuite/btcd/btcutil v1.1.3 // indirect github.com/bytedance/sonic v1.9.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -37,12 +40,16 @@ require ( github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/mattn/go-isatty v0.0.19 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/pelletier/go-toml v1.9.3 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect github.com/russross/blackfriday/v2 v2.0.1 // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/tidwall/match v1.1.1 // indirect diff --git a/go.sum b/go.sum index 6ac7bf22..f3cffe16 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,7 @@ github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= @@ -90,6 +91,7 @@ github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZX github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= @@ -454,6 +456,7 @@ github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= @@ -575,16 +578,19 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -592,6 +598,7 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= diff --git a/metrics/prometheusMetrics.go b/metrics/prometheusMetrics.go new file mode 100644 index 00000000..cbfdfbe2 --- /dev/null +++ b/metrics/prometheusMetrics.go @@ -0,0 +1,42 @@ +package metrics + +import ( + "bytes" + "fmt" + + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + "google.golang.org/protobuf/proto" +) + +func counterMetric(metricName string, shardID uint32, count uint64) string { + metricFamily := &dto.MetricFamily{ + Name: proto.String(metricName), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("shardID"), + Value: proto.String(fmt.Sprintf("%d", shardID)), + }, + }, + Counter: &dto.Counter{ + Value: proto.Float64(float64(count)), + }, + }, + }, + } + + return promMetricAsString(metricFamily) +} + +func promMetricAsString(metric *dto.MetricFamily) string { + out := bytes.NewBuffer(make([]byte, 0)) + _, err := expfmt.MetricFamilyToText(out, metric) + if err != nil { + return "" + } + + return out.String() + "\n" +} diff --git a/metrics/statusMetrics.go b/metrics/statusMetrics.go index e2a3ee1f..1026f98e 100644 --- a/metrics/statusMetrics.go +++ b/metrics/statusMetrics.go @@ -50,15 +50,22 @@ func (sm *statusMetrics) GetMetrics() map[string]*request.MetricsResponse { // GetMetricsForPrometheus returns the metrics in a prometheus format func (sm *statusMetrics) GetMetricsForPrometheus() string { sm.mut.RLock() - defer sm.mut.RUnlock() - - //metricsMap := sm.getAll() + metrics := sm.getAllUnprotected() + sm.mut.RUnlock() stringBuilder := strings.Builder{} - // TODO populate with metrics + for topicWithShardID, metricsData := range metrics { + topic, shardID := request.SplitTopicAndShardID(topicWithShardID) + stringBuilder.WriteString(counterMetric(topic, shardID, metricsData.TotalData)) + stringBuilder.WriteString(counterMetric(topic, shardID, metricsData.ErrorsCount)) + stringBuilder.WriteString(counterMetric(topic, shardID, metricsData.OperationsCount)) + stringBuilder.WriteString(counterMetric(topic, shardID, uint64(metricsData.TotalIndexingTime.Milliseconds()))) + } + + promMetricsOutput := stringBuilder.String() - return stringBuilder.String() + return promMetricsOutput } func (sm *statusMetrics) getAllUnprotected() map[string]*request.MetricsResponse { diff --git a/mock/databaseWriterStub.go b/mock/databaseWriterStub.go index 3d468878..3cff3d60 100644 --- a/mock/databaseWriterStub.go +++ b/mock/databaseWriterStub.go @@ -2,6 +2,7 @@ package mock import ( "bytes" + "context" "github.com/elastic/go-elasticsearch/v7/esapi" ) @@ -17,17 +18,17 @@ type DatabaseWriterStub struct { } // UpdateByQuery - -func (dwm *DatabaseWriterStub) UpdateByQuery(_ string, _ *bytes.Buffer) error { +func (dwm *DatabaseWriterStub) UpdateByQuery(_ string, _ *bytes.Buffer, _ context.Context) error { return nil } // DoCountRequest - -func (dwm *DatabaseWriterStub) DoCountRequest(_ string, _ []byte) (uint64, error) { +func (dwm *DatabaseWriterStub) DoCountRequest(_ string, _ []byte, _ context.Context) (uint64, error) { return 0, nil } // DoScrollRequest - -func (dwm *DatabaseWriterStub) DoScrollRequest(index string, body []byte, withSource bool, handlerFunc func(responseBytes []byte) error) error { +func (dwm *DatabaseWriterStub) DoScrollRequest(index string, body []byte, withSource bool, handlerFunc func(responseBytes []byte) error, _ context.Context) error { if dwm.DoScrollRequestCalled != nil { return dwm.DoScrollRequestCalled(index, body, withSource, handlerFunc) } @@ -35,7 +36,7 @@ func (dwm *DatabaseWriterStub) DoScrollRequest(index string, body []byte, withSo } // DoRequest - -func (dwm *DatabaseWriterStub) DoRequest(req *esapi.IndexRequest) error { +func (dwm *DatabaseWriterStub) DoRequest(req *esapi.IndexRequest, _ context.Context) error { if dwm.DoRequestCalled != nil { return dwm.DoRequestCalled(req) } @@ -43,7 +44,7 @@ func (dwm *DatabaseWriterStub) DoRequest(req *esapi.IndexRequest) error { } // DoBulkRequest - -func (dwm *DatabaseWriterStub) DoBulkRequest(buff *bytes.Buffer, index string) error { +func (dwm *DatabaseWriterStub) DoBulkRequest(buff *bytes.Buffer, index string, _ context.Context) error { if dwm.DoBulkRequestCalled != nil { return dwm.DoBulkRequestCalled(buff, index) } @@ -51,7 +52,7 @@ func (dwm *DatabaseWriterStub) DoBulkRequest(buff *bytes.Buffer, index string) e } // DoMultiGet - -func (dwm *DatabaseWriterStub) DoMultiGet(hashes []string, index string, withSource bool, response interface{}) error { +func (dwm *DatabaseWriterStub) DoMultiGet(hashes []string, index string, withSource bool, response interface{}, _ context.Context) error { if dwm.DoMultiGetCalled != nil { return dwm.DoMultiGetCalled(hashes, index, withSource, response) } @@ -60,7 +61,7 @@ func (dwm *DatabaseWriterStub) DoMultiGet(hashes []string, index string, withSou } // DoQueryRemove - -func (dwm *DatabaseWriterStub) DoQueryRemove(index string, body *bytes.Buffer) error { +func (dwm *DatabaseWriterStub) DoQueryRemove(index string, body *bytes.Buffer, _ context.Context) error { if dwm.DoQueryRemoveCalled != nil { return dwm.DoQueryRemoveCalled(index, body) } diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index efcf13a5..5f0a1588 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -2,6 +2,7 @@ package elasticproc import ( "bytes" + "context" "encoding/hex" "fmt" "sync" @@ -12,6 +13,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/alteredAccount" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-es-indexer-go/core/request" "github.com/multiversx/mx-chain-es-indexer-go/data" elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" @@ -205,13 +207,14 @@ func (ei *elasticProcessor) createAliases() error { return nil } -func (ei *elasticProcessor) getExistingObjMap(hashes []string, index string) (map[string]bool, error) { +func (ei *elasticProcessor) getExistingObjMap(hashes []string, index string, shardID uint32) (map[string]bool, error) { if len(hashes) == 0 { return make(map[string]bool), nil } + ctx := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.GetTopic, shardID)) response := make(objectsMap) - err := ei.elasticClient.DoMultiGet(hashes, index, false, &response) + err := ei.elasticClient.DoMultiGet(hashes, index, false, &response, ctx) if err != nil { return make(map[string]bool), err } @@ -269,7 +272,7 @@ func (ei *elasticProcessor) SaveHeader(outportBlockWithHeader *outport.OutportBl return err } - return ei.doBulkRequests("", buffSlice.Buffers()) + return ei.doBulkRequests("", buffSlice.Buffers(), outportBlockWithHeader.ShardID) } func (ei *elasticProcessor) indexEpochInfoData(header coreData.HeaderHandler, buffSlice *data.BufferSlice) error { @@ -288,9 +291,11 @@ func (ei *elasticProcessor) RemoveHeader(header coreData.HeaderHandler) error { return err } + ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.RemoveTopic, header.GetShardID())) return ei.elasticClient.DoQueryRemove( elasticIndexer.BlockIndex, converters.PrepareHashesForQueryRemove([]string{hex.EncodeToString(headerHash)}), + ctxWithValue, ) } @@ -301,32 +306,35 @@ func (ei *elasticProcessor) RemoveMiniblocks(header coreData.HeaderHandler, body return nil } + ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.RemoveTopic, header.GetShardID())) return ei.elasticClient.DoQueryRemove( elasticIndexer.MiniblocksIndex, converters.PrepareHashesForQueryRemove(encodedMiniblocksHashes), + ctxWithValue, ) } // RemoveTransactions will remove transaction that are in miniblock from the elasticsearch server func (ei *elasticProcessor) RemoveTransactions(header coreData.HeaderHandler, body *block.Body) error { encodedTxsHashes, encodedScrsHashes := ei.transactionsProc.GetHexEncodedHashesForRemove(header, body) + shardID := header.GetShardID() - err := ei.removeIfHashesNotEmpty(elasticIndexer.TransactionsIndex, encodedTxsHashes) + err := ei.removeIfHashesNotEmpty(elasticIndexer.TransactionsIndex, encodedTxsHashes, shardID) if err != nil { return err } - err = ei.removeIfHashesNotEmpty(elasticIndexer.ScResultsIndex, encodedScrsHashes) + err = ei.removeIfHashesNotEmpty(elasticIndexer.ScResultsIndex, encodedScrsHashes, shardID) if err != nil { return err } - err = ei.removeIfHashesNotEmpty(elasticIndexer.OperationsIndex, append(encodedTxsHashes, encodedScrsHashes...)) + err = ei.removeIfHashesNotEmpty(elasticIndexer.OperationsIndex, append(encodedTxsHashes, encodedScrsHashes...), shardID) if err != nil { return err } - err = ei.removeIfHashesNotEmpty(elasticIndexer.LogsIndex, append(encodedTxsHashes, encodedScrsHashes...)) + err = ei.removeIfHashesNotEmpty(elasticIndexer.LogsIndex, append(encodedTxsHashes, encodedScrsHashes...), shardID) if err != nil { return err } @@ -343,27 +351,32 @@ func (ei *elasticProcessor) updateDelegatorsInCaseOfRevert(header coreData.Heade return nil } + ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.UpdateTopic, header.GetShardID())) delegatorsQuery := ei.logsAndEventsProc.PrepareDelegatorsQueryInCaseOfRevert(header.GetTimeStamp()) - return ei.elasticClient.UpdateByQuery(elasticIndexer.DelegatorsIndex, delegatorsQuery) + return ei.elasticClient.UpdateByQuery(elasticIndexer.DelegatorsIndex, delegatorsQuery, ctxWithValue) } -func (ei *elasticProcessor) removeIfHashesNotEmpty(index string, hashes []string) error { +func (ei *elasticProcessor) removeIfHashesNotEmpty(index string, hashes []string, shardID uint32) error { if len(hashes) == 0 { return nil } + ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.RemoveTopic, shardID)) return ei.elasticClient.DoQueryRemove( index, converters.PrepareHashesForQueryRemove(hashes), + ctxWithValue, ) } // RemoveAccountsESDT will remove data from accountsesdt index and accountsesdthistory func (ei *elasticProcessor) RemoveAccountsESDT(headerTimestamp uint64, shardID uint32) error { + ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.RemoveTopic, shardID)) query := fmt.Sprintf(`{"query": {"bool": {"must": [{"match": {"shardID": {"query": %d,"operator": "AND"}}},{"match": {"timestamp": {"query": "%d","operator": "AND"}}}]}}}`, shardID, headerTimestamp) err := ei.elasticClient.DoQueryRemove( elasticIndexer.AccountsESDTIndex, bytes.NewBuffer([]byte(query)), + ctxWithValue, ) if err != nil { return err @@ -372,6 +385,7 @@ func (ei *elasticProcessor) RemoveAccountsESDT(headerTimestamp uint64, shardID u return ei.elasticClient.DoQueryRemove( elasticIndexer.AccountsESDTHistoryIndex, bytes.NewBuffer([]byte(query)), + ctxWithValue, ) } @@ -386,7 +400,7 @@ func (ei *elasticProcessor) SaveMiniblocks(header coreData.HeaderHandler, body * return nil } - miniblocksInDBMap, err := ei.miniblocksInDBMap(mbs) + miniblocksInDBMap, err := ei.miniblocksInDBMap(mbs, header.GetShardID()) if err != nil { log.Warn("elasticProcessor.SaveMiniblocks cannot get indexed miniblocks", "error", err) } @@ -394,16 +408,16 @@ func (ei *elasticProcessor) SaveMiniblocks(header coreData.HeaderHandler, body * buffSlice := data.NewBufferSlice(ei.bulkRequestMaxSize) ei.miniblocksProc.SerializeBulkMiniBlocks(mbs, miniblocksInDBMap, buffSlice, elasticIndexer.MiniblocksIndex, header.GetShardID()) - return ei.doBulkRequests("", buffSlice.Buffers()) + return ei.doBulkRequests("", buffSlice.Buffers(), header.GetShardID()) } -func (ei *elasticProcessor) miniblocksInDBMap(mbs []*data.Miniblock) (map[string]bool, error) { +func (ei *elasticProcessor) miniblocksInDBMap(mbs []*data.Miniblock, shardID uint32) (map[string]bool, error) { mbsHashes := make([]string, len(mbs)) for idx := range mbs { mbsHashes[idx] = mbs[idx].Hash } - return ei.getExistingObjMap(mbsHashes, elasticIndexer.MiniblocksIndex) + return ei.getExistingObjMap(mbsHashes, elasticIndexer.MiniblocksIndex, shardID) } // SaveTransactions will prepare and save information about a transactions in elasticsearch server @@ -429,7 +443,7 @@ func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader return err } - err = ei.indexNFTCreateInfo(logsData.Tokens, obh.AlteredAccounts, buffers) + err = ei.indexNFTCreateInfo(logsData.Tokens, obh.AlteredAccounts, buffers, obh.ShardID) if err != nil { return err } @@ -460,7 +474,7 @@ func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader return err } - err = ei.indexTokens(logsData.TokensInfo, logsData.NFTsDataUpdates, buffers) + err = ei.indexTokens(logsData.TokensInfo, logsData.NFTsDataUpdates, buffers, obh.ShardID) if err != nil { return err } @@ -470,7 +484,7 @@ func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader return err } - err = ei.indexNFTBurnInfo(logsData.TokensSupply, buffers) + err = ei.indexNFTBurnInfo(logsData.TokensSupply, buffers, obh.ShardID) if err != nil { return err } @@ -489,7 +503,7 @@ func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader return err } - return ei.doBulkRequests("", buffers.Buffers()) + return ei.doBulkRequests("", buffers.Buffers(), obh.ShardID) } func (ei *elasticProcessor) prepareAndIndexRolesData(tokenRolesAndProperties *tokeninfo.TokenRolesAndProperties, buffSlice *data.BufferSlice, index string) error { @@ -580,7 +594,7 @@ func (ei *elasticProcessor) SaveValidatorsRating(ratingData *outport.ValidatorsR return err } - return ei.doBulkRequests(elasticIndexer.RatingIndex, buffSlice) + return ei.doBulkRequests(elasticIndexer.RatingIndex, buffSlice, ratingData.ShardID) } // SaveShardValidatorsPubKeys will prepare and save information about a shard validators public keys in elasticsearch server @@ -594,7 +608,7 @@ func (ei *elasticProcessor) SaveShardValidatorsPubKeys(validatorsPubKeys *outpor return err } - return ei.doBulkRequests(elasticIndexer.ValidatorsIndex, buffSlice) + return ei.doBulkRequests(elasticIndexer.ValidatorsIndex, buffSlice, validatorsPubKeys.ShardID) } // SaveRoundsInfo will prepare and save information about a slice of rounds in elasticsearch server @@ -605,7 +619,8 @@ func (ei *elasticProcessor) SaveRoundsInfo(rounds *outport.RoundsInfo) error { buff := ei.statisticsProc.SerializeRoundsInfo(rounds) - return ei.elasticClient.DoBulkRequest(buff, elasticIndexer.RoundsIndex) + ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.BulkTopic, rounds.ShardID)) + return ei.elasticClient.DoBulkRequest(buff, elasticIndexer.RoundsIndex, ctxWithValue) } func (ei *elasticProcessor) indexAlteredAccounts( @@ -635,7 +650,7 @@ func (ei *elasticProcessor) saveAccountsESDT( shardID uint32, ) error { accountsESDTMap, tokensData := ei.accountsProc.PrepareAccountsMapESDT(timestamp, wrappedAccounts, tagsCount, shardID) - err := ei.addTokenTypeAndCurrentOwnerInAccountsESDT(tokensData, accountsESDTMap) + err := ei.addTokenTypeAndCurrentOwnerInAccountsESDT(tokensData, accountsESDTMap, shardID) if err != nil { return err } @@ -648,13 +663,14 @@ func (ei *elasticProcessor) saveAccountsESDT( return ei.saveAccountsESDTHistory(timestamp, accountsESDTMap, buffSlice, shardID) } -func (ei *elasticProcessor) addTokenTypeAndCurrentOwnerInAccountsESDT(tokensData data.TokensHandler, accountsESDTMap map[string]*data.AccountInfo) error { +func (ei *elasticProcessor) addTokenTypeAndCurrentOwnerInAccountsESDT(tokensData data.TokensHandler, accountsESDTMap map[string]*data.AccountInfo, shardID uint32) error { if check.IfNil(tokensData) || tokensData.Len() == 0 { return nil } responseTokens := &data.ResponseTokens{} - err := ei.elasticClient.DoMultiGet(tokensData.GetAllTokens(), elasticIndexer.TokensIndex, true, responseTokens) + ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.GetTopic, shardID)) + err := ei.elasticClient.DoMultiGet(tokensData.GetAllTokens(), elasticIndexer.TokensIndex, true, responseTokens, ctxWithValue) if err != nil { return err } @@ -686,14 +702,15 @@ func (ei *elasticProcessor) indexAccountsESDT( return ei.accountsProc.SerializeAccountsESDT(accountsESDTMap, updatesNFTsData, buffSlice, elasticIndexer.AccountsESDTIndex) } -func (ei *elasticProcessor) indexNFTCreateInfo(tokensData data.TokensHandler, coreAlteredAccounts map[string]*alteredAccount.AlteredAccount, buffSlice *data.BufferSlice) error { +func (ei *elasticProcessor) indexNFTCreateInfo(tokensData data.TokensHandler, coreAlteredAccounts map[string]*alteredAccount.AlteredAccount, buffSlice *data.BufferSlice, shardID uint32) error { shouldSkipIndex := !ei.isIndexEnabled(elasticIndexer.TokensIndex) || tokensData.Len() == 0 if shouldSkipIndex { return nil } + ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.GetTopic, shardID)) responseTokens := &data.ResponseTokens{} - err := ei.elasticClient.DoMultiGet(tokensData.GetAllTokens(), elasticIndexer.TokensIndex, true, responseTokens) + err := ei.elasticClient.DoMultiGet(tokensData.GetAllTokens(), elasticIndexer.TokensIndex, true, responseTokens, ctxWithValue) if err != nil { return err } @@ -706,14 +723,15 @@ func (ei *elasticProcessor) indexNFTCreateInfo(tokensData data.TokensHandler, co return ei.accountsProc.SerializeNFTCreateInfo(tokens, buffSlice, elasticIndexer.TokensIndex) } -func (ei *elasticProcessor) indexNFTBurnInfo(tokensData data.TokensHandler, buffSlice *data.BufferSlice) error { +func (ei *elasticProcessor) indexNFTBurnInfo(tokensData data.TokensHandler, buffSlice *data.BufferSlice, shardID uint32) error { shouldSkipIndex := !ei.isIndexEnabled(elasticIndexer.TokensIndex) || tokensData.Len() == 0 if shouldSkipIndex { return nil } + ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.GetTopic, shardID)) responseTokens := &data.ResponseTokens{} - err := ei.elasticClient.DoMultiGet(tokensData.GetAllTokens(), elasticIndexer.TokensIndex, true, responseTokens) + err := ei.elasticClient.DoMultiGet(tokensData.GetAllTokens(), elasticIndexer.TokensIndex, true, responseTokens, ctxWithValue) if err != nil { return err } @@ -805,10 +823,11 @@ func (ei *elasticProcessor) isIndexEnabled(index string) bool { return isEnabled } -func (ei *elasticProcessor) doBulkRequests(index string, buffSlice []*bytes.Buffer) error { +func (ei *elasticProcessor) doBulkRequests(index string, buffSlice []*bytes.Buffer, shardID uint32) error { var err error for idx := range buffSlice { - err = ei.elasticClient.DoBulkRequest(buffSlice[idx], index) + ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.BulkTopic, shardID)) + err = ei.elasticClient.DoBulkRequest(buffSlice[idx], index, ctxWithValue) if err != nil { return err } diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index be7e9166..679d6ce3 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -2,6 +2,7 @@ package elasticproc import ( "bytes" + "context" "github.com/elastic/go-elasticsearch/v7/esapi" coreData "github.com/multiversx/mx-chain-core-go/data" @@ -14,13 +15,13 @@ import ( // DatabaseClientHandler defines the actions that a component that handles requests should do type DatabaseClientHandler interface { - DoRequest(req *esapi.IndexRequest) error - DoBulkRequest(buff *bytes.Buffer, index string) error - DoQueryRemove(index string, buff *bytes.Buffer) error - DoMultiGet(ids []string, index string, withSource bool, res interface{}) error - DoScrollRequest(index string, body []byte, withSource bool, handlerFunc func(responseBytes []byte) error) error - DoCountRequest(index string, body []byte) (uint64, error) - UpdateByQuery(index string, buff *bytes.Buffer) error + DoRequest(req *esapi.IndexRequest, ctx context.Context) error + DoBulkRequest(buff *bytes.Buffer, index string, ctx context.Context) error + DoQueryRemove(index string, buff *bytes.Buffer, ctx context.Context) error + DoMultiGet(ids []string, index string, withSource bool, res interface{}, ctx context.Context) error + DoScrollRequest(index string, body []byte, withSource bool, handlerFunc func(responseBytes []byte) error, ctx context.Context) error + DoCountRequest(index string, body []byte, ctx context.Context) (uint64, error) + UpdateByQuery(index string, buff *bytes.Buffer, ctx context.Context) error CheckAndCreateIndex(index string) error CheckAndCreateAlias(alias string, index string) error diff --git a/process/elasticproc/updateTokenType.go b/process/elasticproc/updateTokenType.go index fe14e162..1173ff04 100644 --- a/process/elasticproc/updateTokenType.go +++ b/process/elasticproc/updateTokenType.go @@ -1,16 +1,18 @@ package elasticproc import ( + "context" "encoding/json" "fmt" "time" "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-es-indexer-go/core/request" "github.com/multiversx/mx-chain-es-indexer-go/data" elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" ) -func (ei *elasticProcessor) indexTokens(tokensData []*data.TokenInfo, updateNFTData []*data.NFTDataUpdate, buffSlice *data.BufferSlice) error { +func (ei *elasticProcessor) indexTokens(tokensData []*data.TokenInfo, updateNFTData []*data.NFTDataUpdate, buffSlice *data.BufferSlice, shardID uint32) error { err := ei.prepareAndAddSerializedDataForTokens(tokensData, updateNFTData, buffSlice, elasticIndexer.ESDTsIndex) if err != nil { return err @@ -20,12 +22,12 @@ func (ei *elasticProcessor) indexTokens(tokensData []*data.TokenInfo, updateNFTD return err } - err = ei.addTokenType(tokensData, elasticIndexer.AccountsESDTIndex) + err = ei.addTokenType(tokensData, elasticIndexer.AccountsESDTIndex, shardID) if err != nil { return err } - return ei.addTokenType(tokensData, elasticIndexer.TokensIndex) + return ei.addTokenType(tokensData, elasticIndexer.TokensIndex, shardID) } func (ei *elasticProcessor) prepareAndAddSerializedDataForTokens(tokensData []*data.TokenInfo, updateNFTData []*data.NFTDataUpdate, buffSlice *data.BufferSlice, index string) error { @@ -36,7 +38,7 @@ func (ei *elasticProcessor) prepareAndAddSerializedDataForTokens(tokensData []*d return ei.logsAndEventsProc.SerializeTokens(tokensData, updateNFTData, buffSlice, index) } -func (ei *elasticProcessor) addTokenType(tokensData []*data.TokenInfo, index string) error { +func (ei *elasticProcessor) addTokenType(tokensData []*data.TokenInfo, index string, shardID uint32) error { if len(tokensData) == 0 { return nil } @@ -68,16 +70,18 @@ func (ei *elasticProcessor) addTokenType(tokensData []*data.TokenInfo, index str return err } - return ei.doBulkRequests(index, buffSlice.Buffers()) + return ei.doBulkRequests(index, buffSlice.Buffers(), shardID) } + ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.GetTopic, shardID)) query := fmt.Sprintf(`{"query": {"bool": {"must": [{"match": {"token": {"query": "%s","operator": "AND"}}}],"must_not":[{"exists": {"field": "type"}}]}}}`, td.Token) - resultsCount, err := ei.elasticClient.DoCountRequest(index, []byte(query)) + resultsCount, err := ei.elasticClient.DoCountRequest(index, []byte(query), ctxWithValue) if err != nil || resultsCount == 0 { return err } - err = ei.elasticClient.DoScrollRequest(index, []byte(query), false, handlerFunc) + ctxWithValue = context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.ScrollTopic, shardID)) + err = ei.elasticClient.DoScrollRequest(index, []byte(query), false, handlerFunc, ctxWithValue) if err != nil { return err } From 9ce1650f269bbc83108ec22d2d8ff864e04e4aa0 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 21 Jul 2023 11:09:11 +0300 Subject: [PATCH 124/189] fix linter issues --- core/request/requests.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/core/request/requests.go b/core/request/requests.go index 22d2daa8..fe49acf2 100644 --- a/core/request/requests.go +++ b/core/request/requests.go @@ -7,15 +7,17 @@ import ( "time" ) +type StringKey string + const ( separator = "_" - ContextKey = "key" - RemoveTopic = "req_remove" - GetTopic = "req_get" - BulkTopic = "req_bulk" - UpdateTopic = "req_update" - ScrollTopic = "req_scroll" + ContextKey StringKey = "key" + RemoveTopic = "req_remove" + GetTopic = "req_get" + BulkTopic = "req_bulk" + UpdateTopic = "req_update" + ScrollTopic = "req_scroll" ) // MetricsResponse defines the response for status metrics endpoint From cb4d47414ddea46036b42211f5cc0f53aaa77343 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 21 Jul 2023 11:11:25 +0300 Subject: [PATCH 125/189] fix linter issues --- core/request/requests.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/core/request/requests.go b/core/request/requests.go index fe49acf2..5fb12931 100644 --- a/core/request/requests.go +++ b/core/request/requests.go @@ -7,17 +7,17 @@ import ( "time" ) -type StringKey string +// StringKeyType defines the type for the context key +type StringKeyType string const ( - separator = "_" - - ContextKey StringKey = "key" - RemoveTopic = "req_remove" - GetTopic = "req_get" - BulkTopic = "req_bulk" - UpdateTopic = "req_update" - ScrollTopic = "req_scroll" + ContextKey StringKeyType = "key" + separator string = "_" + RemoveTopic string = "req_remove" + GetTopic string = "req_get" + BulkTopic string = "req_bulk" + UpdateTopic string = "req_update" + ScrollTopic string = "req_scroll" ) // MetricsResponse defines the response for status metrics endpoint From 2119794b041d5fbd43f8597014bd00d96d5a369e Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 21 Jul 2023 11:19:51 +0300 Subject: [PATCH 126/189] fix integration tests --- core/request/requests.go | 20 ++++++++++++------- .../accountsBalanceNftTransfer_test.go | 7 ++++--- .../accountsBalanceWithLowerTimestamp_test.go | 17 ++++++++-------- integrationtests/accountsESDTRollback_test.go | 5 +++-- .../accountsESDTWithTokenType_test.go | 13 ++++++------ integrationtests/claimRewards_test.go | 3 ++- integrationtests/createNFTWithTags_test.go | 9 +++++---- integrationtests/delegators_test.go | 11 +++++----- integrationtests/esdtTransfer_test.go | 3 ++- .../issueTokenAndSetRoles_test.go | 13 ++++++------ integrationtests/issueToken_test.go | 13 ++++++------ integrationtests/logsCrossShard_test.go | 9 +++++---- ...sferWithScCallAndErrorSignaledBySC_test.go | 5 +++-- integrationtests/nftIssueCreateBurn_test.go | 7 ++++--- .../nftTransferCrossShard_test.go | 15 +++++++------- .../nftTransferCrossWithScCall_test.go | 3 ++- integrationtests/relayedTx_test.go | 7 ++++--- integrationtests/scCallIntraShard_test.go | 5 +++-- integrationtests/scCallWithIssueEsdt_test.go | 11 +++++----- integrationtests/scDeploy_test.go | 5 +++-- integrationtests/transactions_test.go | 3 ++- integrationtests/updateNFT_test.go | 13 ++++++------ 22 files changed, 112 insertions(+), 85 deletions(-) diff --git a/core/request/requests.go b/core/request/requests.go index 5fb12931..a567afe5 100644 --- a/core/request/requests.go +++ b/core/request/requests.go @@ -11,13 +11,19 @@ import ( type StringKeyType string const ( - ContextKey StringKeyType = "key" - separator string = "_" - RemoveTopic string = "req_remove" - GetTopic string = "req_get" - BulkTopic string = "req_bulk" - UpdateTopic string = "req_update" - ScrollTopic string = "req_scroll" + // ContextKey the key for the value that will be added in the context + ContextKey StringKeyType = "key" + separator string = "_" + // RemoveTopic is the identifier for the remove requests metrics + RemoveTopic string = "req_remove" + // GetTopic is the identifier for the get requests metrics + GetTopic string = "req_get" + // BulkTopic is the identifier for the bulk requests metrics + BulkTopic string = "req_bulk" + // UpdateTopic is the identifier for the update requests metrics + UpdateTopic string = "req_update" + // ScrollTopic is the identifier for the scroll requests metrics + ScrollTopic string = "req_scroll" ) // MetricsResponse defines the response for status metrics endpoint diff --git a/integrationtests/accountsBalanceNftTransfer_test.go b/integrationtests/accountsBalanceNftTransfer_test.go index 6d586e89..ff1dca7c 100644 --- a/integrationtests/accountsBalanceNftTransfer_test.go +++ b/integrationtests/accountsBalanceNftTransfer_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "fmt" "math/big" @@ -94,7 +95,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { ids := []string{fmt.Sprintf("%s-NFT-abcdef-718863", addr)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsBalanceNftTransfer/balance-nft-after-create.json"), string(genericResponse.Docs[0].Source)) @@ -155,13 +156,13 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { ids = []string{fmt.Sprintf("%s-NFT-abcdef-718863", addr)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.False(t, genericResponse.Docs[0].Found) ids = []string{fmt.Sprintf("%s-NFT-abcdef-718863", addrReceiver)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsBalanceNftTransfer/balance-nft-after-transfer.json"), string(genericResponse.Docs[0].Source)) } diff --git a/integrationtests/accountsBalanceWithLowerTimestamp_test.go b/integrationtests/accountsBalanceWithLowerTimestamp_test.go index 4aa250af..7e0996b2 100644 --- a/integrationtests/accountsBalanceWithLowerTimestamp_test.go +++ b/integrationtests/accountsBalanceWithLowerTimestamp_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "fmt" "math/big" @@ -83,13 +84,13 @@ func TestIndexAccountsBalance(t *testing.T) { ids := []string{addr} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.AccountsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsBalanceWithLowerTimestamp/account-balance-first-update.json"), string(genericResponse.Docs[0].Source)) ids = []string{fmt.Sprintf("%s-TTTT-abcd-00", addr)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsBalanceWithLowerTimestamp/account-balance-esdt-first-update.json"), string(genericResponse.Docs[0].Source)) @@ -106,13 +107,13 @@ func TestIndexAccountsBalance(t *testing.T) { ids = []string{addr} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.AccountsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsBalanceWithLowerTimestamp/account-balance-first-update.json"), string(genericResponse.Docs[0].Source)) ids = []string{fmt.Sprintf("%s-TTTT-abcd-00", addr)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsBalanceWithLowerTimestamp/account-balance-esdt-first-update.json"), string(genericResponse.Docs[0].Source)) @@ -168,13 +169,13 @@ func TestIndexAccountsBalance(t *testing.T) { ids = []string{addr} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.AccountsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsBalanceWithLowerTimestamp/account-balance-second-update.json"), string(genericResponse.Docs[0].Source)) ids = []string{fmt.Sprintf("%s-TTTT-abcd-00", addr)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsBalanceWithLowerTimestamp/account-balance-esdt-second-update.json"), string(genericResponse.Docs[0].Source)) @@ -203,13 +204,13 @@ func TestIndexAccountsBalance(t *testing.T) { ids = []string{addr} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.AccountsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsBalanceWithLowerTimestamp/account-balance-second-update.json"), string(genericResponse.Docs[0].Source)) ids = []string{fmt.Sprintf("%s-TTTT-abcd-00", addr)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.False(t, genericResponse.Docs[0].Found) } diff --git a/integrationtests/accountsESDTRollback_test.go b/integrationtests/accountsESDTRollback_test.go index 9f6cdda5..d848a041 100644 --- a/integrationtests/accountsESDTRollback_test.go +++ b/integrationtests/accountsESDTRollback_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "encoding/json" "fmt" @@ -85,7 +86,7 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { ids := []string{fmt.Sprintf("%s-TOKEN-eeee-02", addr)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsESDTRollback/account-after-create.json"), string(genericResponse.Docs[0].Source)) @@ -93,7 +94,7 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { err = esProc.RemoveAccountsESDT(5040, 2) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.False(t, genericResponse.Docs[0].Found) } diff --git a/integrationtests/accountsESDTWithTokenType_test.go b/integrationtests/accountsESDTWithTokenType_test.go index bb25ce0c..4158521e 100644 --- a/integrationtests/accountsESDTWithTokenType_test.go +++ b/integrationtests/accountsESDTWithTokenType_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "encoding/json" "fmt" @@ -62,7 +63,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { ids := []string{"SEMI-abcd"} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/token-after-issue.json"), string(genericResponse.Docs[0].Source)) @@ -124,7 +125,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { ids = []string{fmt.Sprintf("%s-SEMI-abcd-02", address)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/account-esdt.json"), string(genericResponse.Docs[0].Source)) @@ -197,7 +198,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) ids := []string{fmt.Sprintf("%s-TTTT-abcd-02", address)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/account-esdt-without-type.json"), string(genericResponse.Docs[0].Source)) @@ -237,19 +238,19 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) ids = []string{"TTTT-abcd"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/semi-fungible-token.json"), string(genericResponse.Docs[0].Source)) ids = []string{fmt.Sprintf("%s-TTTT-abcd-02", address)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/account-esdt-with-type.json"), string(genericResponse.Docs[0].Source)) ids = []string{"TTTT-abcd-02"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/semi-fungible-token-after-create.json"), string(genericResponse.Docs[0].Source)) } diff --git a/integrationtests/claimRewards_test.go b/integrationtests/claimRewards_test.go index 8cf7eac1..49ab0ba9 100644 --- a/integrationtests/claimRewards_test.go +++ b/integrationtests/claimRewards_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "math/big" "testing" @@ -127,7 +128,7 @@ func TestTransactionWithClaimRewardsGasRefund(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, diff --git a/integrationtests/createNFTWithTags_test.go b/integrationtests/createNFTWithTags_test.go index 09a8d46a..52bb719b 100644 --- a/integrationtests/createNFTWithTags_test.go +++ b/integrationtests/createNFTWithTags_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "encoding/json" "fmt" @@ -94,13 +95,13 @@ func TestCreateNFTWithTags(t *testing.T) { ids := []string{fmt.Sprintf("%s-DESK-abcd-01", address1)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/createNFTWithTags/accounts-esdt-address-balance.json"), string(genericResponse.Docs[0].Source)) ids = []string{"bXVzaWM=", "aGVsbG8=", "Z2FsbGVyeQ==", "ZG8=", "YXJ0", "c29tZXRoaW5n"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TagsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TagsIndex, true, genericResponse, context.Background()) require.Nil(t, err) tagsChecked := 0 @@ -141,7 +142,7 @@ func TestCreateNFTWithTags(t *testing.T) { require.Nil(t, err) genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TagsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TagsIndex, true, genericResponse, context.Background()) require.Nil(t, err) tagsChecked = 0 @@ -191,7 +192,7 @@ func TestCreateNFTWithTags(t *testing.T) { ids = append(ids, "XFxcXFxcXFxcXFxcXFxcXFxcXA==", "JycnJw==", "PDw8Pj4+JiYmJiYmJiYmJiYmJiYm") genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TagsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TagsIndex, true, genericResponse, context.Background()) require.Nil(t, err) tagsChecked = 0 diff --git a/integrationtests/delegators_test.go b/integrationtests/delegators_test.go index 85ca507f..bb12baec 100644 --- a/integrationtests/delegators_test.go +++ b/integrationtests/delegators_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "math/big" "testing" @@ -64,7 +65,7 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { ids := []string{"9v/pLAXxUZJ4Oy1U+x5al/Xg5sebh1dYCRTeZwg/u68="} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/delegators/delegator-after-delegate.json"), string(genericResponse.Docs[0].Source)) @@ -94,7 +95,7 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/delegators/delegator-after-un-delegate-1.json"), string(genericResponse.Docs[0].Source)) @@ -124,7 +125,7 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/delegators/delegator-after-un-delegate-2.json"), string(genericResponse.Docs[0].Source)) time.Sleep(time.Second) @@ -135,7 +136,7 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { require.Nil(t, err) time.Sleep(time.Second) - err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/delegators/delegator-after-revert.json"), string(genericResponse.Docs[0].Source)) @@ -164,7 +165,7 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/delegators/delegator-after-withdraw.json"), string(genericResponse.Docs[0].Source)) } diff --git a/integrationtests/esdtTransfer_test.go b/integrationtests/esdtTransfer_test.go index 132b743e..9f549329 100644 --- a/integrationtests/esdtTransfer_test.go +++ b/integrationtests/esdtTransfer_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "math/big" "testing" @@ -107,7 +108,7 @@ func TestESDTTransferTooMuchGasProvided(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/esdtTransfer/esdt-transfer.json"), string(genericResponse.Docs[0].Source)) diff --git a/integrationtests/issueTokenAndSetRoles_test.go b/integrationtests/issueTokenAndSetRoles_test.go index 53524d24..a3ab5ab9 100644 --- a/integrationtests/issueTokenAndSetRoles_test.go +++ b/integrationtests/issueTokenAndSetRoles_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "math/big" "testing" @@ -63,7 +64,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { ids := []string{"TOK-abcd"} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-issue-ok.json"), string(genericResponse.Docs[0].Source)) @@ -93,7 +94,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { ids = []string{"TOK-abcd"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-set-role.json"), string(genericResponse.Docs[0].Source)) @@ -127,7 +128,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { ids = []string{"TOK-abcd"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-transfer-role.json"), string(genericResponse.Docs[0].Source)) @@ -157,7 +158,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { ids = []string{"TOK-abcd"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-unset-role.json"), string(genericResponse.Docs[0].Source)) } @@ -205,7 +206,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { ids := []string{"TTT-abcd"} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-set-roles-first.json"), string(genericResponse.Docs[0].Source)) @@ -234,7 +235,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { ids = []string{"TTT-abcd"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-set-roles-and-issue.json"), string(genericResponse.Docs[0].Source)) } diff --git a/integrationtests/issueToken_test.go b/integrationtests/issueToken_test.go index 11cf5ded..4ee784fb 100644 --- a/integrationtests/issueToken_test.go +++ b/integrationtests/issueToken_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "math/big" "testing" @@ -57,11 +58,11 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { ids := []string{"SSSS-abcd"} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueToken/token-semi.json"), string(genericResponse.Docs[0].Source)) - err = esClient.DoMultiGet(ids, indexerdata.ESDTsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.ESDTsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueToken/token-semi.json"), string(genericResponse.Docs[0].Source)) @@ -89,11 +90,11 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueToken/token-semi-after-transfer-ownership.json"), string(genericResponse.Docs[0].Source)) - err = esClient.DoMultiGet(ids, indexerdata.ESDTsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.ESDTsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueToken/token-semi-after-transfer-ownership.json"), string(genericResponse.Docs[0].Source)) @@ -121,7 +122,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueToken/token-semi-after-pause.json"), string(genericResponse.Docs[0].Source)) @@ -149,7 +150,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueToken/token-semi-after-un-pause.json"), string(genericResponse.Docs[0].Source)) } diff --git a/integrationtests/logsCrossShard_test.go b/integrationtests/logsCrossShard_test.go index 52869697..576fbf1a 100644 --- a/integrationtests/logsCrossShard_test.go +++ b/integrationtests/logsCrossShard_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "math/big" "testing" @@ -60,7 +61,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { ids := []string{logID} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/logsCrossShard/log-at-source.json"), @@ -99,7 +100,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/logsCrossShard/log-at-destination.json"), @@ -132,7 +133,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/logsCrossShard/log-at-destination.json"), @@ -158,7 +159,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { err = esProc.RemoveTransactions(header, body) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.False(t, genericResponse.Docs[0].Found) diff --git a/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go b/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go index 9b08e3c3..863466a9 100644 --- a/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go +++ b/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "math/big" "testing" @@ -95,7 +96,7 @@ func TestMultiTransferCrossShardAndScCallErrorSignaledBySC(t *testing.T) { require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, @@ -164,7 +165,7 @@ func TestMultiTransferCrossShardAndScCallErrorSignaledBySC(t *testing.T) { ids = []string{hex.EncodeToString(txHash)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, diff --git a/integrationtests/nftIssueCreateBurn_test.go b/integrationtests/nftIssueCreateBurn_test.go index 74a652e1..97df96c5 100644 --- a/integrationtests/nftIssueCreateBurn_test.go +++ b/integrationtests/nftIssueCreateBurn_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "encoding/json" "math/big" @@ -60,7 +61,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { ids := []string{"NON-abcd"} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/nftIssueCreateBurn/non-fungible-after-issue.json"), string(genericResponse.Docs[0].Source)) @@ -105,7 +106,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { ids = []string{"NON-abcd-02"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/nftIssueCreateBurn/non-fungible-after-create.json"), string(genericResponse.Docs[0].Source)) @@ -141,7 +142,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { ids = []string{"NON-abcd-02"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.False(t, genericResponse.Docs[0].Found) } diff --git a/integrationtests/nftTransferCrossShard_test.go b/integrationtests/nftTransferCrossShard_test.go index 4227924a..abc27611 100644 --- a/integrationtests/nftTransferCrossShard_test.go +++ b/integrationtests/nftTransferCrossShard_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "math/big" "testing" @@ -97,7 +98,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, @@ -135,7 +136,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, @@ -144,7 +145,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { ) genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.OperationsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.OperationsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/nftTransferCrossShard/op-nft-transfer-sc-call-after-refund.json"), @@ -241,7 +242,7 @@ func TestNFTTransferCrossShard(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, @@ -291,7 +292,7 @@ func TestNFTTransferCrossShard(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, @@ -373,7 +374,7 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, @@ -442,7 +443,7 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, diff --git a/integrationtests/nftTransferCrossWithScCall_test.go b/integrationtests/nftTransferCrossWithScCall_test.go index 336f5da3..196606fe 100644 --- a/integrationtests/nftTransferCrossWithScCall_test.go +++ b/integrationtests/nftTransferCrossWithScCall_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "math/big" "testing" @@ -92,7 +93,7 @@ func TestNFTTransferCrossShardWithScCall(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/nftTransferCrossShardWithScCall/cross-shard-transfer-with-sc-call.json"), diff --git a/integrationtests/relayedTx_test.go b/integrationtests/relayedTx_test.go index 08868c26..4de8b2af 100644 --- a/integrationtests/relayedTx_test.go +++ b/integrationtests/relayedTx_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "math/big" "testing" @@ -95,7 +96,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, @@ -135,7 +136,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, @@ -236,7 +237,7 @@ func TestRelayedTransactionIntraShard(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, diff --git a/integrationtests/scCallIntraShard_test.go b/integrationtests/scCallIntraShard_test.go index 3c45d6ae..e8c5f2f3 100644 --- a/integrationtests/scCallIntraShard_test.go +++ b/integrationtests/scCallIntraShard_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "math/big" "testing" @@ -102,7 +103,7 @@ func TestTransactionWithSCCallFail(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, @@ -182,7 +183,7 @@ func TestTransactionWithScCallSuccess(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, diff --git a/integrationtests/scCallWithIssueEsdt_test.go b/integrationtests/scCallWithIssueEsdt_test.go index 8dea7be4..c1b9234e 100644 --- a/integrationtests/scCallWithIssueEsdt_test.go +++ b/integrationtests/scCallWithIssueEsdt_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "math/big" "testing" @@ -101,7 +102,7 @@ func TestScCallIntraShardWithIssueESDT(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, @@ -110,7 +111,7 @@ func TestScCallIntraShardWithIssueESDT(t *testing.T) { ) ids = []string{hex.EncodeToString(scrWithIssueHash)} - err = esClient.DoMultiGet(ids, indexerData.OperationsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerData.OperationsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, @@ -163,7 +164,7 @@ func TestScCallIntraShardWithIssueESDT(t *testing.T) { require.Nil(t, err) ids = []string{hex.EncodeToString(scrWithIssueHash), hex.EncodeToString(scrWithCallBackHash)} - err = esClient.DoMultiGet(ids, indexerData.OperationsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerData.OperationsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, @@ -216,7 +217,7 @@ func TestScCallIntraShardWithIssueESDT(t *testing.T) { require.Nil(t, err) ids = []string{hex.EncodeToString(txHash)} - err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, @@ -225,7 +226,7 @@ func TestScCallIntraShardWithIssueESDT(t *testing.T) { ) ids = []string{hex.EncodeToString(txHash), hex.EncodeToString(scrWithCallBackHash)} - err = esClient.DoMultiGet(ids, indexerData.OperationsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerData.OperationsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, diff --git a/integrationtests/scDeploy_test.go b/integrationtests/scDeploy_test.go index 2ed3f944..4d6f0f2a 100644 --- a/integrationtests/scDeploy_test.go +++ b/integrationtests/scDeploy_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "math/big" "testing" @@ -87,7 +88,7 @@ func TestTransactionWithSCDeploy(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, @@ -96,7 +97,7 @@ func TestTransactionWithSCDeploy(t *testing.T) { ) ids = []string{"erd1qqqqqqqqqqqqqpgq4t2tqxpst9a6qttpak8cz8wvz6a0nses5qfqel6rhy"} - err = esClient.DoMultiGet(ids, indexerData.SCDeploysIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerData.SCDeploysIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, diff --git a/integrationtests/transactions_test.go b/integrationtests/transactions_test.go index 55ec3910..96a041c6 100644 --- a/integrationtests/transactions_test.go +++ b/integrationtests/transactions_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "math/big" "testing" @@ -68,7 +69,7 @@ func TestElasticIndexerSaveTransactions(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, diff --git a/integrationtests/updateNFT_test.go b/integrationtests/updateNFT_test.go index 17dff437..923906b1 100644 --- a/integrationtests/updateNFT_test.go +++ b/integrationtests/updateNFT_test.go @@ -3,6 +3,7 @@ package integrationtests import ( + "context" "encoding/hex" "encoding/json" "math/big" @@ -64,7 +65,7 @@ func TestNFTUpdateMetadata(t *testing.T) { ids := []string{"NFT-abcd-0e"} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/updateNFT/token.json"), string(genericResponse.Docs[0].Source)) @@ -113,7 +114,7 @@ func TestNFTUpdateMetadata(t *testing.T) { // Update attributes 1 ids = []string{"NFT-abcd-0e"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/updateNFT/token-after-add-uris.json"), string(genericResponse.Docs[0].Source)) @@ -139,7 +140,7 @@ func TestNFTUpdateMetadata(t *testing.T) { ids = []string{"NFT-abcd-0e"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/updateNFT/token-after-update-attributes.json"), string(genericResponse.Docs[0].Source)) @@ -167,7 +168,7 @@ func TestNFTUpdateMetadata(t *testing.T) { ids = []string{"NFT-abcd-0e"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/updateNFT/token-after-update-attributes-second.json"), string(genericResponse.Docs[0].Source)) @@ -193,7 +194,7 @@ func TestNFTUpdateMetadata(t *testing.T) { require.Nil(t, err) ids = []string{"NFT-abcd-0e"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/updateNFT/token-after-freeze.json"), string(genericResponse.Docs[0].Source)) @@ -219,7 +220,7 @@ func TestNFTUpdateMetadata(t *testing.T) { require.Nil(t, err) ids = []string{"NFT-abcd-0e"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse) + err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/updateNFT/token-after-un-freeze.json"), string(genericResponse.Docs[0].Source)) } From 5385bd4274931c1966467b727f7b09a45d03deb5 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 21 Jul 2023 11:50:31 +0300 Subject: [PATCH 127/189] unit tests metricsHandler and transport --- client/transport/transport.go | 8 +++++ client/transport/transport_test.go | 52 ++++++++++++++++++++++++++++ metrics/statusMetrics_test.go | 54 ++++++++++++++++++++++++++++++ mock/mockTransport.go | 14 ++++++++ 4 files changed, 128 insertions(+) create mode 100644 client/transport/transport_test.go create mode 100644 metrics/statusMetrics_test.go create mode 100644 mock/mockTransport.go diff --git a/client/transport/transport.go b/client/transport/transport.go index d70aefc5..b5aa2901 100644 --- a/client/transport/transport.go +++ b/client/transport/transport.go @@ -5,6 +5,7 @@ import ( "net/http" "time" + "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-es-indexer-go/core" "github.com/multiversx/mx-chain-es-indexer-go/core/request" "github.com/multiversx/mx-chain-es-indexer-go/metrics" @@ -15,13 +16,20 @@ type metricsTransport struct { transport http.RoundTripper } +// NewMetricsTransport will create a new instance of metricsTransport func NewMetricsTransport(statusMetrics core.StatusMetricsHandler) (*metricsTransport, error) { + if check.IfNil(statusMetrics) { + return nil, core.ErrNilMetricsHandler + } + return &metricsTransport{ statusMetrics: statusMetrics, transport: http.DefaultTransport, }, nil } +// RoundTrip implements the http.RoundTripper interface and is used as a wrapper around the underlying +// transport to collect and record metrics related to the HTTP request/response cycle. func (m *metricsTransport) RoundTrip(req *http.Request) (*http.Response, error) { startTime := time.Now() size := req.ContentLength diff --git a/client/transport/transport_test.go b/client/transport/transport_test.go new file mode 100644 index 00000000..446e92be --- /dev/null +++ b/client/transport/transport_test.go @@ -0,0 +1,52 @@ +package transport + +import ( + "bytes" + "context" + "net/http" + "testing" + + "github.com/multiversx/mx-chain-es-indexer-go/core" + "github.com/multiversx/mx-chain-es-indexer-go/core/request" + "github.com/multiversx/mx-chain-es-indexer-go/metrics" + "github.com/multiversx/mx-chain-es-indexer-go/mock" + "github.com/stretchr/testify/require" +) + +func TestNewMetricsTransport(t *testing.T) { + t.Parallel() + + transportHandler, err := NewMetricsTransport(nil) + require.Nil(t, transportHandler) + require.Equal(t, core.ErrNilMetricsHandler, err) + + metricsHandler := metrics.NewStatusMetrics() + transportHandler, err = NewMetricsTransport(metricsHandler) + require.Nil(t, err) + require.NotNil(t, transportHandler) +} + +func TestMetricsTransport_RoundTrip(t *testing.T) { + t.Parallel() + + metricsHandler := metrics.NewStatusMetrics() + transportHandler, _ := NewMetricsTransport(metricsHandler) + + transportHandler.transport = &mock.TransportMock{ + Response: &http.Response{ + StatusCode: http.StatusOK, + }, + Err: nil, + } + + testTopic := "test" + contextWithValue := context.WithValue(context.Background(), request.ContextKey, testTopic) + req, _ := http.NewRequestWithContext(contextWithValue, http.MethodGet, "dummy", bytes.NewBuffer([]byte("test"))) + + _, _ = transportHandler.RoundTrip(req) + + metricsMap := metricsHandler.GetMetrics() + require.Equal(t, uint64(1), metricsMap[testTopic].OperationsCount) + require.Equal(t, uint64(0), metricsMap[testTopic].ErrorsCount) + require.Equal(t, uint64(4), metricsMap[testTopic].TotalData) +} diff --git a/metrics/statusMetrics_test.go b/metrics/statusMetrics_test.go new file mode 100644 index 00000000..321a622a --- /dev/null +++ b/metrics/statusMetrics_test.go @@ -0,0 +1,54 @@ +package metrics + +import ( + "testing" + + "github.com/multiversx/mx-chain-es-indexer-go/core/request" + "github.com/stretchr/testify/require" +) + +func TestStatusMetrics_AddIndexingDataAndGetMetrics(t *testing.T) { + t.Parallel() + + statusMetricsHandler := NewStatusMetrics() + require.False(t, statusMetricsHandler.IsInterfaceNil()) + + topic1 := "test1_0" + statusMetricsHandler.AddIndexingData(ArgsAddIndexingData{ + GotError: true, + MessageLen: 100, + Topic: topic1, + Duration: 12, + }) + statusMetricsHandler.AddIndexingData(ArgsAddIndexingData{ + GotError: false, + MessageLen: 222, + Topic: topic1, + Duration: 15, + }) + + topic2 := "test2_2" + statusMetricsHandler.AddIndexingData(ArgsAddIndexingData{ + GotError: true, + MessageLen: 100, + Topic: topic2, + Duration: 12, + }) + + metrics := statusMetricsHandler.GetMetrics() + require.Equal(t, &request.MetricsResponse{ + OperationsCount: 2, + ErrorsCount: 1, + TotalIndexingTime: 27, + TotalData: 322, + }, metrics[topic1]) + require.Equal(t, &request.MetricsResponse{ + OperationsCount: 1, + ErrorsCount: 1, + TotalIndexingTime: 12, + TotalData: 100, + }, metrics[topic2]) + + prometheusMetrics := statusMetricsHandler.GetMetricsForPrometheus() + require.Equal(t, "# TYPE test1 counter\ntest1{shardID=\"0\"} 322\n\n# TYPE test1 counter\ntest1{shardID=\"0\"} 1\n\n# TYPE test1 counter\ntest1{shardID=\"0\"} 2\n\n# TYPE test1 counter\ntest1{shardID=\"0\"} 0\n\n# TYPE test2 counter\ntest2{shardID=\"2\"} 100\n\n# TYPE test2 counter\ntest2{shardID=\"2\"} 1\n\n# TYPE test2 counter\ntest2{shardID=\"2\"} 1\n\n# TYPE test2 counter\ntest2{shardID=\"2\"} 0\n\n", prometheusMetrics) +} diff --git a/mock/mockTransport.go b/mock/mockTransport.go new file mode 100644 index 00000000..d40e425f --- /dev/null +++ b/mock/mockTransport.go @@ -0,0 +1,14 @@ +package mock + +import "net/http" + +// TransportMock - +type TransportMock struct { + Response *http.Response + Err error +} + +// RoundTrip - +func (m *TransportMock) RoundTrip(_ *http.Request) (*http.Response, error) { + return m.Response, m.Err +} From 60fbbb9b97948c8f597331ba21574da09acfe03f Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 21 Jul 2023 11:55:58 +0300 Subject: [PATCH 128/189] fix test --- metrics/statusMetrics_test.go | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/metrics/statusMetrics_test.go b/metrics/statusMetrics_test.go index 321a622a..3820c752 100644 --- a/metrics/statusMetrics_test.go +++ b/metrics/statusMetrics_test.go @@ -27,14 +27,6 @@ func TestStatusMetrics_AddIndexingDataAndGetMetrics(t *testing.T) { Duration: 15, }) - topic2 := "test2_2" - statusMetricsHandler.AddIndexingData(ArgsAddIndexingData{ - GotError: true, - MessageLen: 100, - Topic: topic2, - Duration: 12, - }) - metrics := statusMetricsHandler.GetMetrics() require.Equal(t, &request.MetricsResponse{ OperationsCount: 2, @@ -42,13 +34,7 @@ func TestStatusMetrics_AddIndexingDataAndGetMetrics(t *testing.T) { TotalIndexingTime: 27, TotalData: 322, }, metrics[topic1]) - require.Equal(t, &request.MetricsResponse{ - OperationsCount: 1, - ErrorsCount: 1, - TotalIndexingTime: 12, - TotalData: 100, - }, metrics[topic2]) prometheusMetrics := statusMetricsHandler.GetMetricsForPrometheus() - require.Equal(t, "# TYPE test1 counter\ntest1{shardID=\"0\"} 322\n\n# TYPE test1 counter\ntest1{shardID=\"0\"} 1\n\n# TYPE test1 counter\ntest1{shardID=\"0\"} 2\n\n# TYPE test1 counter\ntest1{shardID=\"0\"} 0\n\n# TYPE test2 counter\ntest2{shardID=\"2\"} 100\n\n# TYPE test2 counter\ntest2{shardID=\"2\"} 1\n\n# TYPE test2 counter\ntest2{shardID=\"2\"} 1\n\n# TYPE test2 counter\ntest2{shardID=\"2\"} 0\n\n", prometheusMetrics) + require.Equal(t, "# TYPE test1 counter\ntest1{shardID=\"0\"} 322\n\n# TYPE test1 counter\ntest1{shardID=\"0\"} 1\n\n# TYPE test1 counter\ntest1{shardID=\"0\"} 2\n\n# TYPE test1 counter\ntest1{shardID=\"0\"} 0\n\n", prometheusMetrics) } From e33c4a078acaf79039f2026f3e2bf596c3cda5bf Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 21 Jul 2023 13:59:21 +0300 Subject: [PATCH 129/189] fixes after first review --- client/transport/transport_test.go | 20 +++++++++++++++++ core/request/requests.go | 5 +++-- core/request/requests_test.go | 36 ++++++++++++++++++++++++++++++ process/factory/indexerFactory.go | 2 +- 4 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 core/request/requests_test.go diff --git a/client/transport/transport_test.go b/client/transport/transport_test.go index 446e92be..ce34f464 100644 --- a/client/transport/transport_test.go +++ b/client/transport/transport_test.go @@ -50,3 +50,23 @@ func TestMetricsTransport_RoundTrip(t *testing.T) { require.Equal(t, uint64(0), metricsMap[testTopic].ErrorsCount) require.Equal(t, uint64(4), metricsMap[testTopic].TotalData) } + +func TestMetricsTransport_RoundTripNoValueInContextShouldNoAddMetrics(t *testing.T) { + t.Parallel() + + metricsHandler := metrics.NewStatusMetrics() + transportHandler, _ := NewMetricsTransport(metricsHandler) + + transportHandler.transport = &mock.TransportMock{ + Response: &http.Response{ + StatusCode: http.StatusOK, + }, + Err: nil, + } + + req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "dummy", bytes.NewBuffer([]byte("test"))) + _, _ = transportHandler.RoundTrip(req) + + metricsMap := metricsHandler.GetMetrics() + require.Len(t, metricsMap, 0) +} diff --git a/core/request/requests.go b/core/request/requests.go index a567afe5..13f02182 100644 --- a/core/request/requests.go +++ b/core/request/requests.go @@ -46,11 +46,12 @@ func SplitTopicAndShardID(topicWithShardID string) (string, uint32) { return topicWithShardID, 0 } - shardIDStr := split[len(split)-1] + shardIDIndex := len(split) - 1 + shardIDStr := split[shardIDIndex] shardID, err := strconv.ParseUint(shardIDStr, 10, 32) if err != nil { return topicWithShardID, 0 } - return strings.Join(split[:len(split)-1], separator), uint32(shardID) + return strings.Join(split[:shardIDIndex], separator), uint32(shardID) } diff --git a/core/request/requests_test.go b/core/request/requests_test.go new file mode 100644 index 00000000..5792b565 --- /dev/null +++ b/core/request/requests_test.go @@ -0,0 +1,36 @@ +package request + +import ( + "testing" + + "github.com/multiversx/mx-chain-core-go/core" + "github.com/stretchr/testify/require" +) + +func TestExtendTopicWithShardID(t *testing.T) { + t.Parallel() + + require.Equal(t, "req_update_2", ExtendTopicWithShardID(UpdateTopic, 2)) + require.Equal(t, "req_bulk_0", ExtendTopicWithShardID(BulkTopic, 0)) + require.Equal(t, "req_get_1", ExtendTopicWithShardID(GetTopic, 1)) + require.Equal(t, "req_scroll_4294967295", ExtendTopicWithShardID(ScrollTopic, core.MetachainShardId)) +} + +func TestSplitTopicAndShardID(t *testing.T) { + + topic, shardID := SplitTopicAndShardID("req_update_2") + require.Equal(t, UpdateTopic, topic) + require.Equal(t, uint32(2), shardID) + + topic, shardID = SplitTopicAndShardID("req_scroll_4294967295") + require.Equal(t, ScrollTopic, topic) + require.Equal(t, core.MetachainShardId, shardID) + + topic, shardID = SplitTopicAndShardID("req") + require.Equal(t, "req", topic) + require.Equal(t, uint32(0), shardID) + + topic, shardID = SplitTopicAndShardID("req_aaaa") + require.Equal(t, "req_aaaa", topic) + require.Equal(t, uint32(0), shardID) +} diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index 5e298d53..0666edf7 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -110,7 +110,7 @@ func createElasticClient(args ArgsIndexerFactory) (elasticproc.DatabaseClientHan RetryBackoff: retryBackOff, } - if args.StatusMetrics == nil { + if check.IfNil(args.StatusMetrics) { return client.NewElasticClient(argsEsClient) } From 8834ef9d4e15c05fc6ed12ed825c667c67167859 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 21 Jul 2023 14:07:43 +0300 Subject: [PATCH 130/189] extra check --- factory/webServerFactory.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/factory/webServerFactory.go b/factory/webServerFactory.go index 14830649..d3a504bc 100644 --- a/factory/webServerFactory.go +++ b/factory/webServerFactory.go @@ -1,6 +1,7 @@ package factory import ( + "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-es-indexer-go/api/gin" "github.com/multiversx/mx-chain-es-indexer-go/config" "github.com/multiversx/mx-chain-es-indexer-go/core" @@ -9,6 +10,10 @@ import ( // CreateWebServer will create a new instance of core.WebServerHandler func CreateWebServer(apiConfig config.ApiRoutesConfig, statusMetricsHandler core.StatusMetricsHandler) (core.WebServerHandler, error) { + if check.IfNil(statusMetricsHandler) { + return nil, core.ErrNilMetricsHandler + } + metricsFacade, err := facade.NewMetricsFacade(statusMetricsHandler) if err != nil { return nil, err From 0d2a479e413dfe409b9547c2091d01a19e157f1f Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 21 Jul 2023 15:43:30 +0300 Subject: [PATCH 131/189] fixes after second review --- client/transport/transport_test.go | 2 +- go.mod | 10 +++------ go.sum | 7 +----- metrics/prometheusMetrics.go | 6 +++++- metrics/statusMetrics.go | 34 ++++++++++++++++++++++++++---- metrics/statusMetrics_test.go | 13 +++++++++++- process/wsindexer/indexer.go | 1 + 7 files changed, 53 insertions(+), 20 deletions(-) diff --git a/client/transport/transport_test.go b/client/transport/transport_test.go index ce34f464..7bd03c27 100644 --- a/client/transport/transport_test.go +++ b/client/transport/transport_test.go @@ -51,7 +51,7 @@ func TestMetricsTransport_RoundTrip(t *testing.T) { require.Equal(t, uint64(4), metricsMap[testTopic].TotalData) } -func TestMetricsTransport_RoundTripNoValueInContextShouldNoAddMetrics(t *testing.T) { +func TestMetricsTransport_RoundTripNoValueInContextShouldNotAddMetrics(t *testing.T) { t.Parallel() metricsHandler := metrics.NewStatusMetrics() diff --git a/go.mod b/go.mod index f16466c9..78f32077 100644 --- a/go.mod +++ b/go.mod @@ -10,17 +10,17 @@ require ( github.com/multiversx/mx-chain-core-go v1.2.10-0.20230719111700-ba37de156180 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.1 - github.com/prometheus/client_golang v1.12.1 + github.com/prometheus/client_model v0.2.0 + github.com/prometheus/common v0.37.0 github.com/stretchr/testify v1.8.4 github.com/tidwall/gjson v1.14.0 github.com/urfave/cli v1.22.10 + google.golang.org/protobuf v1.28.1 ) require ( - github.com/beorn7/perks v1.0.1 // indirect github.com/btcsuite/btcd/btcutil v1.1.3 // indirect github.com/bytedance/sonic v1.9.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -47,9 +47,6 @@ require ( github.com/pelletier/go-toml v1.9.3 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect github.com/russross/blackfriday/v2 v2.0.1 // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/tidwall/match v1.1.1 // indirect @@ -61,6 +58,5 @@ require ( golang.org/x/net v0.10.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index f3cffe16..1627e93f 100644 --- a/go.sum +++ b/go.sum @@ -56,7 +56,6 @@ github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= @@ -91,7 +90,6 @@ github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZX github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= @@ -578,7 +576,6 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -598,7 +595,6 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -1139,9 +1135,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/metrics/prometheusMetrics.go b/metrics/prometheusMetrics.go index cbfdfbe2..e5e6e3fd 100644 --- a/metrics/prometheusMetrics.go +++ b/metrics/prometheusMetrics.go @@ -9,13 +9,17 @@ import ( "google.golang.org/protobuf/proto" ) -func counterMetric(metricName string, shardID uint32, count uint64) string { +func counterMetric(metricName, operation string, shardID uint32, count uint64) string { metricFamily := &dto.MetricFamily{ Name: proto.String(metricName), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{ { Label: []*dto.LabelPair{ + { + Name: proto.String("operation"), + Value: proto.String(operation), + }, { Name: proto.String("shardID"), Value: proto.String(fmt.Sprintf("%d", shardID)), diff --git a/metrics/statusMetrics.go b/metrics/statusMetrics.go index 1026f98e..611d8ce2 100644 --- a/metrics/statusMetrics.go +++ b/metrics/statusMetrics.go @@ -1,12 +1,21 @@ package metrics import ( + "bytes" "strings" "sync" + "unicode" "github.com/multiversx/mx-chain-es-indexer-go/core/request" ) +const ( + operationCount = "operations_count" + errorsCount = "errors_count" + totalTime = "total_time" + totalData = "total_data" +) + type statusMetrics struct { metrics map[string]*request.MetricsResponse mut sync.RWMutex @@ -57,10 +66,10 @@ func (sm *statusMetrics) GetMetricsForPrometheus() string { for topicWithShardID, metricsData := range metrics { topic, shardID := request.SplitTopicAndShardID(topicWithShardID) - stringBuilder.WriteString(counterMetric(topic, shardID, metricsData.TotalData)) - stringBuilder.WriteString(counterMetric(topic, shardID, metricsData.ErrorsCount)) - stringBuilder.WriteString(counterMetric(topic, shardID, metricsData.OperationsCount)) - stringBuilder.WriteString(counterMetric(topic, shardID, uint64(metricsData.TotalIndexingTime.Milliseconds()))) + stringBuilder.WriteString(counterMetric(camelToSnake(topic), totalData, shardID, metricsData.TotalData)) + stringBuilder.WriteString(counterMetric(camelToSnake(topic), errorsCount, shardID, metricsData.ErrorsCount)) + stringBuilder.WriteString(counterMetric(camelToSnake(topic), operationCount, shardID, metricsData.OperationsCount)) + stringBuilder.WriteString(counterMetric(camelToSnake(topic), totalTime, shardID, uint64(metricsData.TotalIndexingTime.Milliseconds()))) } promMetricsOutput := stringBuilder.String() @@ -81,3 +90,20 @@ func (sm *statusMetrics) getAllUnprotected() map[string]*request.MetricsResponse func (sm *statusMetrics) IsInterfaceNil() bool { return sm == nil } + +func camelToSnake(camelStr string) string { + var snakeBuf bytes.Buffer + + for i, r := range camelStr { + if unicode.IsUpper(r) { + if i > 0 && unicode.IsLower(rune(camelStr[i-1])) { + snakeBuf.WriteRune('_') + } + snakeBuf.WriteRune(unicode.ToLower(r)) + } else { + snakeBuf.WriteRune(r) + } + } + + return snakeBuf.String() +} diff --git a/metrics/statusMetrics_test.go b/metrics/statusMetrics_test.go index 3820c752..c34865c4 100644 --- a/metrics/statusMetrics_test.go +++ b/metrics/statusMetrics_test.go @@ -3,6 +3,7 @@ package metrics import ( "testing" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-es-indexer-go/core/request" "github.com/stretchr/testify/require" ) @@ -36,5 +37,15 @@ func TestStatusMetrics_AddIndexingDataAndGetMetrics(t *testing.T) { }, metrics[topic1]) prometheusMetrics := statusMetricsHandler.GetMetricsForPrometheus() - require.Equal(t, "# TYPE test1 counter\ntest1{shardID=\"0\"} 322\n\n# TYPE test1 counter\ntest1{shardID=\"0\"} 1\n\n# TYPE test1 counter\ntest1{shardID=\"0\"} 2\n\n# TYPE test1 counter\ntest1{shardID=\"0\"} 0\n\n", prometheusMetrics) + require.Equal(t, "# TYPE test1 counter\ntest1{operation=\"total_data\",shardID=\"0\"} 322\n\n# TYPE test1 counter\ntest1{operation=\"errors_count\",shardID=\"0\"} 1\n\n# TYPE test1 counter\ntest1{operation=\"operations_count\",shardID=\"0\"} 2\n\n# TYPE test1 counter\ntest1{operation=\"total_time\",shardID=\"0\"} 0\n\n", prometheusMetrics) +} + +func TestCamelCaseToSnakeCase(t *testing.T) { + t.Parallel() + + require.Equal(t, "settings", camelToSnake(outport.TopicSettings)) + require.Equal(t, "save_validators_pub_keys", camelToSnake(outport.TopicSaveValidatorsPubKeys)) + require.Equal(t, "000000000000000", camelToSnake("000000000000000")) + require.Equal(t, "one_one_one", camelToSnake("One_One_One")) + require.Equal(t, "req_block", camelToSnake("req_block")) } diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index b72d22b4..98a4a2d0 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -19,6 +19,7 @@ var ( errNilDataIndexer = errors.New("nil data indexer") ) +// ArgsIndexer holds all the components needed to create a new instance of indexer type ArgsIndexer struct { Marshaller marshal.Marshalizer DataIndexer DataIndexer From 496936daf0898f2698bb36e9c5c0803be278701d Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 24 Jul 2023 13:28:21 +0300 Subject: [PATCH 132/189] small fix --- factory/webServerFactory.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/factory/webServerFactory.go b/factory/webServerFactory.go index d3a504bc..14830649 100644 --- a/factory/webServerFactory.go +++ b/factory/webServerFactory.go @@ -1,7 +1,6 @@ package factory import ( - "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-es-indexer-go/api/gin" "github.com/multiversx/mx-chain-es-indexer-go/config" "github.com/multiversx/mx-chain-es-indexer-go/core" @@ -10,10 +9,6 @@ import ( // CreateWebServer will create a new instance of core.WebServerHandler func CreateWebServer(apiConfig config.ApiRoutesConfig, statusMetricsHandler core.StatusMetricsHandler) (core.WebServerHandler, error) { - if check.IfNil(statusMetricsHandler) { - return nil, core.ErrNilMetricsHandler - } - metricsFacade, err := facade.NewMetricsFacade(statusMetricsHandler) if err != nil { return nil, err From 28e344d163277390406bbd71b7f0609210ac7ff9 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 25 Jul 2023 12:02:23 +0300 Subject: [PATCH 133/189] fixes after review --- client/elasticClient.go | 24 +++++++++++------------- client/elasticClientScroll.go | 4 ++-- client/elasticClientScroll_test.go | 2 +- client/elasticClient_test.go | 4 ++-- client/transport/transport.go | 7 +++++++ client/transport/transport_test.go | 8 ++++++++ core/request/requests.go | 11 ++++++----- core/request/requests_test.go | 9 +++++---- metrics/prometheusMetrics.go | 5 ++--- metrics/statusMetrics.go | 12 ++++++------ mock/databaseWriterStub.go | 14 +++++++------- process/elasticproc/elasticProcessor.go | 24 ++++++++++++------------ process/elasticproc/interface.go | 14 +++++++------- process/elasticproc/updateTokenType.go | 4 ++-- scripts/observers/.env | 2 ++ 15 files changed, 80 insertions(+), 64 deletions(-) diff --git a/client/elasticClient.go b/client/elasticClient.go index f5c39728..8c9ba482 100644 --- a/client/elasticClient.go +++ b/client/elasticClient.go @@ -10,7 +10,6 @@ import ( "github.com/elastic/go-elasticsearch/v7" "github.com/elastic/go-elasticsearch/v7/esapi" - "github.com/multiversx/mx-chain-es-indexer-go/core/request" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" logger "github.com/multiversx/mx-chain-logger-go" @@ -95,7 +94,7 @@ func (ec *elasticClient) CheckAndCreateAlias(alias string, indexName string) err } // DoRequest will do a request to elastic server -func (ec *elasticClient) DoRequest(req *esapi.IndexRequest, ctx context.Context) error { +func (ec *elasticClient) DoRequest(ctx context.Context, req *esapi.IndexRequest) error { res, err := req.Do(ctx, ec.client) if err != nil { return err @@ -105,7 +104,7 @@ func (ec *elasticClient) DoRequest(req *esapi.IndexRequest, ctx context.Context) } // DoBulkRequest will do a bulk of request to elastic server -func (ec *elasticClient) DoBulkRequest(buff *bytes.Buffer, index string, ctx context.Context) error { +func (ec *elasticClient) DoBulkRequest(ctx context.Context, buff *bytes.Buffer, index string) error { reader := bytes.NewReader(buff.Bytes()) options := make([]func(*esapi.BulkRequest), 0) @@ -129,7 +128,7 @@ func (ec *elasticClient) DoBulkRequest(buff *bytes.Buffer, index string, ctx con } // DoMultiGet wil do a multi get request to Elasticsearch server -func (ec *elasticClient) DoMultiGet(ids []string, index string, withSource bool, resBody interface{}, ctx context.Context) error { +func (ec *elasticClient) DoMultiGet(ctx context.Context, ids []string, index string, withSource bool, resBody interface{}) error { obj := getDocumentsByIDsQuery(ids, withSource) body, err := encode(obj) if err != nil { @@ -158,9 +157,10 @@ func (ec *elasticClient) DoMultiGet(ids []string, index string, withSource bool, } // DoQueryRemove will do a query remove to elasticsearch server -func (ec *elasticClient) DoQueryRemove(index string, body *bytes.Buffer, ctx context.Context) error { - if err := ec.doRefresh(index); err != nil { - log.Warn("elasticClient.doRefresh", "cannot do refresh", err.Error()) +func (ec *elasticClient) DoQueryRemove(ctx context.Context, index string, body *bytes.Buffer) error { + err := ec.doRefresh(index) + if err != nil { + log.Warn("elasticClient.doRefresh", "cannot do refresh", err) } res, err := ec.client.DeleteByQuery( @@ -172,13 +172,13 @@ func (ec *elasticClient) DoQueryRemove(index string, body *bytes.Buffer, ctx con ) if err != nil { - log.Warn("elasticClient.DoQueryRemove", "cannot do query remove", err.Error()) + log.Warn("elasticClient.DoQueryRemove", "cannot do query remove", err) return err } err = parseResponse(res, nil, elasticDefaultErrorResponseHandler) if err != nil { - log.Warn("elasticClient.DoQueryRemove", "error parsing response", err.Error()) + log.Warn("elasticClient.DoQueryRemove", "error parsing response", err) return err } @@ -334,14 +334,12 @@ func (ec *elasticClient) createAlias(alias string, index string) error { } // UpdateByQuery will update all the documents that match the provided query from the provided index -func (ec *elasticClient) UpdateByQuery(index string, buff *bytes.Buffer, ctx context.Context) error { +func (ec *elasticClient) UpdateByQuery(ctx context.Context, index string, buff *bytes.Buffer) error { reader := bytes.NewReader(buff.Bytes()) - - valueCtx := context.WithValue(context.Background(), request.ContextKey, request.UpdateTopic) res, err := ec.client.UpdateByQuery( []string{index}, ec.client.UpdateByQuery.WithBody(reader), - ec.client.UpdateByQuery.WithContext(valueCtx), + ec.client.UpdateByQuery.WithContext(ctx), ) if err != nil { return err diff --git a/client/elasticClientScroll.go b/client/elasticClientScroll.go index 8044e7a0..1e1c5037 100644 --- a/client/elasticClientScroll.go +++ b/client/elasticClientScroll.go @@ -14,7 +14,7 @@ import ( ) // DoCountRequest will get the number of elements that correspond with the provided query -func (ec *elasticClient) DoCountRequest(index string, body []byte, ctx context.Context) (uint64, error) { +func (ec *elasticClient) DoCountRequest(ctx context.Context, index string, body []byte) (uint64, error) { res, err := ec.client.Count( ec.client.Count.WithIndex(index), ec.client.Count.WithBody(bytes.NewBuffer(body)), @@ -39,11 +39,11 @@ func (ec *elasticClient) DoCountRequest(index string, body []byte, ctx context.C // DoScrollRequest will perform a documents request using scroll api func (ec *elasticClient) DoScrollRequest( + ctx context.Context, index string, body []byte, withSource bool, handlerFunc func(responseBytes []byte) error, - ctx context.Context, ) error { ec.countScroll++ res, err := ec.client.Search( diff --git a/client/elasticClientScroll_test.go b/client/elasticClientScroll_test.go index 9be41496..b0fe3d7b 100644 --- a/client/elasticClientScroll_test.go +++ b/client/elasticClientScroll_test.go @@ -33,7 +33,7 @@ func TestElasticClient_DoCountRequest(t *testing.T) { Logger: &logging.CustomLogger{}, }) - count, err := esClient.DoCountRequest("tokens", []byte(`{}`), context.Background()) + count, err := esClient.DoCountRequest(context.Background(), "tokens", []byte(`{}`)) require.Nil(t, err) require.Equal(t, uint64(112671), count) } diff --git a/client/elasticClient_test.go b/client/elasticClient_test.go index dbe7039e..96b552ec 100644 --- a/client/elasticClient_test.go +++ b/client/elasticClient_test.go @@ -64,12 +64,12 @@ func TestElasticClient_DoMultiGet(t *testing.T) { ids := []string{"id"} res := &data.ResponseTokens{} - err := esClient.DoMultiGet(ids, "tokens", true, res, context.Background()) + err := esClient.DoMultiGet(context.Background(), ids, "tokens", true, res) require.Nil(t, err) require.Len(t, res.Docs, 3) resMap := make(objectsMap) - err = esClient.DoMultiGet(ids, "tokens", true, &resMap, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, "tokens", true, &resMap) require.Nil(t, err) _, ok := resMap["docs"] diff --git a/client/transport/transport.go b/client/transport/transport.go index b5aa2901..fa44b361 100644 --- a/client/transport/transport.go +++ b/client/transport/transport.go @@ -1,6 +1,7 @@ package transport import ( + "errors" "fmt" "net/http" "time" @@ -11,6 +12,8 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/metrics" ) +var errNilRequest = errors.New("nil request") + type metricsTransport struct { statusMetrics core.StatusMetricsHandler transport http.RoundTripper @@ -31,6 +34,10 @@ func NewMetricsTransport(statusMetrics core.StatusMetricsHandler) (*metricsTrans // RoundTrip implements the http.RoundTripper interface and is used as a wrapper around the underlying // transport to collect and record metrics related to the HTTP request/response cycle. func (m *metricsTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req == nil { + return nil, errNilRequest + } + startTime := time.Now() size := req.ContentLength resp, err := m.transport.RoundTrip(req) diff --git a/client/transport/transport_test.go b/client/transport/transport_test.go index 7bd03c27..26d8513e 100644 --- a/client/transport/transport_test.go +++ b/client/transport/transport_test.go @@ -26,6 +26,14 @@ func TestNewMetricsTransport(t *testing.T) { require.NotNil(t, transportHandler) } +func TestMetricsTransport_NilRequest(t *testing.T) { + metricsHandler := metrics.NewStatusMetrics() + transportHandler, _ := NewMetricsTransport(metricsHandler) + + _, err := transportHandler.RoundTrip(nil) + require.Equal(t, errNilRequest, err) +} + func TestMetricsTransport_RoundTrip(t *testing.T) { t.Parallel() diff --git a/core/request/requests.go b/core/request/requests.go index 13f02182..cdbb50f9 100644 --- a/core/request/requests.go +++ b/core/request/requests.go @@ -11,6 +11,7 @@ import ( type StringKeyType string const ( + noShardID = "#" // ContextKey the key for the value that will be added in the context ContextKey StringKeyType = "key" separator string = "_" @@ -40,18 +41,18 @@ func ExtendTopicWithShardID(topic string, shardID uint32) string { } // SplitTopicAndShardID will extract shard id from the provided topic -func SplitTopicAndShardID(topicWithShardID string) (string, uint32) { +func SplitTopicAndShardID(topicWithShardID string) (string, string) { split := strings.Split(topicWithShardID, separator) if len(split) < 2 { - return topicWithShardID, 0 + return topicWithShardID, noShardID } shardIDIndex := len(split) - 1 shardIDStr := split[shardIDIndex] - shardID, err := strconv.ParseUint(shardIDStr, 10, 32) + _, err := strconv.ParseUint(shardIDStr, 10, 32) if err != nil { - return topicWithShardID, 0 + return topicWithShardID, noShardID } - return strings.Join(split[:shardIDIndex], separator), uint32(shardID) + return strings.Join(split[:shardIDIndex], separator), shardIDStr } diff --git a/core/request/requests_test.go b/core/request/requests_test.go index 5792b565..d824c1ae 100644 --- a/core/request/requests_test.go +++ b/core/request/requests_test.go @@ -1,6 +1,7 @@ package request import ( + "fmt" "testing" "github.com/multiversx/mx-chain-core-go/core" @@ -20,17 +21,17 @@ func TestSplitTopicAndShardID(t *testing.T) { topic, shardID := SplitTopicAndShardID("req_update_2") require.Equal(t, UpdateTopic, topic) - require.Equal(t, uint32(2), shardID) + require.Equal(t, "2", shardID) topic, shardID = SplitTopicAndShardID("req_scroll_4294967295") require.Equal(t, ScrollTopic, topic) - require.Equal(t, core.MetachainShardId, shardID) + require.Equal(t, fmt.Sprintf("%d", core.MetachainShardId), shardID) topic, shardID = SplitTopicAndShardID("req") require.Equal(t, "req", topic) - require.Equal(t, uint32(0), shardID) + require.Equal(t, noShardID, shardID) topic, shardID = SplitTopicAndShardID("req_aaaa") require.Equal(t, "req_aaaa", topic) - require.Equal(t, uint32(0), shardID) + require.Equal(t, noShardID, shardID) } diff --git a/metrics/prometheusMetrics.go b/metrics/prometheusMetrics.go index e5e6e3fd..f564102a 100644 --- a/metrics/prometheusMetrics.go +++ b/metrics/prometheusMetrics.go @@ -2,14 +2,13 @@ package metrics import ( "bytes" - "fmt" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" "google.golang.org/protobuf/proto" ) -func counterMetric(metricName, operation string, shardID uint32, count uint64) string { +func counterMetric(metricName, operation string, shardIDStr string, count uint64) string { metricFamily := &dto.MetricFamily{ Name: proto.String(metricName), Type: dto.MetricType_COUNTER.Enum(), @@ -22,7 +21,7 @@ func counterMetric(metricName, operation string, shardID uint32, count uint64) s }, { Name: proto.String("shardID"), - Value: proto.String(fmt.Sprintf("%d", shardID)), + Value: proto.String(shardIDStr), }, }, Counter: &dto.Counter{ diff --git a/metrics/statusMetrics.go b/metrics/statusMetrics.go index 611d8ce2..58e0fbad 100644 --- a/metrics/statusMetrics.go +++ b/metrics/statusMetrics.go @@ -33,7 +33,7 @@ func (sm *statusMetrics) AddIndexingData(args ArgsAddIndexingData) { sm.mut.Lock() defer sm.mut.Unlock() - topic := args.Topic + topic := camelToSnake(args.Topic) _, found := sm.metrics[topic] if !found { sm.metrics[topic] = &request.MetricsResponse{} @@ -65,11 +65,11 @@ func (sm *statusMetrics) GetMetricsForPrometheus() string { stringBuilder := strings.Builder{} for topicWithShardID, metricsData := range metrics { - topic, shardID := request.SplitTopicAndShardID(topicWithShardID) - stringBuilder.WriteString(counterMetric(camelToSnake(topic), totalData, shardID, metricsData.TotalData)) - stringBuilder.WriteString(counterMetric(camelToSnake(topic), errorsCount, shardID, metricsData.ErrorsCount)) - stringBuilder.WriteString(counterMetric(camelToSnake(topic), operationCount, shardID, metricsData.OperationsCount)) - stringBuilder.WriteString(counterMetric(camelToSnake(topic), totalTime, shardID, uint64(metricsData.TotalIndexingTime.Milliseconds()))) + topic, shardIDStr := request.SplitTopicAndShardID(topicWithShardID) + stringBuilder.WriteString(counterMetric(topic, totalData, shardIDStr, metricsData.TotalData)) + stringBuilder.WriteString(counterMetric(topic, errorsCount, shardIDStr, metricsData.ErrorsCount)) + stringBuilder.WriteString(counterMetric(topic, operationCount, shardIDStr, metricsData.OperationsCount)) + stringBuilder.WriteString(counterMetric(topic, totalTime, shardIDStr, uint64(metricsData.TotalIndexingTime.Milliseconds()))) } promMetricsOutput := stringBuilder.String() diff --git a/mock/databaseWriterStub.go b/mock/databaseWriterStub.go index 3cff3d60..23dd74ea 100644 --- a/mock/databaseWriterStub.go +++ b/mock/databaseWriterStub.go @@ -18,17 +18,17 @@ type DatabaseWriterStub struct { } // UpdateByQuery - -func (dwm *DatabaseWriterStub) UpdateByQuery(_ string, _ *bytes.Buffer, _ context.Context) error { +func (dwm *DatabaseWriterStub) UpdateByQuery(_ context.Context, _ string, _ *bytes.Buffer) error { return nil } // DoCountRequest - -func (dwm *DatabaseWriterStub) DoCountRequest(_ string, _ []byte, _ context.Context) (uint64, error) { +func (dwm *DatabaseWriterStub) DoCountRequest(_ context.Context, _ string, _ []byte) (uint64, error) { return 0, nil } // DoScrollRequest - -func (dwm *DatabaseWriterStub) DoScrollRequest(index string, body []byte, withSource bool, handlerFunc func(responseBytes []byte) error, _ context.Context) error { +func (dwm *DatabaseWriterStub) DoScrollRequest(_ context.Context, index string, body []byte, withSource bool, handlerFunc func(responseBytes []byte) error) error { if dwm.DoScrollRequestCalled != nil { return dwm.DoScrollRequestCalled(index, body, withSource, handlerFunc) } @@ -36,7 +36,7 @@ func (dwm *DatabaseWriterStub) DoScrollRequest(index string, body []byte, withSo } // DoRequest - -func (dwm *DatabaseWriterStub) DoRequest(req *esapi.IndexRequest, _ context.Context) error { +func (dwm *DatabaseWriterStub) DoRequest(_ context.Context, req *esapi.IndexRequest) error { if dwm.DoRequestCalled != nil { return dwm.DoRequestCalled(req) } @@ -44,7 +44,7 @@ func (dwm *DatabaseWriterStub) DoRequest(req *esapi.IndexRequest, _ context.Cont } // DoBulkRequest - -func (dwm *DatabaseWriterStub) DoBulkRequest(buff *bytes.Buffer, index string, _ context.Context) error { +func (dwm *DatabaseWriterStub) DoBulkRequest(_ context.Context, buff *bytes.Buffer, index string) error { if dwm.DoBulkRequestCalled != nil { return dwm.DoBulkRequestCalled(buff, index) } @@ -52,7 +52,7 @@ func (dwm *DatabaseWriterStub) DoBulkRequest(buff *bytes.Buffer, index string, _ } // DoMultiGet - -func (dwm *DatabaseWriterStub) DoMultiGet(hashes []string, index string, withSource bool, response interface{}, _ context.Context) error { +func (dwm *DatabaseWriterStub) DoMultiGet(_ context.Context, hashes []string, index string, withSource bool, response interface{}) error { if dwm.DoMultiGetCalled != nil { return dwm.DoMultiGetCalled(hashes, index, withSource, response) } @@ -61,7 +61,7 @@ func (dwm *DatabaseWriterStub) DoMultiGet(hashes []string, index string, withSou } // DoQueryRemove - -func (dwm *DatabaseWriterStub) DoQueryRemove(index string, body *bytes.Buffer, _ context.Context) error { +func (dwm *DatabaseWriterStub) DoQueryRemove(_ context.Context, index string, body *bytes.Buffer) error { if dwm.DoQueryRemoveCalled != nil { return dwm.DoQueryRemoveCalled(index, body) } diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index 5f0a1588..887ac938 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -214,7 +214,7 @@ func (ei *elasticProcessor) getExistingObjMap(hashes []string, index string, sha ctx := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.GetTopic, shardID)) response := make(objectsMap) - err := ei.elasticClient.DoMultiGet(hashes, index, false, &response, ctx) + err := ei.elasticClient.DoMultiGet(ctx, hashes, index, false, &response) if err != nil { return make(map[string]bool), err } @@ -293,9 +293,9 @@ func (ei *elasticProcessor) RemoveHeader(header coreData.HeaderHandler) error { ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.RemoveTopic, header.GetShardID())) return ei.elasticClient.DoQueryRemove( + ctxWithValue, elasticIndexer.BlockIndex, converters.PrepareHashesForQueryRemove([]string{hex.EncodeToString(headerHash)}), - ctxWithValue, ) } @@ -308,9 +308,9 @@ func (ei *elasticProcessor) RemoveMiniblocks(header coreData.HeaderHandler, body ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.RemoveTopic, header.GetShardID())) return ei.elasticClient.DoQueryRemove( + ctxWithValue, elasticIndexer.MiniblocksIndex, converters.PrepareHashesForQueryRemove(encodedMiniblocksHashes), - ctxWithValue, ) } @@ -353,7 +353,7 @@ func (ei *elasticProcessor) updateDelegatorsInCaseOfRevert(header coreData.Heade ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.UpdateTopic, header.GetShardID())) delegatorsQuery := ei.logsAndEventsProc.PrepareDelegatorsQueryInCaseOfRevert(header.GetTimeStamp()) - return ei.elasticClient.UpdateByQuery(elasticIndexer.DelegatorsIndex, delegatorsQuery, ctxWithValue) + return ei.elasticClient.UpdateByQuery(ctxWithValue, elasticIndexer.DelegatorsIndex, delegatorsQuery) } func (ei *elasticProcessor) removeIfHashesNotEmpty(index string, hashes []string, shardID uint32) error { @@ -363,9 +363,9 @@ func (ei *elasticProcessor) removeIfHashesNotEmpty(index string, hashes []string ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.RemoveTopic, shardID)) return ei.elasticClient.DoQueryRemove( + ctxWithValue, index, converters.PrepareHashesForQueryRemove(hashes), - ctxWithValue, ) } @@ -374,18 +374,18 @@ func (ei *elasticProcessor) RemoveAccountsESDT(headerTimestamp uint64, shardID u ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.RemoveTopic, shardID)) query := fmt.Sprintf(`{"query": {"bool": {"must": [{"match": {"shardID": {"query": %d,"operator": "AND"}}},{"match": {"timestamp": {"query": "%d","operator": "AND"}}}]}}}`, shardID, headerTimestamp) err := ei.elasticClient.DoQueryRemove( + ctxWithValue, elasticIndexer.AccountsESDTIndex, bytes.NewBuffer([]byte(query)), - ctxWithValue, ) if err != nil { return err } return ei.elasticClient.DoQueryRemove( + ctxWithValue, elasticIndexer.AccountsESDTHistoryIndex, bytes.NewBuffer([]byte(query)), - ctxWithValue, ) } @@ -620,7 +620,7 @@ func (ei *elasticProcessor) SaveRoundsInfo(rounds *outport.RoundsInfo) error { buff := ei.statisticsProc.SerializeRoundsInfo(rounds) ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.BulkTopic, rounds.ShardID)) - return ei.elasticClient.DoBulkRequest(buff, elasticIndexer.RoundsIndex, ctxWithValue) + return ei.elasticClient.DoBulkRequest(ctxWithValue, buff, elasticIndexer.RoundsIndex) } func (ei *elasticProcessor) indexAlteredAccounts( @@ -670,7 +670,7 @@ func (ei *elasticProcessor) addTokenTypeAndCurrentOwnerInAccountsESDT(tokensData responseTokens := &data.ResponseTokens{} ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.GetTopic, shardID)) - err := ei.elasticClient.DoMultiGet(tokensData.GetAllTokens(), elasticIndexer.TokensIndex, true, responseTokens, ctxWithValue) + err := ei.elasticClient.DoMultiGet(ctxWithValue, tokensData.GetAllTokens(), elasticIndexer.TokensIndex, true, responseTokens) if err != nil { return err } @@ -710,7 +710,7 @@ func (ei *elasticProcessor) indexNFTCreateInfo(tokensData data.TokensHandler, co ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.GetTopic, shardID)) responseTokens := &data.ResponseTokens{} - err := ei.elasticClient.DoMultiGet(tokensData.GetAllTokens(), elasticIndexer.TokensIndex, true, responseTokens, ctxWithValue) + err := ei.elasticClient.DoMultiGet(ctxWithValue, tokensData.GetAllTokens(), elasticIndexer.TokensIndex, true, responseTokens) if err != nil { return err } @@ -731,7 +731,7 @@ func (ei *elasticProcessor) indexNFTBurnInfo(tokensData data.TokensHandler, buff ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.GetTopic, shardID)) responseTokens := &data.ResponseTokens{} - err := ei.elasticClient.DoMultiGet(tokensData.GetAllTokens(), elasticIndexer.TokensIndex, true, responseTokens, ctxWithValue) + err := ei.elasticClient.DoMultiGet(ctxWithValue, tokensData.GetAllTokens(), elasticIndexer.TokensIndex, true, responseTokens) if err != nil { return err } @@ -827,7 +827,7 @@ func (ei *elasticProcessor) doBulkRequests(index string, buffSlice []*bytes.Buff var err error for idx := range buffSlice { ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.BulkTopic, shardID)) - err = ei.elasticClient.DoBulkRequest(buffSlice[idx], index, ctxWithValue) + err = ei.elasticClient.DoBulkRequest(ctxWithValue, buffSlice[idx], index) if err != nil { return err } diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index 679d6ce3..bdd38f97 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -15,13 +15,13 @@ import ( // DatabaseClientHandler defines the actions that a component that handles requests should do type DatabaseClientHandler interface { - DoRequest(req *esapi.IndexRequest, ctx context.Context) error - DoBulkRequest(buff *bytes.Buffer, index string, ctx context.Context) error - DoQueryRemove(index string, buff *bytes.Buffer, ctx context.Context) error - DoMultiGet(ids []string, index string, withSource bool, res interface{}, ctx context.Context) error - DoScrollRequest(index string, body []byte, withSource bool, handlerFunc func(responseBytes []byte) error, ctx context.Context) error - DoCountRequest(index string, body []byte, ctx context.Context) (uint64, error) - UpdateByQuery(index string, buff *bytes.Buffer, ctx context.Context) error + DoRequest(ctx context.Context, req *esapi.IndexRequest) error + DoBulkRequest(ctx context.Context, buff *bytes.Buffer, index string) error + DoQueryRemove(ctx context.Context, index string, buff *bytes.Buffer) error + DoMultiGet(ctx context.Context, ids []string, index string, withSource bool, res interface{}) error + DoScrollRequest(ctx context.Context, index string, body []byte, withSource bool, handlerFunc func(responseBytes []byte) error) error + DoCountRequest(ctx context.Context, index string, body []byte) (uint64, error) + UpdateByQuery(ctx context.Context, index string, buff *bytes.Buffer) error CheckAndCreateIndex(index string) error CheckAndCreateAlias(alias string, index string) error diff --git a/process/elasticproc/updateTokenType.go b/process/elasticproc/updateTokenType.go index 1173ff04..18eded2e 100644 --- a/process/elasticproc/updateTokenType.go +++ b/process/elasticproc/updateTokenType.go @@ -75,13 +75,13 @@ func (ei *elasticProcessor) addTokenType(tokensData []*data.TokenInfo, index str ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.GetTopic, shardID)) query := fmt.Sprintf(`{"query": {"bool": {"must": [{"match": {"token": {"query": "%s","operator": "AND"}}}],"must_not":[{"exists": {"field": "type"}}]}}}`, td.Token) - resultsCount, err := ei.elasticClient.DoCountRequest(index, []byte(query), ctxWithValue) + resultsCount, err := ei.elasticClient.DoCountRequest(ctxWithValue, index, []byte(query)) if err != nil || resultsCount == 0 { return err } ctxWithValue = context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.ScrollTopic, shardID)) - err = ei.elasticClient.DoScrollRequest(index, []byte(query), false, handlerFunc, ctxWithValue) + err = ei.elasticClient.DoScrollRequest(ctxWithValue, index, []byte(query), false, handlerFunc) if err != nil { return err } diff --git a/scripts/observers/.env b/scripts/observers/.env index da050862..5d154fa8 100644 --- a/scripts/observers/.env +++ b/scripts/observers/.env @@ -2,6 +2,8 @@ MX_CHAIN_DEPLOY_GO_URL=https://github.com/multiversx/mx-chain-deploy-go MX_CHAIN_PROXY_URL=https://github.com/multiversx/mx-chain-proxy-go NODE_GO_URL="https://github.com/multiversx/mx-chain-go" + +# TODO change the branch after tests NODE_GO_BRANCH="indexer-with-metrics" WORKING_DIRECTORY="IndexerObservers" From f332f35bbc416bd7415fcc5da2ee161e63c3b48b Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 25 Jul 2023 12:12:23 +0300 Subject: [PATCH 134/189] fix integration tests --- .../accountsBalanceNftTransfer_test.go | 6 +++--- .../accountsBalanceWithLowerTimestamp_test.go | 16 ++++++++-------- integrationtests/accountsESDTRollback_test.go | 4 ++-- .../accountsESDTWithTokenType_test.go | 12 ++++++------ integrationtests/claimRewards_test.go | 2 +- integrationtests/createNFTWithTags_test.go | 8 ++++---- integrationtests/delegators_test.go | 10 +++++----- integrationtests/esdtTransfer_test.go | 2 +- integrationtests/issueTokenAndSetRoles_test.go | 12 ++++++------ integrationtests/issueToken_test.go | 12 ++++++------ integrationtests/logsCrossShard_test.go | 8 ++++---- ...ransferWithScCallAndErrorSignaledBySC_test.go | 4 ++-- integrationtests/nftIssueCreateBurn_test.go | 6 +++--- integrationtests/nftTransferCrossShard_test.go | 14 +++++++------- .../nftTransferCrossWithScCall_test.go | 2 +- integrationtests/relayedTx_test.go | 6 +++--- integrationtests/scCallIntraShard_test.go | 4 ++-- integrationtests/scCallWithIssueEsdt_test.go | 10 +++++----- integrationtests/scDeploy_test.go | 4 ++-- integrationtests/transactions_test.go | 2 +- integrationtests/updateNFT_test.go | 12 ++++++------ 21 files changed, 78 insertions(+), 78 deletions(-) diff --git a/integrationtests/accountsBalanceNftTransfer_test.go b/integrationtests/accountsBalanceNftTransfer_test.go index ff1dca7c..7dbd2023 100644 --- a/integrationtests/accountsBalanceNftTransfer_test.go +++ b/integrationtests/accountsBalanceNftTransfer_test.go @@ -95,7 +95,7 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { ids := []string{fmt.Sprintf("%s-NFT-abcdef-718863", addr)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.AccountsESDTIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsBalanceNftTransfer/balance-nft-after-create.json"), string(genericResponse.Docs[0].Source)) @@ -156,13 +156,13 @@ func TestAccountBalanceNFTTransfer(t *testing.T) { ids = []string{fmt.Sprintf("%s-NFT-abcdef-718863", addr)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.AccountsESDTIndex, true, genericResponse) require.Nil(t, err) require.False(t, genericResponse.Docs[0].Found) ids = []string{fmt.Sprintf("%s-NFT-abcdef-718863", addrReceiver)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.AccountsESDTIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsBalanceNftTransfer/balance-nft-after-transfer.json"), string(genericResponse.Docs[0].Source)) } diff --git a/integrationtests/accountsBalanceWithLowerTimestamp_test.go b/integrationtests/accountsBalanceWithLowerTimestamp_test.go index 7e0996b2..afc20028 100644 --- a/integrationtests/accountsBalanceWithLowerTimestamp_test.go +++ b/integrationtests/accountsBalanceWithLowerTimestamp_test.go @@ -84,13 +84,13 @@ func TestIndexAccountsBalance(t *testing.T) { ids := []string{addr} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.AccountsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsBalanceWithLowerTimestamp/account-balance-first-update.json"), string(genericResponse.Docs[0].Source)) ids = []string{fmt.Sprintf("%s-TTTT-abcd-00", addr)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.AccountsESDTIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsBalanceWithLowerTimestamp/account-balance-esdt-first-update.json"), string(genericResponse.Docs[0].Source)) @@ -107,13 +107,13 @@ func TestIndexAccountsBalance(t *testing.T) { ids = []string{addr} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.AccountsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsBalanceWithLowerTimestamp/account-balance-first-update.json"), string(genericResponse.Docs[0].Source)) ids = []string{fmt.Sprintf("%s-TTTT-abcd-00", addr)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.AccountsESDTIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsBalanceWithLowerTimestamp/account-balance-esdt-first-update.json"), string(genericResponse.Docs[0].Source)) @@ -169,13 +169,13 @@ func TestIndexAccountsBalance(t *testing.T) { ids = []string{addr} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.AccountsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsBalanceWithLowerTimestamp/account-balance-second-update.json"), string(genericResponse.Docs[0].Source)) ids = []string{fmt.Sprintf("%s-TTTT-abcd-00", addr)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.AccountsESDTIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsBalanceWithLowerTimestamp/account-balance-esdt-second-update.json"), string(genericResponse.Docs[0].Source)) @@ -204,13 +204,13 @@ func TestIndexAccountsBalance(t *testing.T) { ids = []string{addr} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.AccountsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsBalanceWithLowerTimestamp/account-balance-second-update.json"), string(genericResponse.Docs[0].Source)) ids = []string{fmt.Sprintf("%s-TTTT-abcd-00", addr)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.AccountsESDTIndex, true, genericResponse) require.Nil(t, err) require.False(t, genericResponse.Docs[0].Found) } diff --git a/integrationtests/accountsESDTRollback_test.go b/integrationtests/accountsESDTRollback_test.go index d848a041..9fd3a06f 100644 --- a/integrationtests/accountsESDTRollback_test.go +++ b/integrationtests/accountsESDTRollback_test.go @@ -86,7 +86,7 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { ids := []string{fmt.Sprintf("%s-TOKEN-eeee-02", addr)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.AccountsESDTIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsESDTRollback/account-after-create.json"), string(genericResponse.Docs[0].Source)) @@ -94,7 +94,7 @@ func TestAccountsESDTDeleteOnRollback(t *testing.T) { err = esProc.RemoveAccountsESDT(5040, 2) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.AccountsESDTIndex, true, genericResponse) require.Nil(t, err) require.False(t, genericResponse.Docs[0].Found) } diff --git a/integrationtests/accountsESDTWithTokenType_test.go b/integrationtests/accountsESDTWithTokenType_test.go index 4158521e..22bebdfd 100644 --- a/integrationtests/accountsESDTWithTokenType_test.go +++ b/integrationtests/accountsESDTWithTokenType_test.go @@ -63,7 +63,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { ids := []string{"SEMI-abcd"} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/token-after-issue.json"), string(genericResponse.Docs[0].Source)) @@ -125,7 +125,7 @@ func TestIndexAccountESDTWithTokenType(t *testing.T) { ids = []string{fmt.Sprintf("%s-SEMI-abcd-02", address)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.AccountsESDTIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/account-esdt.json"), string(genericResponse.Docs[0].Source)) @@ -198,7 +198,7 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) ids := []string{fmt.Sprintf("%s-TTTT-abcd-02", address)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.AccountsESDTIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/account-esdt-without-type.json"), string(genericResponse.Docs[0].Source)) @@ -238,19 +238,19 @@ func TestIndexAccountESDTWithTokenTypeShardFirstAndMetachainAfter(t *testing.T) ids = []string{"TTTT-abcd"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/semi-fungible-token.json"), string(genericResponse.Docs[0].Source)) ids = []string{fmt.Sprintf("%s-TTTT-abcd-02", address)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.AccountsESDTIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/account-esdt-with-type.json"), string(genericResponse.Docs[0].Source)) ids = []string{"TTTT-abcd-02"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/accountsESDTWithTokenType/semi-fungible-token-after-create.json"), string(genericResponse.Docs[0].Source)) } diff --git a/integrationtests/claimRewards_test.go b/integrationtests/claimRewards_test.go index 49ab0ba9..8901fe1f 100644 --- a/integrationtests/claimRewards_test.go +++ b/integrationtests/claimRewards_test.go @@ -128,7 +128,7 @@ func TestTransactionWithClaimRewardsGasRefund(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, diff --git a/integrationtests/createNFTWithTags_test.go b/integrationtests/createNFTWithTags_test.go index 52bb719b..c3824215 100644 --- a/integrationtests/createNFTWithTags_test.go +++ b/integrationtests/createNFTWithTags_test.go @@ -95,13 +95,13 @@ func TestCreateNFTWithTags(t *testing.T) { ids := []string{fmt.Sprintf("%s-DESK-abcd-01", address1)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.AccountsESDTIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.AccountsESDTIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/createNFTWithTags/accounts-esdt-address-balance.json"), string(genericResponse.Docs[0].Source)) ids = []string{"bXVzaWM=", "aGVsbG8=", "Z2FsbGVyeQ==", "ZG8=", "YXJ0", "c29tZXRoaW5n"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TagsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TagsIndex, true, genericResponse) require.Nil(t, err) tagsChecked := 0 @@ -142,7 +142,7 @@ func TestCreateNFTWithTags(t *testing.T) { require.Nil(t, err) genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TagsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TagsIndex, true, genericResponse) require.Nil(t, err) tagsChecked = 0 @@ -192,7 +192,7 @@ func TestCreateNFTWithTags(t *testing.T) { ids = append(ids, "XFxcXFxcXFxcXFxcXFxcXFxcXA==", "JycnJw==", "PDw8Pj4+JiYmJiYmJiYmJiYmJiYm") genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TagsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TagsIndex, true, genericResponse) require.Nil(t, err) tagsChecked = 0 diff --git a/integrationtests/delegators_test.go b/integrationtests/delegators_test.go index bb12baec..3517b26e 100644 --- a/integrationtests/delegators_test.go +++ b/integrationtests/delegators_test.go @@ -65,7 +65,7 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { ids := []string{"9v/pLAXxUZJ4Oy1U+x5al/Xg5sebh1dYCRTeZwg/u68="} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.DelegatorsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/delegators/delegator-after-delegate.json"), string(genericResponse.Docs[0].Source)) @@ -95,7 +95,7 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.DelegatorsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/delegators/delegator-after-un-delegate-1.json"), string(genericResponse.Docs[0].Source)) @@ -125,7 +125,7 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.DelegatorsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/delegators/delegator-after-un-delegate-2.json"), string(genericResponse.Docs[0].Source)) time.Sleep(time.Second) @@ -136,7 +136,7 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { require.Nil(t, err) time.Sleep(time.Second) - err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.DelegatorsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/delegators/delegator-after-revert.json"), string(genericResponse.Docs[0].Source)) @@ -165,7 +165,7 @@ func TestDelegateUnDelegateAndWithdraw(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.DelegatorsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.DelegatorsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/delegators/delegator-after-withdraw.json"), string(genericResponse.Docs[0].Source)) } diff --git a/integrationtests/esdtTransfer_test.go b/integrationtests/esdtTransfer_test.go index 9f549329..285802d8 100644 --- a/integrationtests/esdtTransfer_test.go +++ b/integrationtests/esdtTransfer_test.go @@ -108,7 +108,7 @@ func TestESDTTransferTooMuchGasProvided(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/esdtTransfer/esdt-transfer.json"), string(genericResponse.Docs[0].Source)) diff --git a/integrationtests/issueTokenAndSetRoles_test.go b/integrationtests/issueTokenAndSetRoles_test.go index a3ab5ab9..31ae30ca 100644 --- a/integrationtests/issueTokenAndSetRoles_test.go +++ b/integrationtests/issueTokenAndSetRoles_test.go @@ -64,7 +64,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { ids := []string{"TOK-abcd"} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-issue-ok.json"), string(genericResponse.Docs[0].Source)) @@ -94,7 +94,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { ids = []string{"TOK-abcd"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-set-role.json"), string(genericResponse.Docs[0].Source)) @@ -128,7 +128,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { ids = []string{"TOK-abcd"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-transfer-role.json"), string(genericResponse.Docs[0].Source)) @@ -158,7 +158,7 @@ func TestIssueTokenAndSetRole(t *testing.T) { ids = []string{"TOK-abcd"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-unset-role.json"), string(genericResponse.Docs[0].Source)) } @@ -206,7 +206,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { ids := []string{"TTT-abcd"} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-set-roles-first.json"), string(genericResponse.Docs[0].Source)) @@ -235,7 +235,7 @@ func TestIssueSetRolesEventAndAfterTokenIssue(t *testing.T) { ids = []string{"TTT-abcd"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueTokenAndSetRoles/token-after-set-roles-and-issue.json"), string(genericResponse.Docs[0].Source)) } diff --git a/integrationtests/issueToken_test.go b/integrationtests/issueToken_test.go index 4ee784fb..04243666 100644 --- a/integrationtests/issueToken_test.go +++ b/integrationtests/issueToken_test.go @@ -58,11 +58,11 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { ids := []string{"SSSS-abcd"} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueToken/token-semi.json"), string(genericResponse.Docs[0].Source)) - err = esClient.DoMultiGet(ids, indexerdata.ESDTsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.ESDTsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueToken/token-semi.json"), string(genericResponse.Docs[0].Source)) @@ -90,11 +90,11 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueToken/token-semi-after-transfer-ownership.json"), string(genericResponse.Docs[0].Source)) - err = esClient.DoMultiGet(ids, indexerdata.ESDTsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.ESDTsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueToken/token-semi-after-transfer-ownership.json"), string(genericResponse.Docs[0].Source)) @@ -122,7 +122,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueToken/token-semi-after-pause.json"), string(genericResponse.Docs[0].Source)) @@ -150,7 +150,7 @@ func TestIssueTokenAndTransferOwnership(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/issueToken/token-semi-after-un-pause.json"), string(genericResponse.Docs[0].Source)) } diff --git a/integrationtests/logsCrossShard_test.go b/integrationtests/logsCrossShard_test.go index 576fbf1a..aa62f94a 100644 --- a/integrationtests/logsCrossShard_test.go +++ b/integrationtests/logsCrossShard_test.go @@ -61,7 +61,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { ids := []string{logID} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.LogsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/logsCrossShard/log-at-source.json"), @@ -100,7 +100,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.LogsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/logsCrossShard/log-at-destination.json"), @@ -133,7 +133,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.LogsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/logsCrossShard/log-at-destination.json"), @@ -159,7 +159,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) { err = esProc.RemoveTransactions(header, body) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.LogsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.LogsIndex, true, genericResponse) require.Nil(t, err) require.False(t, genericResponse.Docs[0].Found) diff --git a/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go b/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go index 863466a9..9c037646 100644 --- a/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go +++ b/integrationtests/multiTransferWithScCallAndErrorSignaledBySC_test.go @@ -96,7 +96,7 @@ func TestMultiTransferCrossShardAndScCallErrorSignaledBySC(t *testing.T) { require.Nil(t, err) ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, @@ -165,7 +165,7 @@ func TestMultiTransferCrossShardAndScCallErrorSignaledBySC(t *testing.T) { ids = []string{hex.EncodeToString(txHash)} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, diff --git a/integrationtests/nftIssueCreateBurn_test.go b/integrationtests/nftIssueCreateBurn_test.go index 97df96c5..02fda26b 100644 --- a/integrationtests/nftIssueCreateBurn_test.go +++ b/integrationtests/nftIssueCreateBurn_test.go @@ -61,7 +61,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { ids := []string{"NON-abcd"} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/nftIssueCreateBurn/non-fungible-after-issue.json"), string(genericResponse.Docs[0].Source)) @@ -106,7 +106,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { ids = []string{"NON-abcd-02"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/nftIssueCreateBurn/non-fungible-after-create.json"), string(genericResponse.Docs[0].Source)) @@ -142,7 +142,7 @@ func TestIssueNFTCreateAndBurn(t *testing.T) { ids = []string{"NON-abcd-02"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.False(t, genericResponse.Docs[0].Found) } diff --git a/integrationtests/nftTransferCrossShard_test.go b/integrationtests/nftTransferCrossShard_test.go index abc27611..45c961e7 100644 --- a/integrationtests/nftTransferCrossShard_test.go +++ b/integrationtests/nftTransferCrossShard_test.go @@ -98,7 +98,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, @@ -136,7 +136,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, @@ -145,7 +145,7 @@ func TestNFTTransferCrossShardWithSCCall(t *testing.T) { ) genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.OperationsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.OperationsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/nftTransferCrossShard/op-nft-transfer-sc-call-after-refund.json"), @@ -242,7 +242,7 @@ func TestNFTTransferCrossShard(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, @@ -292,7 +292,7 @@ func TestNFTTransferCrossShard(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, @@ -374,7 +374,7 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, @@ -443,7 +443,7 @@ func TestNFTTransferCrossShardImportDBScenarioFirstIndexDestinationAfterSource(t err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, diff --git a/integrationtests/nftTransferCrossWithScCall_test.go b/integrationtests/nftTransferCrossWithScCall_test.go index 196606fe..5210506b 100644 --- a/integrationtests/nftTransferCrossWithScCall_test.go +++ b/integrationtests/nftTransferCrossWithScCall_test.go @@ -93,7 +93,7 @@ func TestNFTTransferCrossShardWithScCall(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/nftTransferCrossShardWithScCall/cross-shard-transfer-with-sc-call.json"), diff --git a/integrationtests/relayedTx_test.go b/integrationtests/relayedTx_test.go index 4de8b2af..67b758bc 100644 --- a/integrationtests/relayedTx_test.go +++ b/integrationtests/relayedTx_test.go @@ -96,7 +96,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, @@ -136,7 +136,7 @@ func TestRelayedTransactionGasUsedCrossShard(t *testing.T) { err = esProc.SaveTransactions(createOutportBlockWithHeader(bodyDstShard, header, poolDstShard, nil, testNumOfShards)) require.Nil(t, err) - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, @@ -237,7 +237,7 @@ func TestRelayedTransactionIntraShard(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, diff --git a/integrationtests/scCallIntraShard_test.go b/integrationtests/scCallIntraShard_test.go index e8c5f2f3..ce31024d 100644 --- a/integrationtests/scCallIntraShard_test.go +++ b/integrationtests/scCallIntraShard_test.go @@ -103,7 +103,7 @@ func TestTransactionWithSCCallFail(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerData.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, @@ -183,7 +183,7 @@ func TestTransactionWithScCallSuccess(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerData.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, diff --git a/integrationtests/scCallWithIssueEsdt_test.go b/integrationtests/scCallWithIssueEsdt_test.go index c1b9234e..3c3b5548 100644 --- a/integrationtests/scCallWithIssueEsdt_test.go +++ b/integrationtests/scCallWithIssueEsdt_test.go @@ -102,7 +102,7 @@ func TestScCallIntraShardWithIssueESDT(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerData.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, @@ -111,7 +111,7 @@ func TestScCallIntraShardWithIssueESDT(t *testing.T) { ) ids = []string{hex.EncodeToString(scrWithIssueHash)} - err = esClient.DoMultiGet(ids, indexerData.OperationsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerData.OperationsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, @@ -164,7 +164,7 @@ func TestScCallIntraShardWithIssueESDT(t *testing.T) { require.Nil(t, err) ids = []string{hex.EncodeToString(scrWithIssueHash), hex.EncodeToString(scrWithCallBackHash)} - err = esClient.DoMultiGet(ids, indexerData.OperationsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerData.OperationsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, @@ -217,7 +217,7 @@ func TestScCallIntraShardWithIssueESDT(t *testing.T) { require.Nil(t, err) ids = []string{hex.EncodeToString(txHash)} - err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerData.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, @@ -226,7 +226,7 @@ func TestScCallIntraShardWithIssueESDT(t *testing.T) { ) ids = []string{hex.EncodeToString(txHash), hex.EncodeToString(scrWithCallBackHash)} - err = esClient.DoMultiGet(ids, indexerData.OperationsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerData.OperationsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, diff --git a/integrationtests/scDeploy_test.go b/integrationtests/scDeploy_test.go index 4d6f0f2a..d14dfe8e 100644 --- a/integrationtests/scDeploy_test.go +++ b/integrationtests/scDeploy_test.go @@ -88,7 +88,7 @@ func TestTransactionWithSCDeploy(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerData.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, @@ -97,7 +97,7 @@ func TestTransactionWithSCDeploy(t *testing.T) { ) ids = []string{"erd1qqqqqqqqqqqqqpgq4t2tqxpst9a6qttpak8cz8wvz6a0nses5qfqel6rhy"} - err = esClient.DoMultiGet(ids, indexerData.SCDeploysIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerData.SCDeploysIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, diff --git a/integrationtests/transactions_test.go b/integrationtests/transactions_test.go index 96a041c6..48e19b21 100644 --- a/integrationtests/transactions_test.go +++ b/integrationtests/transactions_test.go @@ -69,7 +69,7 @@ func TestElasticIndexerSaveTransactions(t *testing.T) { ids := []string{hex.EncodeToString(txHash)} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerData.TransactionsIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerData.TransactionsIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, diff --git a/integrationtests/updateNFT_test.go b/integrationtests/updateNFT_test.go index 923906b1..898117db 100644 --- a/integrationtests/updateNFT_test.go +++ b/integrationtests/updateNFT_test.go @@ -65,7 +65,7 @@ func TestNFTUpdateMetadata(t *testing.T) { ids := []string{"NFT-abcd-0e"} genericResponse := &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/updateNFT/token.json"), string(genericResponse.Docs[0].Source)) @@ -114,7 +114,7 @@ func TestNFTUpdateMetadata(t *testing.T) { // Update attributes 1 ids = []string{"NFT-abcd-0e"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/updateNFT/token-after-add-uris.json"), string(genericResponse.Docs[0].Source)) @@ -140,7 +140,7 @@ func TestNFTUpdateMetadata(t *testing.T) { ids = []string{"NFT-abcd-0e"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/updateNFT/token-after-update-attributes.json"), string(genericResponse.Docs[0].Source)) @@ -168,7 +168,7 @@ func TestNFTUpdateMetadata(t *testing.T) { ids = []string{"NFT-abcd-0e"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/updateNFT/token-after-update-attributes-second.json"), string(genericResponse.Docs[0].Source)) @@ -194,7 +194,7 @@ func TestNFTUpdateMetadata(t *testing.T) { require.Nil(t, err) ids = []string{"NFT-abcd-0e"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/updateNFT/token-after-freeze.json"), string(genericResponse.Docs[0].Source)) @@ -220,7 +220,7 @@ func TestNFTUpdateMetadata(t *testing.T) { require.Nil(t, err) ids = []string{"NFT-abcd-0e"} genericResponse = &GenericResponse{} - err = esClient.DoMultiGet(ids, indexerdata.TokensIndex, true, genericResponse, context.Background()) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TokensIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, readExpectedResult("./testdata/updateNFT/token-after-un-freeze.json"), string(genericResponse.Docs[0].Source)) } From 9ea8ef565e9e7b73061148d14945d79117cd793c Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 25 Jul 2023 12:18:25 +0300 Subject: [PATCH 135/189] proper tag --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f3665175..2f6b6236 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/multiversx/mx-chain-communication-go v1.0.3 - github.com/multiversx/mx-chain-core-go v1.2.10-0.20230712104444-e818eaa01c7e + github.com/multiversx/mx-chain-core-go v1.2.11 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.1 github.com/stretchr/testify v1.8.2 diff --git a/go.sum b/go.sum index c7a2f55c..ac4cb910 100644 --- a/go.sum +++ b/go.sum @@ -476,8 +476,8 @@ github.com/multiversx/mx-chain-communication-go v1.0.3/go.mod h1:7oTI77XfWmRWwVE github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.5/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= -github.com/multiversx/mx-chain-core-go v1.2.10-0.20230712104444-e818eaa01c7e h1:ZITU8iHolDswJGMToGHIO7Wc8K6p6k6rsaxV/Z8UO3E= -github.com/multiversx/mx-chain-core-go v1.2.10-0.20230712104444-e818eaa01c7e/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= +github.com/multiversx/mx-chain-core-go v1.2.11 h1:C5W6P696SwTb8H0wvCUGX/+Ix5hhF4iDAmFXnDM0Jdo= +github.com/multiversx/mx-chain-core-go v1.2.11/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-crypto-go v1.2.6/go.mod h1:rOj0Rr19HTOYt9YTeym7RKxlHt91NXln3LVKjHKVmA0= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= From 337df6bd76197f44d724221406483674074bf902 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 27 Jul 2023 10:39:13 +0300 Subject: [PATCH 136/189] fixes after merge --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 78f32077..b40e7657 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.0 github.com/multiversx/mx-chain-communication-go v1.0.3 - github.com/multiversx/mx-chain-core-go v1.2.10-0.20230719111700-ba37de156180 + github.com/multiversx/mx-chain-core-go v1.2.12-0.20230725101425-a34bd6d3e3b9 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.1 github.com/prometheus/client_model v0.2.0 diff --git a/go.sum b/go.sum index 1627e93f..5b4c6d8b 100644 --- a/go.sum +++ b/go.sum @@ -525,8 +525,8 @@ github.com/multiversx/mx-chain-communication-go v1.0.3/go.mod h1:7oTI77XfWmRWwVE github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.5/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= -github.com/multiversx/mx-chain-core-go v1.2.10-0.20230719111700-ba37de156180 h1:SZkaCv6L/3EBEELkAr+F/HCCrYgia/OLVgFHFQSggJw= -github.com/multiversx/mx-chain-core-go v1.2.10-0.20230719111700-ba37de156180/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= +github.com/multiversx/mx-chain-core-go v1.2.12-0.20230725101425-a34bd6d3e3b9 h1:oa48Q+QpF/3LHVR0ug7eFENoIVKKZL9PfldABT9cL3A= +github.com/multiversx/mx-chain-core-go v1.2.12-0.20230725101425-a34bd6d3e3b9/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-crypto-go v1.2.6/go.mod h1:rOj0Rr19HTOYt9YTeym7RKxlHt91NXln3LVKjHKVmA0= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= From 5941763f71cb95e9f5dec9a348f20a7c370f9c4e Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 27 Jul 2023 12:24:49 +0300 Subject: [PATCH 137/189] refactor the indexing part of the miniblocks --- client/elasticClient.go | 10 ---- process/elasticproc/elasticProcessor.go | 57 +------------------ process/elasticproc/interface.go | 4 +- process/elasticproc/miniblocks/serialize.go | 41 ++++++++----- .../elasticproc/miniblocks/serialize_test.go | 40 ++++++------- 5 files changed, 44 insertions(+), 108 deletions(-) diff --git a/client/elasticClient.go b/client/elasticClient.go index 8c9ba482..3f0a66ed 100644 --- a/client/elasticClient.go +++ b/client/elasticClient.go @@ -93,16 +93,6 @@ func (ec *elasticClient) CheckAndCreateAlias(alias string, indexName string) err return ec.createAlias(alias, indexName) } -// DoRequest will do a request to elastic server -func (ec *elasticClient) DoRequest(ctx context.Context, req *esapi.IndexRequest) error { - res, err := req.Do(ctx, ec.client) - if err != nil { - return err - } - - return parseResponse(res, nil, elasticDefaultErrorResponseHandler) -} - // DoBulkRequest will do a bulk of request to elastic server func (ec *elasticClient) DoBulkRequest(ctx context.Context, buff *bytes.Buffer, index string) error { reader := bytes.NewReader(buff.Bytes()) diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index 9b0c3fff..5baad078 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -22,13 +22,6 @@ import ( logger "github.com/multiversx/mx-chain-logger-go" ) -const ( - docsKey = "docs" - errorKey = "error" - idKey = "_id" - foundKey = "found" -) - var ( log = logger.GetOrCreate("indexer/process") @@ -207,40 +200,6 @@ func (ei *elasticProcessor) createAliases() error { return nil } -func (ei *elasticProcessor) getExistingObjMap(hashes []string, index string, shardID uint32) (map[string]bool, error) { - if len(hashes) == 0 { - return make(map[string]bool), nil - } - - ctx := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.GetTopic, shardID)) - response := make(objectsMap) - err := ei.elasticClient.DoMultiGet(ctx, hashes, index, false, &response) - if err != nil { - return make(map[string]bool), err - } - - return getDecodedResponseMultiGet(response), nil -} - -func getDecodedResponseMultiGet(response objectsMap) map[string]bool { - founded := make(map[string]bool) - interfaceSlice, ok := response[docsKey].([]interface{}) - if !ok { - return founded - } - - for _, element := range interfaceSlice { - obj := element.(objectsMap) - _, ok = obj[errorKey] - if ok { - continue - } - founded[obj[idKey].(string)] = obj[foundKey].(bool) - } - - return founded -} - func getTemplateByName(templateName string, templateList map[string]*bytes.Buffer) *bytes.Buffer { if template, ok := templateList[templateName]; ok { return template @@ -400,26 +359,12 @@ func (ei *elasticProcessor) SaveMiniblocks(header coreData.HeaderHandler, miniBl return nil } - miniblocksInDBMap, err := ei.miniblocksInDBMap(mbs, header.GetShardID()) - if err != nil { - log.Warn("elasticProcessor.SaveMiniblocks cannot get indexed miniblocks", "error", err) - } - buffSlice := data.NewBufferSlice(ei.bulkRequestMaxSize) - ei.miniblocksProc.SerializeBulkMiniBlocks(mbs, miniblocksInDBMap, buffSlice, elasticIndexer.MiniblocksIndex, header.GetShardID()) + ei.miniblocksProc.SerializeBulkMiniBlocks(mbs, buffSlice, elasticIndexer.MiniblocksIndex, header.GetShardID()) return ei.doBulkRequests("", buffSlice.Buffers(), header.GetShardID()) } -func (ei *elasticProcessor) miniblocksInDBMap(mbs []*data.Miniblock, shardID uint32) (map[string]bool, error) { - mbsHashes := make([]string, len(mbs)) - for idx := range mbs { - mbsHashes[idx] = mbs[idx].Hash - } - - return ei.getExistingObjMap(mbsHashes, elasticIndexer.MiniblocksIndex, shardID) -} - // SaveTransactions will prepare and save information about a transactions in elasticsearch server func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader) error { headerTimestamp := obh.Header.GetTimeStamp() diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index 0164b4a3..829a306c 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -4,7 +4,6 @@ import ( "bytes" "context" - "github.com/elastic/go-elasticsearch/v7/esapi" coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/alteredAccount" "github.com/multiversx/mx-chain-core-go/data/block" @@ -15,7 +14,6 @@ import ( // DatabaseClientHandler defines the actions that a component that handles requests should do type DatabaseClientHandler interface { - DoRequest(ctx context.Context, req *esapi.IndexRequest) error DoBulkRequest(ctx context.Context, buff *bytes.Buffer, index string) error DoQueryRemove(ctx context.Context, index string, buff *bytes.Buffer) error DoMultiGet(ctx context.Context, ids []string, index string, withSource bool, res interface{}) error @@ -77,7 +75,7 @@ type DBMiniblocksHandler interface { PrepareDBMiniblocks(header coreData.HeaderHandler, miniBlocks []*block.MiniBlock) []*data.Miniblock GetMiniblocksHashesHexEncoded(header coreData.HeaderHandler, body *block.Body) []string - SerializeBulkMiniBlocks(bulkMbs []*data.Miniblock, mbsInDB map[string]bool, buffSlice *data.BufferSlice, index string, shardID uint32) + SerializeBulkMiniBlocks(bulkMbs []*data.Miniblock, buffSlice *data.BufferSlice, index string, shardID uint32) } // DBStatisticsHandler defines the actions that a database statistics handler should do diff --git a/process/elasticproc/miniblocks/serialize.go b/process/elasticproc/miniblocks/serialize.go index 20844538..08385b91 100644 --- a/process/elasticproc/miniblocks/serialize.go +++ b/process/elasticproc/miniblocks/serialize.go @@ -12,13 +12,12 @@ import ( // SerializeBulkMiniBlocks will serialize the provided miniblocks slice in a way that Elasticsearch expects a bulk request func (mp *miniblocksProcessor) SerializeBulkMiniBlocks( bulkMbs []*data.Miniblock, - existsInDb map[string]bool, buffSlice *data.BufferSlice, index string, shardID uint32, ) { for _, mb := range bulkMbs { - meta, serializedData, err := mp.prepareMiniblockData(mb, existsInDb[mb.Hash], index, shardID) + meta, serializedData, err := mp.prepareMiniblockData(mb, index, shardID) if err != nil { log.Warn("miniblocksProcessor.prepareMiniblockData cannot prepare miniblock data", "error", err) continue @@ -32,28 +31,40 @@ func (mp *miniblocksProcessor) SerializeBulkMiniBlocks( } } -func (mp *miniblocksProcessor) prepareMiniblockData(miniblockDB *data.Miniblock, isInDB bool, index string, shardID uint32) ([]byte, []byte, error) { +func (mp *miniblocksProcessor) prepareMiniblockData(miniblockDB *data.Miniblock, index string, shardID uint32) ([]byte, []byte, error) { mbHash := miniblockDB.Hash miniblockDB.Hash = "" - if !isInDB { - meta := []byte(fmt.Sprintf(`{ "index" : { "_index":"%s", "_id" : "%s"} }%s`, index, converters.JsonEscape(mbHash), "\n")) - serializedData, err := json.Marshal(miniblockDB) - - return meta, serializedData, err + mbBytes, errMarshal := json.Marshal(miniblockDB) + if errMarshal != nil { + return nil, nil, errMarshal } // prepare data for update operation meta := []byte(fmt.Sprintf(`{ "update" : {"_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(mbHash), "\n")) - if shardID == miniblockDB.SenderShardID && miniblockDB.ProcessingTypeOnDestination != block.Processed.String() { - // prepare for update sender block hash - serializedData := []byte(fmt.Sprintf(`{ "doc" : { "senderBlockHash" : "%s", "procTypeS": "%s" } }`, converters.JsonEscape(miniblockDB.SenderBlockHash), converters.JsonEscape(miniblockDB.ProcessingTypeOnSource))) - return meta, serializedData, nil + onSourceNotProcessed := shardID == miniblockDB.SenderShardID && miniblockDB.ProcessingTypeOnDestination != block.Processed.String() + codeToExecute := ` + if ('create' == ctx.op) { + ctx._source = params.mb + } else { + if (params.osnp) { + ctx._source.senderBlockHash = params.mb.senderBlockHash; + ctx._source.procTypeS = params.mb.procTypeS; + } else { + ctx._source.receiverBlockHash = params.mb.receiverBlockHash; + ctx._source.procTypeD = params.mb.procTypeD; + } } +` - // prepare for update receiver block hash - serializedData := []byte(fmt.Sprintf(`{ "doc" : { "receiverBlockHash" : "%s", "procTypeD": "%s" } }`, converters.JsonEscape(miniblockDB.ReceiverBlockHash), converters.JsonEscape(miniblockDB.ProcessingTypeOnDestination))) + serializedDataStr := fmt.Sprintf(`{"scripted_upsert": true, "script": {`+ + `"source": "%s",`+ + `"lang": "painless",`+ + `"params": { "mb": %s, "osnp": %t }},`+ + `"upsert": {}}`, + converters.FormatPainlessSource(codeToExecute), mbBytes, onSourceNotProcessed, + ) - return meta, serializedData, nil + return meta, []byte(serializedDataStr), nil } diff --git a/process/elasticproc/miniblocks/serialize_test.go b/process/elasticproc/miniblocks/serialize_test.go index 5ce07e53..44d7fdd9 100644 --- a/process/elasticproc/miniblocks/serialize_test.go +++ b/process/elasticproc/miniblocks/serialize_test.go @@ -20,12 +20,12 @@ func TestMiniblocksProcessor_SerializeBulkMiniBlocks(t *testing.T) { } buffSlice := data.NewBufferSlice(data.DefaultMaxBulkSize) - mp.SerializeBulkMiniBlocks(miniblocks, nil, buffSlice, "miniblocks", 0) + mp.SerializeBulkMiniBlocks(miniblocks, buffSlice, "miniblocks", 0) - expectedBuff := `{ "index" : { "_index":"miniblocks", "_id" : "h1"} } -{"senderShard":0,"receiverShard":1,"type":"","timestamp":0} -{ "index" : { "_index":"miniblocks", "_id" : "h2"} } -{"senderShard":0,"receiverShard":2,"type":"","timestamp":0} + expectedBuff := `{ "update" : {"_index":"miniblocks", "_id" : "h1" } } +{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.osnp) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":0,"receiverShard":1,"type":"","timestamp":0}, "osnp": true }},"upsert": {}} +{ "update" : {"_index":"miniblocks", "_id" : "h2" } } +{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.osnp) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":0,"receiverShard":2,"type":"","timestamp":0}, "osnp": true }},"upsert": {}} ` require.Equal(t, expectedBuff, buffSlice.Buffers()[0].String()) } @@ -41,14 +41,12 @@ func TestMiniblocksProcessor_SerializeBulkMiniBlocksInDB(t *testing.T) { } buffSlice := data.NewBufferSlice(data.DefaultMaxBulkSize) - mp.SerializeBulkMiniBlocks(miniblocks, map[string]bool{ - "h1": true, - }, buffSlice, "miniblocks", 0) + mp.SerializeBulkMiniBlocks(miniblocks, buffSlice, "miniblocks", 0) expectedBuff := `{ "update" : {"_index":"miniblocks", "_id" : "h1" } } -{ "doc" : { "senderBlockHash" : "", "procTypeS": "" } } -{ "index" : { "_index":"miniblocks", "_id" : "h2"} } -{"senderShard":0,"receiverShard":2,"type":"","timestamp":0} +{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.osnp) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":0,"receiverShard":1,"type":"","timestamp":0}, "osnp": true }},"upsert": {}} +{ "update" : {"_index":"miniblocks", "_id" : "h2" } } +{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.osnp) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":0,"receiverShard":2,"type":"","timestamp":0}, "osnp": true }},"upsert": {}} ` require.Equal(t, expectedBuff, buffSlice.Buffers()[0].String()) } @@ -61,12 +59,10 @@ func TestSerializeMiniblock_CrossShardNormal(t *testing.T) { } buffSlice := data.NewBufferSlice(data.DefaultMaxBulkSize) - mp.SerializeBulkMiniBlocks(miniblocks, map[string]bool{ - "h1": true, - }, buffSlice, "miniblocks", 1) + mp.SerializeBulkMiniBlocks(miniblocks, buffSlice, "miniblocks", 1) expectedBuff := `{ "update" : {"_index":"miniblocks", "_id" : "h1" } } -{ "doc" : { "receiverBlockHash" : "receiverBlock", "procTypeD": "" } } +{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.osnp) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":0,"receiverShard":1,"receiverBlockHash":"receiverBlock","type":"","timestamp":0}, "osnp": false }},"upsert": {}} ` require.Equal(t, expectedBuff, buffSlice.Buffers()[0].String()) } @@ -80,12 +76,10 @@ func TestSerializeMiniblock_IntraShardScheduled(t *testing.T) { } buffSlice := data.NewBufferSlice(data.DefaultMaxBulkSize) - mp.SerializeBulkMiniBlocks(miniblocks, map[string]bool{ - "h1": false, - }, buffSlice, "miniblocks", 1) + mp.SerializeBulkMiniBlocks(miniblocks, buffSlice, "miniblocks", 1) - expectedBuff := `{ "index" : { "_index":"miniblocks", "_id" : "h1"} } -{"senderShard":1,"receiverShard":1,"senderBlockHash":"senderBlock","type":"","procTypeS":"Scheduled","timestamp":0} + expectedBuff := `{ "update" : {"_index":"miniblocks", "_id" : "h1" } } +{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.osnp) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":1,"receiverShard":1,"senderBlockHash":"senderBlock","type":"","procTypeS":"Scheduled","timestamp":0}, "osnp": true }},"upsert": {}} ` require.Equal(t, expectedBuff, buffSlice.Buffers()[0].String()) @@ -95,12 +89,10 @@ func TestSerializeMiniblock_IntraShardScheduled(t *testing.T) { } buffSlice = data.NewBufferSlice(data.DefaultMaxBulkSize) - mp.SerializeBulkMiniBlocks(miniblocks, map[string]bool{ - "h1": true, - }, buffSlice, "miniblocks", 1) + mp.SerializeBulkMiniBlocks(miniblocks, buffSlice, "miniblocks", 1) expectedBuff = `{ "update" : {"_index":"miniblocks", "_id" : "h1" } } -{ "doc" : { "receiverBlockHash" : "receiverBlock", "procTypeD": "Processed" } } +{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.osnp) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":1,"receiverShard":1,"receiverBlockHash":"receiverBlock","type":"","procTypeD":"Processed","timestamp":0}, "osnp": false }},"upsert": {}} ` require.Equal(t, expectedBuff, buffSlice.Buffers()[0].String()) } From 357160107fd8976a20499a7ab05154d05d4c0d86 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 27 Jul 2023 12:34:09 +0300 Subject: [PATCH 138/189] remove unused function --- mock/databaseWriterStub.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/mock/databaseWriterStub.go b/mock/databaseWriterStub.go index 23dd74ea..9a41e0ff 100644 --- a/mock/databaseWriterStub.go +++ b/mock/databaseWriterStub.go @@ -3,13 +3,10 @@ package mock import ( "bytes" "context" - - "github.com/elastic/go-elasticsearch/v7/esapi" ) // DatabaseWriterStub - type DatabaseWriterStub struct { - DoRequestCalled func(req *esapi.IndexRequest) error DoBulkRequestCalled func(buff *bytes.Buffer, index string) error DoQueryRemoveCalled func(index string, body *bytes.Buffer) error DoMultiGetCalled func(ids []string, index string, withSource bool, response interface{}) error @@ -35,14 +32,6 @@ func (dwm *DatabaseWriterStub) DoScrollRequest(_ context.Context, index string, return nil } -// DoRequest - -func (dwm *DatabaseWriterStub) DoRequest(_ context.Context, req *esapi.IndexRequest) error { - if dwm.DoRequestCalled != nil { - return dwm.DoRequestCalled(req) - } - return nil -} - // DoBulkRequest - func (dwm *DatabaseWriterStub) DoBulkRequest(_ context.Context, buff *bytes.Buffer, index string) error { if dwm.DoBulkRequestCalled != nil { From 8437d47bdf7ef1165b21fa29e309df1fbdf1ee14 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 27 Jul 2023 12:35:04 +0300 Subject: [PATCH 139/189] remove old tests --- process/elasticproc/elasticProcessor_test.go | 83 -------------------- 1 file changed, 83 deletions(-) diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index 7a533f60..381db5db 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -3,15 +3,10 @@ package elasticproc import ( "bytes" "encoding/hex" - "encoding/json" "errors" - "fmt" - "io/ioutil" - "strconv" "strings" "testing" - "github.com/elastic/go-elasticsearch/v7/esapi" "github.com/multiversx/mx-chain-core-go/core" coreData "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" @@ -341,45 +336,6 @@ func TestElasticseachDatabaseSaveHeader_RequestError(t *testing.T) { require.Equal(t, localErr, err) } -func TestElasticseachDatabaseSaveHeader_CheckRequestBody(t *testing.T) { - outportBlock := createEmptyOutportBlockWithHeader() - - miniBlock := &dataBlock.MiniBlock{ - Type: dataBlock.TxBlock, - } - blockBody := &dataBlock.Body{ - MiniBlocks: []*dataBlock.MiniBlock{ - miniBlock, - }, - } - outportBlock.BlockData.Body = blockBody - - arguments := createMockElasticProcessorArgs() - - mbHash, _ := core.CalculateHash(&mock.MarshalizerMock{}, &mock.HasherMock{}, miniBlock) - hexEncodedHash := hex.EncodeToString(mbHash) - - dbWriter := &mock.DatabaseWriterStub{ - DoRequestCalled: func(req *esapi.IndexRequest) error { - require.Equal(t, dataindexer.BlockIndex, req.Index) - - var bl data.Block - blockBytes, _ := ioutil.ReadAll(req.Body) - _ = json.Unmarshal(blockBytes, &bl) - require.Equal(t, outportBlock.Header.GetNonce(), bl.Nonce) - require.Equal(t, hexEncodedHash, bl.MiniBlocksHashes[0]) - require.Equal(t, outportBlock.SignersIndexes, bl.Validators) - - return nil - }, - } - - arguments.BlockProc, _ = block.NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) - elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) - err := elasticDatabase.SaveHeader(outportBlock) - require.Nil(t, err) -} - func TestElasticseachSaveTransactions(t *testing.T) { localErr := errors.New("localErr") arguments := createMockElasticProcessorArgs() @@ -482,45 +438,6 @@ func TestElasticsearch_saveShardValidatorsPubKeys_RequestError(t *testing.T) { require.Equal(t, localErr, err) } -func TestElasticsearch_saveShardValidatorsPubKeys(t *testing.T) { - shardID := uint32(0) - epoch := uint32(0) - valPubKeys := [][]byte{[]byte("key1"), []byte("key2")} - arguments := createMockElasticProcessorArgs() - dbWriter := &mock.DatabaseWriterStub{ - DoRequestCalled: func(req *esapi.IndexRequest) error { - require.Equal(t, fmt.Sprintf("%d_%d", shardID, epoch), req.DocumentID) - return nil - }, - } - elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) - - err := elasticDatabase.SaveShardValidatorsPubKeys(&outport.ValidatorsPubKeys{ - Epoch: epoch, - ShardValidatorsPubKeys: map[uint32]*outport.PubKeys{ - shardID: {Keys: valPubKeys}, - }, - }) - require.Nil(t, err) -} - -func TestElasticsearch_saveRoundInfo(t *testing.T) { - roundInfo := &outport.RoundInfo{ - Round: 1, ShardId: 0, BlockWasProposed: true, - } - arguments := createMockElasticProcessorArgs() - dbWriter := &mock.DatabaseWriterStub{ - DoRequestCalled: func(req *esapi.IndexRequest) error { - require.Equal(t, strconv.FormatUint(uint64(roundInfo.ShardId), 10)+"_"+strconv.FormatUint(roundInfo.Round, 10), req.DocumentID) - return nil - }, - } - elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) - - err := elasticDatabase.SaveRoundsInfo(&outport.RoundsInfo{RoundsInfo: []*outport.RoundInfo{roundInfo}}) - require.Nil(t, err) -} - func TestElasticsearch_saveRoundInfoRequestError(t *testing.T) { roundInfo := &outport.RoundInfo{} localError := errors.New("local err") From 6b7837c83f497f151fa52e0311672f2f4fe4c0bf Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 27 Jul 2023 15:22:40 +0300 Subject: [PATCH 140/189] integration tests miniblocks indexing --- integrationtests/miniblocks_test.go | 122 ++++++++++++++++++ .../cross-miniblock-on-destination-first.json | 8 ++ .../cross-miniblock-on-destination.json | 10 ++ .../cross-miniblock-on-source-second.json | 10 ++ .../miniblocks/cross-miniblock-on-source.json | 8 ++ integrationtests/utils.go | 2 +- .../miniblocks/miniblocksProcessor.go | 9 +- 7 files changed, 165 insertions(+), 4 deletions(-) create mode 100644 integrationtests/miniblocks_test.go create mode 100644 integrationtests/testdata/miniblocks/cross-miniblock-on-destination-first.json create mode 100644 integrationtests/testdata/miniblocks/cross-miniblock-on-destination.json create mode 100644 integrationtests/testdata/miniblocks/cross-miniblock-on-source-second.json create mode 100644 integrationtests/testdata/miniblocks/cross-miniblock-on-source.json diff --git a/integrationtests/miniblocks_test.go b/integrationtests/miniblocks_test.go new file mode 100644 index 00000000..c6dd0381 --- /dev/null +++ b/integrationtests/miniblocks_test.go @@ -0,0 +1,122 @@ +package integrationtests + +import ( + "context" + "testing" + + dataBlock "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-core-go/marshal" + indexerdata "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/stretchr/testify/require" +) + +func TestIndexMiniBlocksOnSourceAndDestination(t *testing.T) { + setLogLevelDebug() + + esClient, err := createESClient(esURL) + require.Nil(t, err) + esProc, err := CreateElasticProcessor(esClient) + require.Nil(t, err) + + // index on the source shard + header := &dataBlock.Header{ + ShardID: 1, + TimeStamp: 1234, + } + miniBlocks := []*dataBlock.MiniBlock{ + { + SenderShardID: 1, + ReceiverShardID: 2, + }, + } + err = esProc.SaveMiniblocks(header, miniBlocks) + require.Nil(t, err) + mbHash := "11a1bb4065e16a2e93b2b5ac5957b7b69f1cfba7579b170b24f30dab2d3162e0" + ids := []string{mbHash} + genericResponse := &GenericResponse{} + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.MiniblocksIndex, true, genericResponse) + require.Nil(t, err) + require.JSONEq(t, readExpectedResult("./testdata/miniblocks/cross-miniblock-on-source.json"), string(genericResponse.Docs[0].Source)) + + // index on the destination shard + mbhr := &dataBlock.MiniBlockHeaderReserved{ + ExecutionType: dataBlock.ProcessingType(1), + } + + marshaller := &marshal.GogoProtoMarshalizer{} + mbhrBytes, _ := marshaller.Marshal(mbhr) + header = &dataBlock.Header{ + ShardID: 2, + TimeStamp: 1234, + MiniBlockHeaders: []dataBlock.MiniBlockHeader{ + { + Reserved: mbhrBytes, + }, + }, + } + + err = esProc.SaveMiniblocks(header, miniBlocks) + require.Nil(t, err) + + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.MiniblocksIndex, true, genericResponse) + require.Nil(t, err) + require.JSONEq(t, readExpectedResult("./testdata/miniblocks/cross-miniblock-on-destination.json"), string(genericResponse.Docs[0].Source)) +} + +func TestIndexMiniBlockFirstOnDestinationAndAfterSource(t *testing.T) { + setLogLevelDebug() + + esClient, err := createESClient(esURL) + require.Nil(t, err) + esProc, err := CreateElasticProcessor(esClient) + require.Nil(t, err) + + // index on destination + mbhr := &dataBlock.MiniBlockHeaderReserved{ + ExecutionType: dataBlock.ProcessingType(2), + } + + marshaller := &marshal.GogoProtoMarshalizer{} + mbhrBytes, _ := marshaller.Marshal(mbhr) + header := &dataBlock.Header{ + ShardID: 0, + TimeStamp: 54321, + MiniBlockHeaders: []dataBlock.MiniBlockHeader{ + { + Reserved: mbhrBytes, + }, + }, + } + miniBlocks := []*dataBlock.MiniBlock{ + { + SenderShardID: 2, + ReceiverShardID: 0, + }, + } + + err = esProc.SaveMiniblocks(header, miniBlocks) + require.Nil(t, err) + genericResponse := &GenericResponse{} + + ids := []string{"2f3ee0ff3b6426916df3b123a10f425b7e2027e2ae8d231229d27b12aa522ade"} + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.MiniblocksIndex, true, genericResponse) + require.Nil(t, err) + require.JSONEq(t, readExpectedResult("./testdata/miniblocks/cross-miniblock-on-destination-first.json"), string(genericResponse.Docs[0].Source)) + + // index on source + mbhr = &dataBlock.MiniBlockHeaderReserved{ + ExecutionType: dataBlock.ProcessingType(0), + } + mbhrBytes, _ = marshaller.Marshal(mbhr) + header.ShardID = 2 + header.MiniBlockHeaders = []dataBlock.MiniBlockHeader{ + { + Reserved: mbhrBytes, + }, + } + err = esProc.SaveMiniblocks(header, miniBlocks) + require.Nil(t, err) + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.MiniblocksIndex, true, genericResponse) + require.Nil(t, err) + require.JSONEq(t, readExpectedResult("./testdata/miniblocks/cross-miniblock-on-source-second.json"), string(genericResponse.Docs[0].Source)) +} diff --git a/integrationtests/testdata/miniblocks/cross-miniblock-on-destination-first.json b/integrationtests/testdata/miniblocks/cross-miniblock-on-destination-first.json new file mode 100644 index 00000000..a18dfa1e --- /dev/null +++ b/integrationtests/testdata/miniblocks/cross-miniblock-on-destination-first.json @@ -0,0 +1,8 @@ +{ + "procTypeD": "Processed", + "receiverShard": 0, + "senderShard": 2, + "receiverBlockHash": "b36435faaa72390772da84f418348ce0d477c74432579519bf0ffea1dc4c36e9", + "type": "TxBlock", + "timestamp": 54321 +} diff --git a/integrationtests/testdata/miniblocks/cross-miniblock-on-destination.json b/integrationtests/testdata/miniblocks/cross-miniblock-on-destination.json new file mode 100644 index 00000000..6baab6f0 --- /dev/null +++ b/integrationtests/testdata/miniblocks/cross-miniblock-on-destination.json @@ -0,0 +1,10 @@ +{ + "procTypeS": "Normal", + "senderBlockHash": "3fede8a9a3c4f2ba6d7e6e01541813606cd61c4d3af2940f8e089827b5d94e50", + "receiverShard": 2, + "senderShard": 1, + "type": "TxBlock", + "timestamp": 1234, + "receiverBlockHash": "d7f1e8003a45c7adbd87bbbb269cb4af3d1f4aedd0c214973bfc096dd0f3b65e", + "procTypeD": "Scheduled" +} diff --git a/integrationtests/testdata/miniblocks/cross-miniblock-on-source-second.json b/integrationtests/testdata/miniblocks/cross-miniblock-on-source-second.json new file mode 100644 index 00000000..0ae4076f --- /dev/null +++ b/integrationtests/testdata/miniblocks/cross-miniblock-on-source-second.json @@ -0,0 +1,10 @@ +{ + "procTypeD": "Processed", + "receiverShard": 0, + "senderShard": 2, + "receiverBlockHash": "b36435faaa72390772da84f418348ce0d477c74432579519bf0ffea1dc4c36e9", + "type": "TxBlock", + "timestamp": 54321, + "senderBlockHash": "b601381e1f41df2aa3da9f2b8eb169f14c86418229e30fc65f9e6b37b7f0d902", + "procTypeS": "Normal" +} diff --git a/integrationtests/testdata/miniblocks/cross-miniblock-on-source.json b/integrationtests/testdata/miniblocks/cross-miniblock-on-source.json new file mode 100644 index 00000000..98dce8aa --- /dev/null +++ b/integrationtests/testdata/miniblocks/cross-miniblock-on-source.json @@ -0,0 +1,8 @@ +{ + "procTypeS": "Normal", + "senderBlockHash": "3fede8a9a3c4f2ba6d7e6e01541813606cd61c4d3af2940f8e089827b5d94e50", + "receiverShard": 2, + "senderShard": 1, + "type": "TxBlock", + "timestamp": 1234 +} diff --git a/integrationtests/utils.go b/integrationtests/utils.go index d76b82c7..6c603cdc 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -58,7 +58,7 @@ func CreateElasticProcessor( DBClient: esClient, EnabledIndexes: []string{dataindexer.TransactionsIndex, dataindexer.LogsIndex, dataindexer.AccountsESDTIndex, dataindexer.ScResultsIndex, dataindexer.ReceiptsIndex, dataindexer.BlockIndex, dataindexer.AccountsIndex, dataindexer.TokensIndex, dataindexer.TagsIndex, - dataindexer.OperationsIndex, dataindexer.DelegatorsIndex, dataindexer.ESDTsIndex, dataindexer.SCDeploysIndex}, + dataindexer.OperationsIndex, dataindexer.DelegatorsIndex, dataindexer.ESDTsIndex, dataindexer.SCDeploysIndex, dataindexer.MiniblocksIndex}, Denomination: 18, } diff --git a/process/elasticproc/miniblocks/miniblocksProcessor.go b/process/elasticproc/miniblocks/miniblocksProcessor.go index 05234be5..db8405e8 100644 --- a/process/elasticproc/miniblocks/miniblocksProcessor.go +++ b/process/elasticproc/miniblocks/miniblocksProcessor.go @@ -88,7 +88,7 @@ func (mp *miniblocksProcessor) prepareMiniblockForDB( isIntraShard := dbMiniblock.SenderShardID == dbMiniblock.ReceiverShardID isCrossOnSource := !isIntraShard && dbMiniblock.SenderShardID == header.GetShardID() if isIntraShard || isCrossOnSource { - mp.setFieldsMBIntraShardAndCrossFromMe(mbIndex, header, encodedHeaderHash, dbMiniblock) + mp.setFieldsMBIntraShardAndCrossFromMe(mbIndex, header, encodedHeaderHash, dbMiniblock, isIntraShard) return dbMiniblock, nil } @@ -105,6 +105,7 @@ func (mp *miniblocksProcessor) setFieldsMBIntraShardAndCrossFromMe( header coreData.HeaderHandler, headerHash string, dbMiniblock *data.Miniblock, + isIntraShard bool, ) { processingType, constructionState := mp.computeProcessingTypeAndConstructionState(mbIndex, header) @@ -112,9 +113,11 @@ func (mp *miniblocksProcessor) setFieldsMBIntraShardAndCrossFromMe( switch { case constructionState == int32(block.Final) && processingType == block.Normal.String(): dbMiniblock.SenderBlockHash = headerHash - dbMiniblock.ReceiverBlockHash = headerHash dbMiniblock.ProcessingTypeOnSource = processingType - dbMiniblock.ProcessingTypeOnDestination = processingType + if isIntraShard { + dbMiniblock.ReceiverBlockHash = headerHash + dbMiniblock.ProcessingTypeOnDestination = processingType + } case constructionState == int32(block.Proposed) && processingType == block.Scheduled.String(): dbMiniblock.SenderBlockHash = headerHash dbMiniblock.ProcessingTypeOnSource = processingType From 00c88f83722a6c825d3dbe402e865c55e87b900a Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 27 Jul 2023 15:24:00 +0300 Subject: [PATCH 141/189] fix --- integrationtests/miniblocks_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integrationtests/miniblocks_test.go b/integrationtests/miniblocks_test.go index c6dd0381..3814964e 100644 --- a/integrationtests/miniblocks_test.go +++ b/integrationtests/miniblocks_test.go @@ -1,3 +1,5 @@ +//go:build integrationtests + package integrationtests import ( From d804412a2f593d33640494ab571fda3a9ee36a7a Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 28 Jul 2023 10:52:07 +0300 Subject: [PATCH 142/189] fixes after review --- process/elasticproc/miniblocks/serialize.go | 8 ++++---- process/elasticproc/miniblocks/serialize_test.go | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/process/elasticproc/miniblocks/serialize.go b/process/elasticproc/miniblocks/serialize.go index 08385b91..495ef22e 100644 --- a/process/elasticproc/miniblocks/serialize.go +++ b/process/elasticproc/miniblocks/serialize.go @@ -43,12 +43,12 @@ func (mp *miniblocksProcessor) prepareMiniblockData(miniblockDB *data.Miniblock, // prepare data for update operation meta := []byte(fmt.Sprintf(`{ "update" : {"_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(mbHash), "\n")) - onSourceNotProcessed := shardID == miniblockDB.SenderShardID && miniblockDB.ProcessingTypeOnDestination != block.Processed.String() + notProcessedOnSource := shardID == miniblockDB.SenderShardID && miniblockDB.ProcessingTypeOnDestination != block.Processed.String() codeToExecute := ` if ('create' == ctx.op) { ctx._source = params.mb } else { - if (params.osnp) { + if (params.npos) { ctx._source.senderBlockHash = params.mb.senderBlockHash; ctx._source.procTypeS = params.mb.procTypeS; } else { @@ -61,9 +61,9 @@ func (mp *miniblocksProcessor) prepareMiniblockData(miniblockDB *data.Miniblock, serializedDataStr := fmt.Sprintf(`{"scripted_upsert": true, "script": {`+ `"source": "%s",`+ `"lang": "painless",`+ - `"params": { "mb": %s, "osnp": %t }},`+ + `"params": { "mb": %s, "npos": %t }},`+ `"upsert": {}}`, - converters.FormatPainlessSource(codeToExecute), mbBytes, onSourceNotProcessed, + converters.FormatPainlessSource(codeToExecute), mbBytes, notProcessedOnSource, ) return meta, []byte(serializedDataStr), nil diff --git a/process/elasticproc/miniblocks/serialize_test.go b/process/elasticproc/miniblocks/serialize_test.go index 44d7fdd9..612f3ecd 100644 --- a/process/elasticproc/miniblocks/serialize_test.go +++ b/process/elasticproc/miniblocks/serialize_test.go @@ -23,9 +23,9 @@ func TestMiniblocksProcessor_SerializeBulkMiniBlocks(t *testing.T) { mp.SerializeBulkMiniBlocks(miniblocks, buffSlice, "miniblocks", 0) expectedBuff := `{ "update" : {"_index":"miniblocks", "_id" : "h1" } } -{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.osnp) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":0,"receiverShard":1,"type":"","timestamp":0}, "osnp": true }},"upsert": {}} +{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.npos) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":0,"receiverShard":1,"type":"","timestamp":0}, "npos": true }},"upsert": {}} { "update" : {"_index":"miniblocks", "_id" : "h2" } } -{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.osnp) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":0,"receiverShard":2,"type":"","timestamp":0}, "osnp": true }},"upsert": {}} +{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.npos) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":0,"receiverShard":2,"type":"","timestamp":0}, "npos": true }},"upsert": {}} ` require.Equal(t, expectedBuff, buffSlice.Buffers()[0].String()) } @@ -44,9 +44,9 @@ func TestMiniblocksProcessor_SerializeBulkMiniBlocksInDB(t *testing.T) { mp.SerializeBulkMiniBlocks(miniblocks, buffSlice, "miniblocks", 0) expectedBuff := `{ "update" : {"_index":"miniblocks", "_id" : "h1" } } -{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.osnp) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":0,"receiverShard":1,"type":"","timestamp":0}, "osnp": true }},"upsert": {}} +{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.npos) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":0,"receiverShard":1,"type":"","timestamp":0}, "npos": true }},"upsert": {}} { "update" : {"_index":"miniblocks", "_id" : "h2" } } -{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.osnp) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":0,"receiverShard":2,"type":"","timestamp":0}, "osnp": true }},"upsert": {}} +{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.npos) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":0,"receiverShard":2,"type":"","timestamp":0}, "npos": true }},"upsert": {}} ` require.Equal(t, expectedBuff, buffSlice.Buffers()[0].String()) } @@ -62,7 +62,7 @@ func TestSerializeMiniblock_CrossShardNormal(t *testing.T) { mp.SerializeBulkMiniBlocks(miniblocks, buffSlice, "miniblocks", 1) expectedBuff := `{ "update" : {"_index":"miniblocks", "_id" : "h1" } } -{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.osnp) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":0,"receiverShard":1,"receiverBlockHash":"receiverBlock","type":"","timestamp":0}, "osnp": false }},"upsert": {}} +{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.npos) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":0,"receiverShard":1,"receiverBlockHash":"receiverBlock","type":"","timestamp":0}, "npos": false }},"upsert": {}} ` require.Equal(t, expectedBuff, buffSlice.Buffers()[0].String()) } @@ -79,7 +79,7 @@ func TestSerializeMiniblock_IntraShardScheduled(t *testing.T) { mp.SerializeBulkMiniBlocks(miniblocks, buffSlice, "miniblocks", 1) expectedBuff := `{ "update" : {"_index":"miniblocks", "_id" : "h1" } } -{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.osnp) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":1,"receiverShard":1,"senderBlockHash":"senderBlock","type":"","procTypeS":"Scheduled","timestamp":0}, "osnp": true }},"upsert": {}} +{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.npos) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":1,"receiverShard":1,"senderBlockHash":"senderBlock","type":"","procTypeS":"Scheduled","timestamp":0}, "npos": true }},"upsert": {}} ` require.Equal(t, expectedBuff, buffSlice.Buffers()[0].String()) @@ -92,7 +92,7 @@ func TestSerializeMiniblock_IntraShardScheduled(t *testing.T) { mp.SerializeBulkMiniBlocks(miniblocks, buffSlice, "miniblocks", 1) expectedBuff = `{ "update" : {"_index":"miniblocks", "_id" : "h1" } } -{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.osnp) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":1,"receiverShard":1,"receiverBlockHash":"receiverBlock","type":"","procTypeD":"Processed","timestamp":0}, "osnp": false }},"upsert": {}} +{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.mb} else {if (params.npos) {ctx._source.senderBlockHash = params.mb.senderBlockHash;ctx._source.procTypeS = params.mb.procTypeS;} else {ctx._source.receiverBlockHash = params.mb.receiverBlockHash;ctx._source.procTypeD = params.mb.procTypeD;}}","lang": "painless","params": { "mb": {"senderShard":1,"receiverShard":1,"receiverBlockHash":"receiverBlock","type":"","procTypeD":"Processed","timestamp":0}, "npos": false }},"upsert": {}} ` require.Equal(t, expectedBuff, buffSlice.Buffers()[0].String()) } From 814805cf0ba409254de8de3a798653546bb3dbaa Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 8 Aug 2023 15:38:20 +0300 Subject: [PATCH 143/189] extend scdeploys index --- data/logs.go | 1 + data/scDeploy.go | 12 ++++-- data/tokens.go | 1 + process/elasticproc/elasticProcessor.go | 11 +++-- process/elasticproc/interface.go | 1 + process/elasticproc/logsevents/interface.go | 1 + .../logsevents/logsAndEventsProcessor.go | 2 + .../logsevents/logsAndEventsProcessor_test.go | 10 +++-- process/elasticproc/logsevents/logsData.go | 2 + process/elasticproc/logsevents/scDeploys.go | 39 +++++++++++++++--- .../elasticproc/logsevents/scDeploys_test.go | 35 ++++++++++++++-- process/elasticproc/logsevents/serialize.go | 41 +++++++++++++++++++ .../elasticproc/logsevents/serialize_test.go | 2 +- 13 files changed, 137 insertions(+), 21 deletions(-) diff --git a/data/logs.go b/data/logs.go index 96754daa..6d7058a3 100644 --- a/data/logs.go +++ b/data/logs.go @@ -30,6 +30,7 @@ type PreparedLogsResults struct { Tokens TokensHandler TokensSupply TokensHandler ScDeploys map[string]*ScDeployInfo + ChangeOwnerOperations map[string]*OwnerData Delegators map[string]*Delegator TxHashStatusInfo map[string]*outport.StatusInfo TokensInfo []*TokenInfo diff --git a/data/scDeploy.go b/data/scDeploy.go index 97c2800a..7727e55e 100644 --- a/data/scDeploy.go +++ b/data/scDeploy.go @@ -2,10 +2,13 @@ package data // ScDeployInfo is the DTO that holds information about a smart contract deployment type ScDeployInfo struct { - TxHash string `json:"deployTxHash"` - Creator string `json:"deployer"` - Timestamp uint64 `json:"timestamp"` - Upgrades []*Upgrade `json:"upgrades"` + TxHash string `json:"deployTxHash"` + Creator string `json:"deployer"` + CurrentOwner string `json:"currentOwner"` + CodeHash []byte `json:"codeHash"` + Timestamp uint64 `json:"timestamp"` + Upgrades []*Upgrade `json:"upgrades"` + OwnersHistory []*OwnerData `json:"owners"` } // Upgrade is the DTO that holds information about a smart contract upgrade @@ -13,4 +16,5 @@ type Upgrade struct { TxHash string `json:"upgradeTxHash"` Upgrader string `json:"upgrader"` Timestamp uint64 `json:"timestamp"` + CodeHash []byte `json:"codeHash"` } diff --git a/data/tokens.go b/data/tokens.go index 5a6e5476..4018c6ea 100644 --- a/data/tokens.go +++ b/data/tokens.go @@ -70,6 +70,7 @@ type TokenProperties struct { // OwnerData is a structure that is needed to store information about an owner type OwnerData struct { + TxHash string `json:"txHash,omitempty"` Address string `json:"address"` Timestamp time.Duration `json:"timestamp"` } diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index 7170b4f8..3b34fefd 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -485,7 +485,7 @@ func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader return err } - err = ei.indexScDeploys(logsData.ScDeploys, buffers) + err = ei.indexScDeploys(logsData.ScDeploys, logsData.ChangeOwnerOperations, buffers) if err != nil { return err } @@ -532,12 +532,17 @@ func (ei *elasticProcessor) prepareAndIndexLogs(logsAndEvents []*outport.LogData return ei.logsAndEventsProc.SerializeLogs(logsDB, buffSlice, elasticIndexer.LogsIndex) } -func (ei *elasticProcessor) indexScDeploys(deployData map[string]*data.ScDeployInfo, buffSlice *data.BufferSlice) error { +func (ei *elasticProcessor) indexScDeploys(deployData map[string]*data.ScDeployInfo, changeOwnerOperation map[string]*data.OwnerData, buffSlice *data.BufferSlice) error { if !ei.isIndexEnabled(elasticIndexer.SCDeploysIndex) { return nil } - return ei.logsAndEventsProc.SerializeSCDeploys(deployData, buffSlice, elasticIndexer.SCDeploysIndex) + err := ei.logsAndEventsProc.SerializeSCDeploys(deployData, buffSlice, elasticIndexer.SCDeploysIndex) + if err != nil { + return err + } + + return ei.logsAndEventsProc.SerializeChangeOwnerOperations(changeOwnerOperation, buffSlice, elasticIndexer.SCDeploysIndex) } func (ei *elasticProcessor) indexTransactions(txs []*data.Transaction, txHashStatusInfo map[string]*outport.StatusInfo, header coreData.HeaderHandler, bytesBuff *data.BufferSlice) error { diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index dcb8ef93..1d260717 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -103,6 +103,7 @@ type DBLogsAndEventsHandler interface { SerializeLogs(logs []*data.Logs, buffSlice *data.BufferSlice, index string) error SerializeSCDeploys(deploysInfo map[string]*data.ScDeployInfo, buffSlice *data.BufferSlice, index string) error + SerializeChangeOwnerOperations(changeOwnerOperations map[string]*data.OwnerData, buffSlice *data.BufferSlice, index string) error SerializeTokens(tokens []*data.TokenInfo, updateNFTData []*data.NFTDataUpdate, buffSlice *data.BufferSlice, index string) error SerializeDelegators(delegators map[string]*data.Delegator, buffSlice *data.BufferSlice, index string) error SerializeSupplyData(tokensSupply data.TokensHandler, buffSlice *data.BufferSlice, index string) error diff --git a/process/elasticproc/logsevents/interface.go b/process/elasticproc/logsevents/interface.go index e65f5271..4d434cef 100644 --- a/process/elasticproc/logsevents/interface.go +++ b/process/elasticproc/logsevents/interface.go @@ -10,6 +10,7 @@ import ( type argsProcessEvent struct { txHashHexEncoded string scDeploys map[string]*data.ScDeployInfo + changeOwnerOperations map[string]*data.OwnerData txs map[string]*data.Transaction scrs map[string]*data.ScResult event coreData.EventHandler diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor.go b/process/elasticproc/logsevents/logsAndEventsProcessor.go index e29e8703..04f0cbca 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor.go @@ -124,6 +124,7 @@ func (lep *logsAndEventsProcessor) ExtractDataFromLogs( NFTsDataUpdates: lep.logsData.nftsDataUpdates, TokenRolesAndProperties: lep.logsData.tokenRolesAndProperties, TxHashStatusInfo: lep.logsData.txHashStatusInfoProc.getAllRecords(), + ChangeOwnerOperations: lep.logsData.changeOwnerOperations, } } @@ -151,6 +152,7 @@ func (lep *logsAndEventsProcessor) processEvent(logHashHexEncoded string, logAdd scrs: lep.logsData.scrsMap, tokenRolesAndProperties: lep.logsData.tokenRolesAndProperties, txHashStatusInfoProc: lep.logsData.txHashStatusInfoProc, + changeOwnerOperations: lep.logsData.changeOwnerOperations, selfShardID: shardID, numOfShards: numOfShards, }) diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go index 99dca8e3..68ed2fdc 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go @@ -67,7 +67,7 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) { Address: []byte("addr"), Identifier: []byte(core.SCDeployIdentifier), - Topics: [][]byte{[]byte("addr1"), []byte("addr2")}, + Topics: [][]byte{[]byte("addr1"), []byte("addr2"), []byte("codeHash")}, }, }, }, @@ -165,9 +165,11 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) require.True(t, res.ScResults[0].HasLogs) require.Equal(t, &data.ScDeployInfo{ - TxHash: "6833", - Creator: "6164647232", - Timestamp: uint64(1000), + TxHash: "6833", + Creator: "6164647232", + CurrentOwner: "6164647232", + Timestamp: uint64(1000), + CodeHash: []byte("codeHash"), }, resLogs.ScDeploys["6164647231"]) require.Equal(t, &data.TokenInfo{ diff --git a/process/elasticproc/logsevents/logsData.go b/process/elasticproc/logsevents/logsData.go index ee12e55c..caacb880 100644 --- a/process/elasticproc/logsevents/logsData.go +++ b/process/elasticproc/logsevents/logsData.go @@ -14,6 +14,7 @@ type logsData struct { txsMap map[string]*data.Transaction scrsMap map[string]*data.ScResult scDeploys map[string]*data.ScDeployInfo + changeOwnerOperations map[string]*data.OwnerData delegators map[string]*data.Delegator tokensInfo []*data.TokenInfo nftsDataUpdates []*data.NFTDataUpdate @@ -35,6 +36,7 @@ func newLogsData( ld.scDeploys = make(map[string]*data.ScDeployInfo) ld.tokensInfo = make([]*data.TokenInfo, 0) ld.delegators = make(map[string]*data.Delegator) + ld.changeOwnerOperations = make(map[string]*data.OwnerData) ld.nftsDataUpdates = make([]*data.NFTDataUpdate, 0) ld.tokenRolesAndProperties = tokeninfo.NewTokenRolesAndProperties() ld.txHashStatusInfoProc = newTxHashStatusInfoProcessor() diff --git a/process/elasticproc/logsevents/scDeploys.go b/process/elasticproc/logsevents/scDeploys.go index 62d77543..6ab0b204 100644 --- a/process/elasticproc/logsevents/scDeploys.go +++ b/process/elasticproc/logsevents/scDeploys.go @@ -1,10 +1,16 @@ package logsevents import ( + "time" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-es-indexer-go/data" ) +const ( + minTopicsContractEvent = 3 +) + type scDeploysProcessor struct { scDeploysIdentifiers map[string]struct{} pubKeyConverter core.PubkeyConverter @@ -14,13 +20,15 @@ func newSCDeploysProcessor(pubKeyConverter core.PubkeyConverter) *scDeploysProce return &scDeploysProcessor{ pubKeyConverter: pubKeyConverter, scDeploysIdentifiers: map[string]struct{}{ - core.SCDeployIdentifier: {}, - core.SCUpgradeIdentifier: {}, + core.SCDeployIdentifier: {}, + core.SCUpgradeIdentifier: {}, + core.BuiltInFunctionChangeOwnerAddress: {}, }, } } func (sdp *scDeploysProcessor) processEvent(args *argsProcessEvent) argOutputProcessEvent { + eventIdentifier := string(args.event.GetIdentifier()) _, ok := sdp.scDeploysIdentifiers[eventIdentifier] if !ok { @@ -28,7 +36,12 @@ func (sdp *scDeploysProcessor) processEvent(args *argsProcessEvent) argOutputPro } topics := args.event.GetTopics() - if len(topics) < 2 { + isChangeOwnerEvent := len(topics) == 1 && eventIdentifier == core.BuiltInFunctionChangeOwnerAddress + if isChangeOwnerEvent { + return sdp.processChangeOwnerEvent(args) + } + + if len(topics) < minTopicsContractEvent { return argOutputProcessEvent{ processed: true, } @@ -38,9 +51,25 @@ func (sdp *scDeploysProcessor) processEvent(args *argsProcessEvent) argOutputPro creatorAddress := sdp.pubKeyConverter.SilentEncode(topics[1], log) args.scDeploys[scAddress] = &data.ScDeployInfo{ + TxHash: args.txHashHexEncoded, + Creator: creatorAddress, + CurrentOwner: creatorAddress, + CodeHash: topics[2], + Timestamp: args.timestamp, + } + + return argOutputProcessEvent{ + processed: true, + } +} + +func (sdp *scDeploysProcessor) processChangeOwnerEvent(args *argsProcessEvent) argOutputProcessEvent { + scAddress := sdp.pubKeyConverter.SilentEncode(args.event.GetAddress(), log) + newOwner := sdp.pubKeyConverter.SilentEncode(args.event.GetTopics()[0], log) + args.changeOwnerOperations[scAddress] = &data.OwnerData{ TxHash: args.txHashHexEncoded, - Creator: creatorAddress, - Timestamp: args.timestamp, + Address: newOwner, + Timestamp: time.Duration(args.timestamp), } return argOutputProcessEvent{ diff --git a/process/elasticproc/logsevents/scDeploys_test.go b/process/elasticproc/logsevents/scDeploys_test.go index dc602cf0..a725fb4b 100644 --- a/process/elasticproc/logsevents/scDeploys_test.go +++ b/process/elasticproc/logsevents/scDeploys_test.go @@ -18,7 +18,7 @@ func TestScDeploysProcessor(t *testing.T) { event := &transaction.Event{ Address: []byte("addr"), Identifier: []byte(core.SCDeployIdentifier), - Topics: [][]byte{[]byte("addr1"), []byte("addr2")}, + Topics: [][]byte{[]byte("addr1"), []byte("addr2"), []byte("codeHash")}, } scDeploys := map[string]*data.ScDeployInfo{} @@ -31,8 +31,35 @@ func TestScDeploysProcessor(t *testing.T) { require.True(t, res.processed) require.Equal(t, &data.ScDeployInfo{ - TxHash: "01020304", - Creator: "6164647232", - Timestamp: uint64(1000), + TxHash: "01020304", + Creator: "6164647232", + Timestamp: uint64(1000), + CurrentOwner: "6164647232", + CodeHash: []byte("codeHash"), }, scDeploys["6164647231"]) } + +func TestScDeploysProcessorChangeOwner(t *testing.T) { + event := &transaction.Event{ + Address: []byte("contractAddr"), + Identifier: []byte(core.BuiltInFunctionChangeOwnerAddress), + Topics: [][]byte{[]byte("newOwner")}, + } + + scDeploysProc := newSCDeploysProcessor(&mock.PubkeyConverterMock{}) + + changeOwnerOperations := map[string]*data.OwnerData{} + res := scDeploysProc.processEvent(&argsProcessEvent{ + event: event, + changeOwnerOperations: changeOwnerOperations, + timestamp: 2000, + txHashHexEncoded: "01020304", + }) + require.True(t, res.processed) + + require.Equal(t, &data.OwnerData{ + TxHash: "01020304", + Address: "6e65774f776e6572", + Timestamp: 2000, + }, changeOwnerOperations["636f6e747261637441646472"]) +} diff --git a/process/elasticproc/logsevents/serialize.go b/process/elasticproc/logsevents/serialize.go index f519074d..8d1e915e 100644 --- a/process/elasticproc/logsevents/serialize.go +++ b/process/elasticproc/logsevents/serialize.go @@ -49,6 +49,44 @@ func (*logsAndEventsProcessor) SerializeLogs(logs []*data.Logs, buffSlice *data. return nil } +// SerializeChangeOwnerOperations will serialize the provided change owner operations in a way that Elasticsearch expects a bulk request +func (*logsAndEventsProcessor) SerializeChangeOwnerOperations(changeOwnerOperations map[string]*data.OwnerData, buffSlice *data.BufferSlice, index string) error { + for scAddr, ownerData := range changeOwnerOperations { + meta := []byte(fmt.Sprintf(`{ "update" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(scAddr), "\n")) + ownerDataBytes, err := json.Marshal(ownerData) + if err != nil { + return err + } + + codeToExecute := ` + if ('create' == ctx.op) { + return + } else { + ctx._source.currentOwner = params.ownerData.address; + if (!ctx._source.containsKey('owners')) { + ctx._source.owners = [params.ownerData]; + } else { + ctx._source.owners.add(params.ownerData); + } + } +` + serializedDataStr := fmt.Sprintf(`{"scripted_upsert": true, "script": {`+ + `"source": "%s",`+ + `"lang": "painless",`+ + `"params": { "ownerData": %s }},`+ + `"upsert": {}}`, + converters.FormatPainlessSource(codeToExecute), ownerDataBytes, + ) + + err = buffSlice.PutData(meta, []byte(serializedDataStr)) + if err != nil { + return err + } + } + + return nil +} + // SerializeSCDeploys will serialize the provided smart contract deploys in a way that Elasticsearch expects a bulk request func (*logsAndEventsProcessor) SerializeSCDeploys(deploys map[string]*data.ScDeployInfo, buffSlice *data.BufferSlice, index string) error { for scAddr, deployInfo := range deploys { @@ -70,6 +108,7 @@ func (*logsAndEventsProcessor) SerializeSCDeploys(deploys map[string]*data.ScDep func serializeDeploy(deployInfo *data.ScDeployInfo) ([]byte, error) { deployInfo.Upgrades = make([]*data.Upgrade, 0) + deployInfo.OwnersHistory = make([]*data.OwnerData, 0) serializedData, errPrepareD := json.Marshal(deployInfo) if errPrepareD != nil { return nil, errPrepareD @@ -79,6 +118,7 @@ func serializeDeploy(deployInfo *data.ScDeployInfo) ([]byte, error) { TxHash: deployInfo.TxHash, Upgrader: deployInfo.Creator, Timestamp: deployInfo.Timestamp, + CodeHash: deployInfo.CodeHash, } upgradeSerialized, errPrepareU := json.Marshal(upgradeData) if errPrepareU != nil { @@ -86,6 +126,7 @@ func serializeDeploy(deployInfo *data.ScDeployInfo) ([]byte, error) { } codeToExecute := ` + ctx._source.codeHash = params.elem.codeHash; if (!ctx._source.containsKey('upgrades')) { ctx._source.upgrades = [params.elem]; } else { diff --git a/process/elasticproc/logsevents/serialize_test.go b/process/elasticproc/logsevents/serialize_test.go index ad01c59f..711060a1 100644 --- a/process/elasticproc/logsevents/serialize_test.go +++ b/process/elasticproc/logsevents/serialize_test.go @@ -57,7 +57,7 @@ func TestLogsAndEventsProcessor_SerializeSCDeploys(t *testing.T) { require.Nil(t, err) expectedRes := `{ "update" : { "_index":"scdeploys", "_id" : "scAddr" } } -{"script": {"source": "if (!ctx._source.containsKey('upgrades')) {ctx._source.upgrades = [params.elem];} else {ctx._source.upgrades.add(params.elem);}","lang": "painless","params": {"elem": {"upgradeTxHash":"hash","upgrader":"creator","timestamp":123}}},"upsert": {"deployTxHash":"hash","deployer":"creator","timestamp":123,"upgrades":[]}} +{"script": {"source": "ctx._source.codeHash = params.elem.codeHash;if (!ctx._source.containsKey('upgrades')) {ctx._source.upgrades = [params.elem];} else {ctx._source.upgrades.add(params.elem);}","lang": "painless","params": {"elem": {"upgradeTxHash":"hash","upgrader":"creator","timestamp":123,"codeHash":null}}},"upsert": {"deployTxHash":"hash","deployer":"creator","currentOwner":"","codeHash":null,"timestamp":123,"upgrades":[],"owners":[]}} ` require.Equal(t, expectedRes, buffSlice.Buffers()[0].String()) } From 1688677f37aa323e41e3fbcda6cd661809ac35e2 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 8 Aug 2023 15:42:52 +0300 Subject: [PATCH 144/189] extend mappings structures --- templates/noKibana/deploys.go | 24 ++++++++++++++++++++++++ templates/withKibana/deploys.go | 24 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/templates/noKibana/deploys.go b/templates/noKibana/deploys.go index 791e8c01..732032ac 100644 --- a/templates/noKibana/deploys.go +++ b/templates/noKibana/deploys.go @@ -21,6 +21,12 @@ var SCDeploys = Object{ "type": "date", "format": "epoch_second", }, + "currentOwner": Object{ + "type": "keyword", + }, + "codeHash": Object{ + "type": "keyword", + }, "upgrades": Object{ "type": "nested", "properties": Object{ @@ -34,6 +40,24 @@ var SCDeploys = Object{ "upgrader": Object{ "type": "keyword", }, + "codeHash": Object{ + "type": "keyword", + }, + }, + }, + "owners": Object{ + "type": "nested", + "properties": Object{ + "timestamp": Object{ + "type": "date", + "format": "epoch_second", + }, + "txHash": Object{ + "type": "keyword", + }, + "address": Object{ + "type": "keyword", + }, }, }, }, diff --git a/templates/withKibana/deploys.go b/templates/withKibana/deploys.go index 17e969bc..9e734eff 100644 --- a/templates/withKibana/deploys.go +++ b/templates/withKibana/deploys.go @@ -21,6 +21,12 @@ var SCDeploys = Object{ "type": "date", "format": "epoch_second", }, + "currentOwner": Object{ + "type": "keyword", + }, + "codeHash": Object{ + "type": "keyword", + }, "upgrades": Object{ "type": "nested", "properties": Object{ @@ -34,6 +40,24 @@ var SCDeploys = Object{ "upgrader": Object{ "type": "keyword", }, + "codeHash": Object{ + "type": "keyword", + }, + }, + }, + "owners": Object{ + "type": "nested", + "properties": Object{ + "timestamp": Object{ + "type": "date", + "format": "epoch_second", + }, + "txHash": Object{ + "type": "keyword", + }, + "address": Object{ + "type": "keyword", + }, }, }, }, From 8a1fdb5c9ddcd455b10249aeec72f29505ce70c4 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 8 Aug 2023 17:25:21 +0300 Subject: [PATCH 145/189] integration tests --- data/scDeploy.go | 2 +- integrationtests/scDeploy_test.go | 66 ++++++++++++++++++- ...deploy-after-upgrade-and-change-owner.json | 22 +++++++ .../scDeploy/deploy-after-upgrade.json | 16 +++++ .../testdata/scDeploy/deploy.json | 5 +- process/elasticproc/logsevents/serialize.go | 1 - templates/noKibana/deploys.go | 2 +- templates/withKibana/deploys.go | 2 +- 8 files changed, 110 insertions(+), 6 deletions(-) create mode 100644 integrationtests/testdata/scDeploy/deploy-after-upgrade-and-change-owner.json create mode 100644 integrationtests/testdata/scDeploy/deploy-after-upgrade.json diff --git a/data/scDeploy.go b/data/scDeploy.go index 7727e55e..7a7bb69c 100644 --- a/data/scDeploy.go +++ b/data/scDeploy.go @@ -5,7 +5,7 @@ type ScDeployInfo struct { TxHash string `json:"deployTxHash"` Creator string `json:"deployer"` CurrentOwner string `json:"currentOwner"` - CodeHash []byte `json:"codeHash"` + CodeHash []byte `json:"initialCodeHash"` Timestamp uint64 `json:"timestamp"` Upgrades []*Upgrade `json:"upgrades"` OwnersHistory []*OwnerData `json:"owners"` diff --git a/integrationtests/scDeploy_test.go b/integrationtests/scDeploy_test.go index 2ed3f944..50d22f90 100644 --- a/integrationtests/scDeploy_test.go +++ b/integrationtests/scDeploy_test.go @@ -74,7 +74,7 @@ func TestTransactionWithSCDeploy(t *testing.T) { { Address: decodeAddress(sndAddress), Identifier: []byte(core.SCDeployIdentifier), - Topics: [][]byte{decodeAddress("erd1qqqqqqqqqqqqqpgq4t2tqxpst9a6qttpak8cz8wvz6a0nses5qfqel6rhy"), decodeAddress("erd12m3x8jp6dl027pj5f2nw6ght2cyhhjfrs86cdwsa8xn83r375qfqrwpdx0")}, + Topics: [][]byte{decodeAddress("erd1qqqqqqqqqqqqqpgq4t2tqxpst9a6qttpak8cz8wvz6a0nses5qfqel6rhy"), decodeAddress("erd12m3x8jp6dl027pj5f2nw6ght2cyhhjfrs86cdwsa8xn83r375qfqrwpdx0"), []byte("codeHash")}, }, nil, }, @@ -103,4 +103,68 @@ func TestTransactionWithSCDeploy(t *testing.T) { readExpectedResult("./testdata/scDeploy/deploy.json"), string(genericResponse.Docs[0].Source), ) + + // UPGRADE contract + header.TimeStamp = 6000 + pool = &outport.TransactionPool{ + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h2")), + Log: &transaction.Log{ + Address: decodeAddress(sndAddress), + Events: []*transaction.Event{ + { + Address: decodeAddress(sndAddress), + Identifier: []byte(core.SCDeployIdentifier), + Topics: [][]byte{decodeAddress("erd1qqqqqqqqqqqqqpgq4t2tqxpst9a6qttpak8cz8wvz6a0nses5qfqel6rhy"), decodeAddress("erd12m3x8jp6dl027pj5f2nw6ght2cyhhjfrs86cdwsa8xn83r375qfqrwpdx0"), []byte("secondCodeHash")}, + }, + nil, + }, + }, + }, + }, + } + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) + require.Nil(t, err) + + ids = []string{"erd1qqqqqqqqqqqqqpgq4t2tqxpst9a6qttpak8cz8wvz6a0nses5qfqel6rhy"} + err = esClient.DoMultiGet(ids, indexerData.SCDeploysIndex, true, genericResponse) + require.Nil(t, err) + + require.JSONEq(t, + readExpectedResult("./testdata/scDeploy/deploy-after-upgrade.json"), + string(genericResponse.Docs[0].Source), + ) + + // CHANGE owner + header.TimeStamp = 7000 + pool = &outport.TransactionPool{ + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h3")), + Log: &transaction.Log{ + Address: decodeAddress(sndAddress), + Events: []*transaction.Event{ + { + Address: decodeAddress("erd1qqqqqqqqqqqqqpgq4t2tqxpst9a6qttpak8cz8wvz6a0nses5qfqel6rhy"), + Identifier: []byte(core.BuiltInFunctionChangeOwnerAddress), + Topics: [][]byte{decodeAddress("erd1d942l8w4yvgjffpqacs8vdwl0mndsv0zn0uxa80hxc3xmq4477eqnyw3dh")}, + }, + nil, + }, + }, + }, + }, + } + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) + require.Nil(t, err) + + ids = []string{"erd1qqqqqqqqqqqqqpgq4t2tqxpst9a6qttpak8cz8wvz6a0nses5qfqel6rhy"} + err = esClient.DoMultiGet(ids, indexerData.SCDeploysIndex, true, genericResponse) + require.Nil(t, err) + + require.JSONEq(t, + readExpectedResult("./testdata/scDeploy/deploy-after-upgrade-and-change-owner.json"), + string(genericResponse.Docs[0].Source), + ) } diff --git a/integrationtests/testdata/scDeploy/deploy-after-upgrade-and-change-owner.json b/integrationtests/testdata/scDeploy/deploy-after-upgrade-and-change-owner.json new file mode 100644 index 00000000..85d1b128 --- /dev/null +++ b/integrationtests/testdata/scDeploy/deploy-after-upgrade-and-change-owner.json @@ -0,0 +1,22 @@ +{ + "deployTxHash": "6831", + "deployer": "erd12m3x8jp6dl027pj5f2nw6ght2cyhhjfrs86cdwsa8xn83r375qfqrwpdx0", + "currentOwner": "erd1d942l8w4yvgjffpqacs8vdwl0mndsv0zn0uxa80hxc3xmq4477eqnyw3dh", + "initialCodeHash": "Y29kZUhhc2g=", + "timestamp": 5040, + "upgrades": [ + { + "upgradeTxHash": "6832", + "codeHash": "c2Vjb25kQ29kZUhhc2g=", + "upgrader": "erd12m3x8jp6dl027pj5f2nw6ght2cyhhjfrs86cdwsa8xn83r375qfqrwpdx0", + "timestamp": 6000 + } + ], + "owners": [ + { + "address": "erd1d942l8w4yvgjffpqacs8vdwl0mndsv0zn0uxa80hxc3xmq4477eqnyw3dh", + "txHash": "6833", + "timestamp": 7000 + } + ] +} \ No newline at end of file diff --git a/integrationtests/testdata/scDeploy/deploy-after-upgrade.json b/integrationtests/testdata/scDeploy/deploy-after-upgrade.json new file mode 100644 index 00000000..09dca523 --- /dev/null +++ b/integrationtests/testdata/scDeploy/deploy-after-upgrade.json @@ -0,0 +1,16 @@ +{ + "deployTxHash": "6831", + "deployer": "erd12m3x8jp6dl027pj5f2nw6ght2cyhhjfrs86cdwsa8xn83r375qfqrwpdx0", + "currentOwner": "erd12m3x8jp6dl027pj5f2nw6ght2cyhhjfrs86cdwsa8xn83r375qfqrwpdx0", + "initialCodeHash": "Y29kZUhhc2g=", + "timestamp": 5040, + "upgrades": [ + { + "upgradeTxHash": "6832", + "codeHash": "c2Vjb25kQ29kZUhhc2g=", + "upgrader": "erd12m3x8jp6dl027pj5f2nw6ght2cyhhjfrs86cdwsa8xn83r375qfqrwpdx0", + "timestamp": 6000 + } + ], + "owners": [] +} \ No newline at end of file diff --git a/integrationtests/testdata/scDeploy/deploy.json b/integrationtests/testdata/scDeploy/deploy.json index 6cf1d915..4f8bd080 100644 --- a/integrationtests/testdata/scDeploy/deploy.json +++ b/integrationtests/testdata/scDeploy/deploy.json @@ -1,6 +1,9 @@ { "deployTxHash": "6831", "deployer": "erd12m3x8jp6dl027pj5f2nw6ght2cyhhjfrs86cdwsa8xn83r375qfqrwpdx0", + "currentOwner": "erd12m3x8jp6dl027pj5f2nw6ght2cyhhjfrs86cdwsa8xn83r375qfqrwpdx0", "timestamp": 5040, - "upgrades": [] + "initialCodeHash": "Y29kZUhhc2g=", + "upgrades": [], + "owners": [] } diff --git a/process/elasticproc/logsevents/serialize.go b/process/elasticproc/logsevents/serialize.go index 8d1e915e..c9db9274 100644 --- a/process/elasticproc/logsevents/serialize.go +++ b/process/elasticproc/logsevents/serialize.go @@ -126,7 +126,6 @@ func serializeDeploy(deployInfo *data.ScDeployInfo) ([]byte, error) { } codeToExecute := ` - ctx._source.codeHash = params.elem.codeHash; if (!ctx._source.containsKey('upgrades')) { ctx._source.upgrades = [params.elem]; } else { diff --git a/templates/noKibana/deploys.go b/templates/noKibana/deploys.go index 732032ac..1142e568 100644 --- a/templates/noKibana/deploys.go +++ b/templates/noKibana/deploys.go @@ -24,7 +24,7 @@ var SCDeploys = Object{ "currentOwner": Object{ "type": "keyword", }, - "codeHash": Object{ + "initialCodeHash": Object{ "type": "keyword", }, "upgrades": Object{ diff --git a/templates/withKibana/deploys.go b/templates/withKibana/deploys.go index 9e734eff..1f75d3db 100644 --- a/templates/withKibana/deploys.go +++ b/templates/withKibana/deploys.go @@ -24,7 +24,7 @@ var SCDeploys = Object{ "currentOwner": Object{ "type": "keyword", }, - "codeHash": Object{ + "initialCodeHash": Object{ "type": "keyword", }, "upgrades": Object{ From 301616af311fad70d9fa9603f06487d7ca56db76 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 8 Aug 2023 17:25:44 +0300 Subject: [PATCH 146/189] empty lines --- .../scDeploy/deploy-after-upgrade-and-change-owner.json | 2 +- integrationtests/testdata/scDeploy/deploy-after-upgrade.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integrationtests/testdata/scDeploy/deploy-after-upgrade-and-change-owner.json b/integrationtests/testdata/scDeploy/deploy-after-upgrade-and-change-owner.json index 85d1b128..e4824dd0 100644 --- a/integrationtests/testdata/scDeploy/deploy-after-upgrade-and-change-owner.json +++ b/integrationtests/testdata/scDeploy/deploy-after-upgrade-and-change-owner.json @@ -19,4 +19,4 @@ "timestamp": 7000 } ] -} \ No newline at end of file +} diff --git a/integrationtests/testdata/scDeploy/deploy-after-upgrade.json b/integrationtests/testdata/scDeploy/deploy-after-upgrade.json index 09dca523..687b1969 100644 --- a/integrationtests/testdata/scDeploy/deploy-after-upgrade.json +++ b/integrationtests/testdata/scDeploy/deploy-after-upgrade.json @@ -13,4 +13,4 @@ } ], "owners": [] -} \ No newline at end of file +} From b84bcf6ca7222791b4c49873c5e8f1dcb3a9e9bd Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 8 Aug 2023 17:27:59 +0300 Subject: [PATCH 147/189] fix unit tests --- process/elasticproc/logsevents/serialize_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process/elasticproc/logsevents/serialize_test.go b/process/elasticproc/logsevents/serialize_test.go index 711060a1..99d80304 100644 --- a/process/elasticproc/logsevents/serialize_test.go +++ b/process/elasticproc/logsevents/serialize_test.go @@ -57,7 +57,7 @@ func TestLogsAndEventsProcessor_SerializeSCDeploys(t *testing.T) { require.Nil(t, err) expectedRes := `{ "update" : { "_index":"scdeploys", "_id" : "scAddr" } } -{"script": {"source": "ctx._source.codeHash = params.elem.codeHash;if (!ctx._source.containsKey('upgrades')) {ctx._source.upgrades = [params.elem];} else {ctx._source.upgrades.add(params.elem);}","lang": "painless","params": {"elem": {"upgradeTxHash":"hash","upgrader":"creator","timestamp":123,"codeHash":null}}},"upsert": {"deployTxHash":"hash","deployer":"creator","currentOwner":"","codeHash":null,"timestamp":123,"upgrades":[],"owners":[]}} +{"script": {"source": "if (!ctx._source.containsKey('upgrades')) {ctx._source.upgrades = [params.elem];} else {ctx._source.upgrades.add(params.elem);}","lang": "painless","params": {"elem": {"upgradeTxHash":"hash","upgrader":"creator","timestamp":123,"codeHash":null}}},"upsert": {"deployTxHash":"hash","deployer":"creator","currentOwner":"","initialCodeHash":null,"timestamp":123,"upgrades":[],"owners":[]}} ` require.Equal(t, expectedRes, buffSlice.Buffers()[0].String()) } From 1f07c82a01d72c07b47e5ced675d7053c03b6fd0 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 10 Aug 2023 09:11:28 +0300 Subject: [PATCH 148/189] extend scripts --- scripts/observers/Makefile | 16 +++++++++++----- scripts/prometheus/prometheus.yml | 16 ++++++++++++++++ scripts/script.sh | 17 +++++++++++++++++ 3 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 scripts/prometheus/prometheus.yml diff --git a/scripts/observers/Makefile b/scripts/observers/Makefile index fa48e6a3..5713cf94 100644 --- a/scripts/observers/Makefile +++ b/scripts/observers/Makefile @@ -1,11 +1,13 @@ # Help section help: @echo "Available targets:" - @echo " config - Configure the local testnet" - @echo " start - Start the local testnet" - @echo " stop - Stop the local testnet" - @echo " clean - Clean up the files of the local testnet" - @echo " cluster - Start an Elasticsearch cluster on port 9200" + @echo " config - Configure the local testnet" + @echo " start - Start the local testnet" + @echo " stop - Stop the local testnet" + @echo " clean - Clean up the files of the local testnet" + @echo " cluster - Start an Elasticsearch cluster on port 9200" + @echo " monitor-start - Start the Prometheus server (port 9090) and the Grafana server (port 3000)" + @echo " monitor-Stop - Stop the Prometheus server (port 9090) and the Grafana server (port 3000)" config: python3 config.py @@ -18,4 +20,8 @@ clean: cd .. && /bin/bash script.sh delete cluster: cd .. && /bin/bash script.sh start +monitor-start: + cd .. && /bin/bash script.sh start_prometheus_and_grafana +monitor-stop: + cd .. && /bin/bash script.sh stop_prometheus_and_grafana diff --git a/scripts/prometheus/prometheus.yml b/scripts/prometheus/prometheus.yml new file mode 100644 index 00000000..2b1e127f --- /dev/null +++ b/scripts/prometheus/prometheus.yml @@ -0,0 +1,16 @@ +global: + scrape_interval: 6s + scrape_timeout: 6s + evaluation_interval: 1m +scrape_configs: + - job_name: prometheus + honor_timestamps: true + scrape_interval: 6s + scrape_timeout: 6s + metrics_path: /status/prometheus-metrics + scheme: http + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 0.0.0.0:8081 diff --git a/scripts/script.sh b/scripts/script.sh index f19a14cc..4d33b119 100755 --- a/scripts/script.sh +++ b/scripts/script.sh @@ -58,4 +58,21 @@ stop_open_search() { docker stop "${IMAGE_OPEN_SEARCH}" } +PROMETHEUS_CONTAINER_NAME=prometheus_container +GRAFANA_CONTAINER_NAME=grafana_container + +start_prometheus_and_grafana() { + docker rm ${PROMETHEUS_CONTAINER_NAME} 2> /dev/null + docker rm ${GRAFANA_CONTAINER_NAME} 2> /dev/null + + PROMETHEUS_CONFIG_FOLDER=$(pwd)/prometheus + docker run --network="host" --name "${PROMETHEUS_CONTAINER_NAME}" -d -p 9090:9090 -v "${PROMETHEUS_CONFIG_FOLDER}/prometheus.yml":/etc/prometheus/prometheus.yml prom/prometheus:v2.46.0 + docker run --network="host" --name "${GRAFANA_CONTAINER_NAME}" -d -p 3000:3000 grafana/grafana:10.0.3 +} + +stop_prometheus_and_grafana() { + docker stop "${PROMETHEUS_CONTAINER_NAME}" + docker stop "${GRAFANA_CONTAINER_NAME}" +} + "$@" From a7e306d8a5445b39d5d1596166fb76b4adeb2bd2 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 10 Aug 2023 09:14:21 +0300 Subject: [PATCH 149/189] refactor script.sh --- scripts/script.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/script.sh b/scripts/script.sh index 4d33b119..83b0867c 100755 --- a/scripts/script.sh +++ b/scripts/script.sh @@ -1,5 +1,9 @@ IMAGE_NAME=elastic-container DEFAULT_ES_VERSION=7.16.2 +PROMETHEUS_CONTAINER_NAME=prometheus_container +GRAFANA_CONTAINER_NAME=grafana_container +GRAFANA_VERSION=10.0.3 +PROMETHEUS_VERSION=v2.46.0 INDICES_LIST=("rating" "transactions" "blocks" "validators" "miniblocks" "rounds" "accounts" "accountshistory" "receipts" "scresults" "accountsesdt" "accountsesdthistory" "epochinfo" "scdeploys" "tokens" "tags" "logs" "delegators" "operations" "esdts") @@ -58,16 +62,13 @@ stop_open_search() { docker stop "${IMAGE_OPEN_SEARCH}" } -PROMETHEUS_CONTAINER_NAME=prometheus_container -GRAFANA_CONTAINER_NAME=grafana_container - start_prometheus_and_grafana() { docker rm ${PROMETHEUS_CONTAINER_NAME} 2> /dev/null docker rm ${GRAFANA_CONTAINER_NAME} 2> /dev/null PROMETHEUS_CONFIG_FOLDER=$(pwd)/prometheus - docker run --network="host" --name "${PROMETHEUS_CONTAINER_NAME}" -d -p 9090:9090 -v "${PROMETHEUS_CONFIG_FOLDER}/prometheus.yml":/etc/prometheus/prometheus.yml prom/prometheus:v2.46.0 - docker run --network="host" --name "${GRAFANA_CONTAINER_NAME}" -d -p 3000:3000 grafana/grafana:10.0.3 + docker run --network="host" --name "${PROMETHEUS_CONTAINER_NAME}" -d -p 9090:9090 -v "${PROMETHEUS_CONFIG_FOLDER}/prometheus.yml":/etc/prometheus/prometheus.yml prom/prometheus:${PROMETHEUS_VERSION} + docker run --network="host" --name "${GRAFANA_CONTAINER_NAME}" -d -p 3000:3000 grafana/grafana:${GRAFANA_VERSION} } stop_prometheus_and_grafana() { From 4c928e83da6a3443ebdeccfa67f0369e50fb06e9 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 10 Aug 2023 10:34:04 +0300 Subject: [PATCH 150/189] add status code in metrics --- client/transport/transport.go | 1 + core/request/requests.go | 9 +++++---- metrics/dtos.go | 1 + metrics/prometheusMetrics.go | 35 +++++++++++++++++++++++++++++++++++ metrics/statusMetrics.go | 19 +++++++++++++++---- metrics/statusMetrics_test.go | 24 ++++++++++++++++++++++-- 6 files changed, 79 insertions(+), 10 deletions(-) diff --git a/client/transport/transport.go b/client/transport/transport.go index fa44b361..931f59b6 100644 --- a/client/transport/transport.go +++ b/client/transport/transport.go @@ -50,6 +50,7 @@ func (m *metricsTransport) RoundTrip(req *http.Request) (*http.Response, error) topic := fmt.Sprintf("%s", valueFromCtx) m.statusMetrics.AddIndexingData(metrics.ArgsAddIndexingData{ + StatusCode: resp.StatusCode, GotError: err != nil, MessageLen: uint64(size), Topic: topic, diff --git a/core/request/requests.go b/core/request/requests.go index cdbb50f9..e0eaaab3 100644 --- a/core/request/requests.go +++ b/core/request/requests.go @@ -29,10 +29,11 @@ const ( // MetricsResponse defines the response for status metrics endpoint type MetricsResponse struct { - OperationsCount uint64 `json:"operations_count"` - ErrorsCount uint64 `json:"errors_count"` - TotalIndexingTime time.Duration `json:"total_time"` - TotalData uint64 `json:"total_data"` + OperationsCount uint64 `json:"operations_count"` + ErrorsCount map[int]uint64 `json:"errors_count,omitempty"` + TotalErrorsCount uint64 `json:"total_errors_count"` + TotalIndexingTime time.Duration `json:"total_time"` + TotalData uint64 `json:"total_data"` } // ExtendTopicWithShardID will concatenate topic with shardID diff --git a/metrics/dtos.go b/metrics/dtos.go index 52e950d6..3caaa05f 100644 --- a/metrics/dtos.go +++ b/metrics/dtos.go @@ -4,6 +4,7 @@ import "time" // ArgsAddIndexingData holds all the data needed for indexing metrics type ArgsAddIndexingData struct { + StatusCode int GotError bool MessageLen uint64 Topic string diff --git a/metrics/prometheusMetrics.go b/metrics/prometheusMetrics.go index f564102a..4eef6e07 100644 --- a/metrics/prometheusMetrics.go +++ b/metrics/prometheusMetrics.go @@ -2,6 +2,7 @@ package metrics import ( "bytes" + "strconv" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" @@ -34,6 +35,40 @@ func counterMetric(metricName, operation string, shardIDStr string, count uint64 return promMetricAsString(metricFamily) } +func errorsMetric(metricName, operation string, shardIDStr string, errorsCount map[int]uint64) string { + metricFamily := &dto.MetricFamily{ + Name: proto.String(metricName), + Type: dto.MetricType_GAUGE.Enum(), + Metric: make([]*dto.Metric, 0, len(errorsCount)), + } + + for code, count := range errorsCount { + m := &dto.Metric{ + Label: []*dto.LabelPair{ + { + Name: proto.String("operation"), + Value: proto.String(operation), + }, + { + Name: proto.String("shardID"), + Value: proto.String(shardIDStr), + }, + { + Name: proto.String("errorCode"), + Value: proto.String(strconv.Itoa(code)), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(float64(count)), + }, + } + + metricFamily.Metric = append(metricFamily.Metric, m) + } + + return promMetricAsString(metricFamily) +} + func promMetricAsString(metric *dto.MetricFamily) string { out := bytes.NewBuffer(make([]byte, 0)) _, err := expfmt.MetricFamilyToText(out, metric) diff --git a/metrics/statusMetrics.go b/metrics/statusMetrics.go index 58e0fbad..4e95c3e4 100644 --- a/metrics/statusMetrics.go +++ b/metrics/statusMetrics.go @@ -2,6 +2,7 @@ package metrics import ( "bytes" + "net/http" "strings" "sync" "unicode" @@ -14,6 +15,7 @@ const ( errorsCount = "errors_count" totalTime = "total_time" totalData = "total_data" + requestsErrors = "requests_errors" ) type statusMetrics struct { @@ -36,15 +38,21 @@ func (sm *statusMetrics) AddIndexingData(args ArgsAddIndexingData) { topic := camelToSnake(args.Topic) _, found := sm.metrics[topic] if !found { - sm.metrics[topic] = &request.MetricsResponse{} + sm.metrics[topic] = &request.MetricsResponse{ + ErrorsCount: map[int]uint64{}, + } } sm.metrics[topic].OperationsCount++ sm.metrics[topic].TotalIndexingTime += args.Duration sm.metrics[topic].TotalData += args.MessageLen - if args.GotError { - sm.metrics[topic].ErrorsCount++ + isErrorCode := args.StatusCode >= http.StatusBadRequest + if args.GotError || isErrorCode { + sm.metrics[topic].TotalErrorsCount++ + } + if isErrorCode { + sm.metrics[topic].ErrorsCount[args.StatusCode]++ } } @@ -67,9 +75,12 @@ func (sm *statusMetrics) GetMetricsForPrometheus() string { for topicWithShardID, metricsData := range metrics { topic, shardIDStr := request.SplitTopicAndShardID(topicWithShardID) stringBuilder.WriteString(counterMetric(topic, totalData, shardIDStr, metricsData.TotalData)) - stringBuilder.WriteString(counterMetric(topic, errorsCount, shardIDStr, metricsData.ErrorsCount)) + stringBuilder.WriteString(counterMetric(topic, errorsCount, shardIDStr, metricsData.TotalErrorsCount)) stringBuilder.WriteString(counterMetric(topic, operationCount, shardIDStr, metricsData.OperationsCount)) stringBuilder.WriteString(counterMetric(topic, totalTime, shardIDStr, uint64(metricsData.TotalIndexingTime.Milliseconds()))) + if len(metricsData.ErrorsCount) > 0 { + stringBuilder.WriteString(errorsMetric(topic, requestsErrors, shardIDStr, metricsData.ErrorsCount)) + } } promMetricsOutput := stringBuilder.String() diff --git a/metrics/statusMetrics_test.go b/metrics/statusMetrics_test.go index c34865c4..08c05d4a 100644 --- a/metrics/statusMetrics_test.go +++ b/metrics/statusMetrics_test.go @@ -1,6 +1,7 @@ package metrics import ( + "net/http" "testing" "github.com/multiversx/mx-chain-core-go/data/outport" @@ -20,6 +21,7 @@ func TestStatusMetrics_AddIndexingDataAndGetMetrics(t *testing.T) { MessageLen: 100, Topic: topic1, Duration: 12, + StatusCode: http.StatusBadRequest, }) statusMetricsHandler.AddIndexingData(ArgsAddIndexingData{ GotError: false, @@ -31,13 +33,31 @@ func TestStatusMetrics_AddIndexingDataAndGetMetrics(t *testing.T) { metrics := statusMetricsHandler.GetMetrics() require.Equal(t, &request.MetricsResponse{ OperationsCount: 2, - ErrorsCount: 1, + TotalErrorsCount: 1, TotalIndexingTime: 27, TotalData: 322, + ErrorsCount: map[int]uint64{ + http.StatusBadRequest: 1, + }, }, metrics[topic1]) prometheusMetrics := statusMetricsHandler.GetMetricsForPrometheus() - require.Equal(t, "# TYPE test1 counter\ntest1{operation=\"total_data\",shardID=\"0\"} 322\n\n# TYPE test1 counter\ntest1{operation=\"errors_count\",shardID=\"0\"} 1\n\n# TYPE test1 counter\ntest1{operation=\"operations_count\",shardID=\"0\"} 2\n\n# TYPE test1 counter\ntest1{operation=\"total_time\",shardID=\"0\"} 0\n\n", prometheusMetrics) + require.Equal(t, `# TYPE test1 counter +test1{operation="total_data",shardID="0"} 322 + +# TYPE test1 counter +test1{operation="errors_count",shardID="0"} 1 + +# TYPE test1 counter +test1{operation="operations_count",shardID="0"} 2 + +# TYPE test1 counter +test1{operation="total_time",shardID="0"} 0 + +# TYPE test1 gauge +test1{operation="requests_errors",shardID="0",errorCode="400"} 1 + +`, prometheusMetrics) } func TestCamelCaseToSnakeCase(t *testing.T) { From 489ff762867c7cbc4de24a102213f46c7a1edcf2 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 10 Aug 2023 10:35:37 +0300 Subject: [PATCH 151/189] change order --- core/request/requests.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/request/requests.go b/core/request/requests.go index e0eaaab3..717bc66c 100644 --- a/core/request/requests.go +++ b/core/request/requests.go @@ -29,11 +29,11 @@ const ( // MetricsResponse defines the response for status metrics endpoint type MetricsResponse struct { + TotalData uint64 `json:"total_data"` OperationsCount uint64 `json:"operations_count"` - ErrorsCount map[int]uint64 `json:"errors_count,omitempty"` TotalErrorsCount uint64 `json:"total_errors_count"` + ErrorsCount map[int]uint64 `json:"errors_count,omitempty"` TotalIndexingTime time.Duration `json:"total_time"` - TotalData uint64 `json:"total_data"` } // ExtendTopicWithShardID will concatenate topic with shardID From 466afd122372f569823d960fa2e08a4b5e095f26 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 10 Aug 2023 10:38:21 +0300 Subject: [PATCH 152/189] consts --- metrics/prometheusMetrics.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/metrics/prometheusMetrics.go b/metrics/prometheusMetrics.go index 4eef6e07..8baccf42 100644 --- a/metrics/prometheusMetrics.go +++ b/metrics/prometheusMetrics.go @@ -9,6 +9,12 @@ import ( "google.golang.org/protobuf/proto" ) +const ( + operationName = "operation" + shardIDName = "shardID" + errorCodeName = "errorCode" +) + func counterMetric(metricName, operation string, shardIDStr string, count uint64) string { metricFamily := &dto.MetricFamily{ Name: proto.String(metricName), @@ -17,11 +23,11 @@ func counterMetric(metricName, operation string, shardIDStr string, count uint64 { Label: []*dto.LabelPair{ { - Name: proto.String("operation"), + Name: proto.String(operationName), Value: proto.String(operation), }, { - Name: proto.String("shardID"), + Name: proto.String(shardIDName), Value: proto.String(shardIDStr), }, }, @@ -46,15 +52,15 @@ func errorsMetric(metricName, operation string, shardIDStr string, errorsCount m m := &dto.Metric{ Label: []*dto.LabelPair{ { - Name: proto.String("operation"), + Name: proto.String(operationName), Value: proto.String(operation), }, { - Name: proto.String("shardID"), + Name: proto.String(shardIDName), Value: proto.String(shardIDStr), }, { - Name: proto.String("errorCode"), + Name: proto.String(errorCodeName), Value: proto.String(strconv.Itoa(code)), }, }, From 86823f09d934f740ccc72b1c68f0edaaee1b43ed Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 10 Aug 2023 10:41:41 +0300 Subject: [PATCH 153/189] fix unit test --- client/transport/transport_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/transport/transport_test.go b/client/transport/transport_test.go index 26d8513e..31c78ef9 100644 --- a/client/transport/transport_test.go +++ b/client/transport/transport_test.go @@ -55,7 +55,7 @@ func TestMetricsTransport_RoundTrip(t *testing.T) { metricsMap := metricsHandler.GetMetrics() require.Equal(t, uint64(1), metricsMap[testTopic].OperationsCount) - require.Equal(t, uint64(0), metricsMap[testTopic].ErrorsCount) + require.Equal(t, uint64(0), metricsMap[testTopic].TotalErrorsCount) require.Equal(t, uint64(4), metricsMap[testTopic].TotalData) } From cfda2617a2ce99e32cac18c995b1ec950e420b46 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 10 Aug 2023 10:59:26 +0300 Subject: [PATCH 154/189] small fix --- metrics/statusMetrics.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metrics/statusMetrics.go b/metrics/statusMetrics.go index 4e95c3e4..06222ff4 100644 --- a/metrics/statusMetrics.go +++ b/metrics/statusMetrics.go @@ -78,9 +78,7 @@ func (sm *statusMetrics) GetMetricsForPrometheus() string { stringBuilder.WriteString(counterMetric(topic, errorsCount, shardIDStr, metricsData.TotalErrorsCount)) stringBuilder.WriteString(counterMetric(topic, operationCount, shardIDStr, metricsData.OperationsCount)) stringBuilder.WriteString(counterMetric(topic, totalTime, shardIDStr, uint64(metricsData.TotalIndexingTime.Milliseconds()))) - if len(metricsData.ErrorsCount) > 0 { - stringBuilder.WriteString(errorsMetric(topic, requestsErrors, shardIDStr, metricsData.ErrorsCount)) - } + stringBuilder.WriteString(errorsMetric(topic, requestsErrors, shardIDStr, metricsData.ErrorsCount)) } promMetricsOutput := stringBuilder.String() From 86531b008c5f6380805fe029909212dccc3fe902 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 10 Aug 2023 11:07:53 +0300 Subject: [PATCH 155/189] cluster stop --- scripts/observers/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/observers/Makefile b/scripts/observers/Makefile index 5713cf94..d7239c3c 100644 --- a/scripts/observers/Makefile +++ b/scripts/observers/Makefile @@ -6,6 +6,7 @@ help: @echo " stop - Stop the local testnet" @echo " clean - Clean up the files of the local testnet" @echo " cluster - Start an Elasticsearch cluster on port 9200" + @echo " cluster-stop - Stop the Elasticsearch cluster" @echo " monitor-start - Start the Prometheus server (port 9090) and the Grafana server (port 3000)" @echo " monitor-Stop - Stop the Prometheus server (port 9090) and the Grafana server (port 3000)" @@ -20,6 +21,8 @@ clean: cd .. && /bin/bash script.sh delete cluster: cd .. && /bin/bash script.sh start +cluster-stop: + cd .. && /bin/bash script.sh stop monitor-start: cd .. && /bin/bash script.sh start_prometheus_and_grafana monitor-stop: From 1760ed9a299ca34128799548477368d1dbcf3ac9 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 10 Aug 2023 14:32:12 +0300 Subject: [PATCH 156/189] fixes after review --- integrationtests/scDeploy_test.go | 36 +++++++++++++++++-- ...after-upgrade-and-change-owner-second.json | 27 ++++++++++++++ process/elasticproc/logsevents/scDeploys.go | 3 +- 3 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 integrationtests/testdata/scDeploy/deploy-after-upgrade-and-change-owner-second.json diff --git a/integrationtests/scDeploy_test.go b/integrationtests/scDeploy_test.go index 50d22f90..95578c22 100644 --- a/integrationtests/scDeploy_test.go +++ b/integrationtests/scDeploy_test.go @@ -115,7 +115,7 @@ func TestTransactionWithSCDeploy(t *testing.T) { Events: []*transaction.Event{ { Address: decodeAddress(sndAddress), - Identifier: []byte(core.SCDeployIdentifier), + Identifier: []byte(core.SCUpgradeIdentifier), Topics: [][]byte{decodeAddress("erd1qqqqqqqqqqqqqpgq4t2tqxpst9a6qttpak8cz8wvz6a0nses5qfqel6rhy"), decodeAddress("erd12m3x8jp6dl027pj5f2nw6ght2cyhhjfrs86cdwsa8xn83r375qfqrwpdx0"), []byte("secondCodeHash")}, }, nil, @@ -136,7 +136,7 @@ func TestTransactionWithSCDeploy(t *testing.T) { string(genericResponse.Docs[0].Source), ) - // CHANGE owner + // CHANGE owner first header.TimeStamp = 7000 pool = &outport.TransactionPool{ Logs: []*outport.LogData{ @@ -167,4 +167,36 @@ func TestTransactionWithSCDeploy(t *testing.T) { readExpectedResult("./testdata/scDeploy/deploy-after-upgrade-and-change-owner.json"), string(genericResponse.Docs[0].Source), ) + + // CHANGE owner second + header.TimeStamp = 8000 + pool = &outport.TransactionPool{ + Logs: []*outport.LogData{ + { + TxHash: hex.EncodeToString([]byte("h4")), + Log: &transaction.Log{ + Address: decodeAddress(sndAddress), + Events: []*transaction.Event{ + { + Address: decodeAddress("erd1qqqqqqqqqqqqqpgq4t2tqxpst9a6qttpak8cz8wvz6a0nses5qfqel6rhy"), + Identifier: []byte(core.BuiltInFunctionChangeOwnerAddress), + Topics: [][]byte{decodeAddress("erd1y78ds2tvzw6ntcggldjld2vk96wgq0mj47mk6auny0nkvn242e3sd4qz7m")}, + }, + nil, + }, + }, + }, + }, + } + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) + require.Nil(t, err) + + ids = []string{"erd1qqqqqqqqqqqqqpgq4t2tqxpst9a6qttpak8cz8wvz6a0nses5qfqel6rhy"} + err = esClient.DoMultiGet(ids, indexerData.SCDeploysIndex, true, genericResponse) + require.Nil(t, err) + + require.JSONEq(t, + readExpectedResult("./testdata/scDeploy/deploy-after-upgrade-and-change-owner-second.json"), + string(genericResponse.Docs[0].Source), + ) } diff --git a/integrationtests/testdata/scDeploy/deploy-after-upgrade-and-change-owner-second.json b/integrationtests/testdata/scDeploy/deploy-after-upgrade-and-change-owner-second.json new file mode 100644 index 00000000..eaadb107 --- /dev/null +++ b/integrationtests/testdata/scDeploy/deploy-after-upgrade-and-change-owner-second.json @@ -0,0 +1,27 @@ +{ + "deployTxHash": "6831", + "deployer": "erd12m3x8jp6dl027pj5f2nw6ght2cyhhjfrs86cdwsa8xn83r375qfqrwpdx0", + "currentOwner": "erd1y78ds2tvzw6ntcggldjld2vk96wgq0mj47mk6auny0nkvn242e3sd4qz7m", + "initialCodeHash": "Y29kZUhhc2g=", + "timestamp": 5040, + "upgrades": [ + { + "upgradeTxHash": "6832", + "codeHash": "c2Vjb25kQ29kZUhhc2g=", + "upgrader": "erd12m3x8jp6dl027pj5f2nw6ght2cyhhjfrs86cdwsa8xn83r375qfqrwpdx0", + "timestamp": 6000 + } + ], + "owners": [ + { + "address": "erd1d942l8w4yvgjffpqacs8vdwl0mndsv0zn0uxa80hxc3xmq4477eqnyw3dh", + "txHash": "6833", + "timestamp": 7000 + }, + { + "address": "erd1y78ds2tvzw6ntcggldjld2vk96wgq0mj47mk6auny0nkvn242e3sd4qz7m", + "txHash": "6834", + "timestamp": 8000 + } + ] +} diff --git a/process/elasticproc/logsevents/scDeploys.go b/process/elasticproc/logsevents/scDeploys.go index 6ab0b204..fe255702 100644 --- a/process/elasticproc/logsevents/scDeploys.go +++ b/process/elasticproc/logsevents/scDeploys.go @@ -8,6 +8,7 @@ import ( ) const ( + numTopicsChangeOwner = 1 minTopicsContractEvent = 3 ) @@ -36,7 +37,7 @@ func (sdp *scDeploysProcessor) processEvent(args *argsProcessEvent) argOutputPro } topics := args.event.GetTopics() - isChangeOwnerEvent := len(topics) == 1 && eventIdentifier == core.BuiltInFunctionChangeOwnerAddress + isChangeOwnerEvent := len(topics) == numTopicsChangeOwner && eventIdentifier == core.BuiltInFunctionChangeOwnerAddress if isChangeOwnerEvent { return sdp.processChangeOwnerEvent(args) } From 074264f5d84170cb5058c070a7988ace0f4d7ca9 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 10 Aug 2023 14:39:18 +0300 Subject: [PATCH 157/189] proper tag --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b40e7657..d072f986 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.0 github.com/multiversx/mx-chain-communication-go v1.0.3 - github.com/multiversx/mx-chain-core-go v1.2.12-0.20230725101425-a34bd6d3e3b9 + github.com/multiversx/mx-chain-core-go v1.2.12 github.com/multiversx/mx-chain-logger-go v1.0.11 github.com/multiversx/mx-chain-vm-common-go v1.4.1 github.com/prometheus/client_model v0.2.0 diff --git a/go.sum b/go.sum index 5b4c6d8b..5fa13668 100644 --- a/go.sum +++ b/go.sum @@ -525,8 +525,8 @@ github.com/multiversx/mx-chain-communication-go v1.0.3/go.mod h1:7oTI77XfWmRWwVE github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= github.com/multiversx/mx-chain-core-go v1.2.5/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= -github.com/multiversx/mx-chain-core-go v1.2.12-0.20230725101425-a34bd6d3e3b9 h1:oa48Q+QpF/3LHVR0ug7eFENoIVKKZL9PfldABT9cL3A= -github.com/multiversx/mx-chain-core-go v1.2.12-0.20230725101425-a34bd6d3e3b9/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= +github.com/multiversx/mx-chain-core-go v1.2.12 h1:k8hS/Mu+t8VxP2ynF28ZkGu5qDZmhN8i5gXxqWUI8JM= +github.com/multiversx/mx-chain-core-go v1.2.12/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= github.com/multiversx/mx-chain-crypto-go v1.2.6/go.mod h1:rOj0Rr19HTOYt9YTeym7RKxlHt91NXln3LVKjHKVmA0= github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= From fde197ff5a1d97fbd8915a548443f39cafc80280 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 10 Aug 2023 15:11:19 +0300 Subject: [PATCH 158/189] fix integration tests --- integrationtests/scDeploy_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integrationtests/scDeploy_test.go b/integrationtests/scDeploy_test.go index 1b8b379e..4a1603bb 100644 --- a/integrationtests/scDeploy_test.go +++ b/integrationtests/scDeploy_test.go @@ -129,7 +129,7 @@ func TestTransactionWithSCDeploy(t *testing.T) { require.Nil(t, err) ids = []string{"erd1qqqqqqqqqqqqqpgq4t2tqxpst9a6qttpak8cz8wvz6a0nses5qfqel6rhy"} - err = esClient.DoMultiGet(ids, indexerData.SCDeploysIndex, true, genericResponse) + err = esClient.DoMultiGet(context.Background(), ids, indexerData.SCDeploysIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, @@ -161,7 +161,7 @@ func TestTransactionWithSCDeploy(t *testing.T) { require.Nil(t, err) ids = []string{"erd1qqqqqqqqqqqqqpgq4t2tqxpst9a6qttpak8cz8wvz6a0nses5qfqel6rhy"} - err = esClient.DoMultiGet(ids, indexerData.SCDeploysIndex, true, genericResponse) + err = esClient.DoMultiGet(context.Background(), ids, indexerData.SCDeploysIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, @@ -193,7 +193,7 @@ func TestTransactionWithSCDeploy(t *testing.T) { require.Nil(t, err) ids = []string{"erd1qqqqqqqqqqqqqpgq4t2tqxpst9a6qttpak8cz8wvz6a0nses5qfqel6rhy"} - err = esClient.DoMultiGet(ids, indexerData.SCDeploysIndex, true, genericResponse) + err = esClient.DoMultiGet(context.Background(), ids, indexerData.SCDeploysIndex, true, genericResponse) require.Nil(t, err) require.JSONEq(t, From 2e376ab7c63c443a4b6878578571cba962769bfc Mon Sep 17 00:00:00 2001 From: Iulian Pascalau Date: Wed, 16 Aug 2023 15:56:25 +0300 Subject: [PATCH 159/189] - new go definition in go.mod - reworked GitHub workflow files --- .github/workflows/golangci-lint.yml | 2 +- .github/workflows/pr-build.yml | 4 +- .github/workflows/pr-integration-tests.yml | 4 +- .github/workflows/pr-tests.yml | 4 +- go.mod | 18 +- go.sum | 542 +-------------------- 6 files changed, 33 insertions(+), 541 deletions(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index ac7d9c40..3c1d0d1b 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -3,7 +3,7 @@ on: push: branches: [ master ] pull_request: - branches: [ master, development, feat/*, rc/* ] + branches: [ master, feat/*, rc/* ] permissions: contents: read diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index d022d309..42dedd2b 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -3,9 +3,9 @@ name: Go build on: push: - branches: [ master, development, feat/*, rc/* ] + branches: [ master, feat/*, rc/* ] pull_request: - branches: [ master, development, feat/*, rc/* ] + branches: [ master, feat/*, rc/* ] jobs: build: diff --git a/.github/workflows/pr-integration-tests.yml b/.github/workflows/pr-integration-tests.yml index 54e20874..93500145 100644 --- a/.github/workflows/pr-integration-tests.yml +++ b/.github/workflows/pr-integration-tests.yml @@ -2,9 +2,9 @@ name: Integration tests on: push: - branches: [ master, development, feat/*, rc/* ] + branches: [ master, feat/*, rc/* ] pull_request: - branches: [ master, development, feat/*, rc/* ] + branches: [ master, feat/*, rc/* ] jobs: test-0: diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 67f814e3..cb27be4b 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -2,9 +2,9 @@ name: Tests on: push: - branches: [ master, development, feat/*, rc/* ] + branches: [ master, feat/*, rc/* ] pull_request: - branches: [ master, development, feat/*, rc/* ] + branches: [ master, feat/*, rc/* ] jobs: test: diff --git a/go.mod b/go.mod index d072f986..34faa2e6 100644 --- a/go.mod +++ b/go.mod @@ -1,21 +1,21 @@ module github.com/multiversx/mx-chain-es-indexer-go -go 1.17 +go 1.20 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gin-contrib/cors v1.4.0 - github.com/gin-gonic/gin v1.9.0 - github.com/multiversx/mx-chain-communication-go v1.0.3 - github.com/multiversx/mx-chain-core-go v1.2.12 - github.com/multiversx/mx-chain-logger-go v1.0.11 - github.com/multiversx/mx-chain-vm-common-go v1.4.1 - github.com/prometheus/client_model v0.2.0 + github.com/gin-gonic/gin v1.9.1 + github.com/multiversx/mx-chain-communication-go v1.0.6-0.20230816102554-567e1f606ee9 + github.com/multiversx/mx-chain-core-go v1.2.13-0.20230816090241-133da66033a9 + github.com/multiversx/mx-chain-logger-go v1.0.13-0.20230816094710-e4e216ffef4e + github.com/multiversx/mx-chain-vm-common-go v1.5.1-0.20230816095353-9830338a04e4 + github.com/prometheus/client_model v0.4.0 github.com/prometheus/common v0.37.0 github.com/stretchr/testify v1.8.4 github.com/tidwall/gjson v1.14.0 github.com/urfave/cli v1.22.10 - google.golang.org/protobuf v1.28.1 + google.golang.org/protobuf v1.30.0 ) require ( @@ -40,7 +40,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/mattn/go-isatty v0.0.19 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mr-tron/base58 v1.2.0 // indirect diff --git a/go.sum b/go.sum index 5fa13668..0cc7e50e 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= @@ -32,37 +30,20 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= -dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= -dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= -filippo.io/edwards25519 v1.0.0/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= -git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= -github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M= -github.com/btcsuite/btcd v0.22.1/go.mod h1:wqgTSL29+50LRkmOVknEdmt8ZojIzhuWvgu/iptuN7Y= -github.com/btcsuite/btcd v0.23.0 h1:V2/ZgjfDFIygAX3ZapeigkVBoVUtOJKSwrhZdlpSvaA= github.com/btcsuite/btcd v0.23.0/go.mod h1:0QJIIN1wwIXF/3G/m87gIwGniDMDQqjVn4SZgnFpsYY= github.com/btcsuite/btcd/btcec/v2 v2.1.0/go.mod h1:2VzYrv4Gm4apmbVVsSq5bqf1Ec8v56E48Vt0Y/umPgA= github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= @@ -74,7 +55,6 @@ github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtyd github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= @@ -82,37 +62,20 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.8.0/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= -github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= -github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -121,48 +84,28 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= -github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= -github.com/dgraph-io/badger v1.6.1/go.mod h1:FRmFw3uxvcpa8zG3Rxs0th+hCLIuaQg8HlNV5bjgnuU= -github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= -github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elastic/go-elasticsearch/v7 v7.12.0 h1:j4tvcMrZJLp39L2NYvBb7f+lHKPqPHSL3nvB8+/DV+s= github.com/elastic/go-elasticsearch/v7 v7.12.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= -github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= -github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= -github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g= github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= -github.com/gin-gonic/gin v1.9.0 h1:OjyFBKICoexlu99ctXNR2gg+c5pKrKMuyjgARg9qeY8= -github.com/gin-gonic/gin v1.9.0/go.mod h1:W1Me9+hsUSyj3CePGrd1/QrKJMSJ1Tu/0hFEH89961k= -github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= +github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -174,12 +117,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= -github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= @@ -187,29 +126,19 @@ github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= -github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s= github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -217,7 +146,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -236,7 +164,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -247,18 +174,11 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= -github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -269,77 +189,16 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= -github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= -github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= -github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/herumi/bls-go-binary v1.0.0/go.mod h1:O4Vp1AfR4raRGwFeQpr9X/PQtncEicMoOe6BQt1oX0Y= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= -github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= -github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= -github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= -github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= -github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= -github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro= -github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= -github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= -github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= -github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= -github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= -github.com/ipfs/go-datastore v0.5.1/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= -github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= -github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= -github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk= -github.com/ipfs/go-ds-badger v0.3.0/go.mod h1:1ke6mXNqeV8K3y5Ak2bAA0osoTfmxUdupVCGm4QUIek= -github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= -github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= -github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= -github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= -github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= -github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= -github.com/ipfs/go-ipns v0.2.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24= -github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= -github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A= -github.com/ipfs/go-log v1.0.4/go.mod h1:oDCg2FkjogeFOhqqb+N39l2RpTNPL6F/StPkB3kPgcs= -github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= -github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= -github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= -github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= -github.com/ipfs/go-log/v2 v2.5.0/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= -github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= -github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= -github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= -github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= -github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= -github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= -github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= -github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -351,127 +210,34 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= -github.com/koron/go-ssdp v0.0.3/go.mod h1:b2MxI6yh02pKrsyNoQUsk4+YNikaGhe4894J+Q5lDvA= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= -github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= -github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= -github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= -github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= -github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= -github.com/libp2p/go-flow-metrics v0.0.2/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= -github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= -github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.22.0/go.mod h1:UDolmweypBSjQb2f7xutPnwZ/fxioLbMBxSjRksxxU4= -github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo= -github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= -github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= -github.com/libp2p/go-libp2p-core v0.2.5/go.mod h1:6+5zJmKhsf7yHn1RbmYDu08qDUpIUxGdqHuEZckmZOA= -github.com/libp2p/go-libp2p-core v0.3.0/go.mod h1:ACp3DmS3/N64c2jDzcV429ukDpicbL6+TrrxANBjPGw= -github.com/libp2p/go-libp2p-core v0.5.3/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y= -github.com/libp2p/go-libp2p-core v0.5.4/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y= -github.com/libp2p/go-libp2p-core v0.6.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= -github.com/libp2p/go-libp2p-core v0.19.0/go.mod h1:AkA+FUKQfYt1FLNef5fOPlo/naAWjKy/RCjkcPjqzYg= -github.com/libp2p/go-libp2p-core v0.20.0/go.mod h1:6zR8H7CvQWgYLsbG4on6oLNSGcyKaYFSEYyDt51+bIY= -github.com/libp2p/go-libp2p-kad-dht v0.18.0/go.mod h1:Gb92MYIPm3K2pJLGn8wl0m8wiKDvHrYpg+rOd0GzzPA= -github.com/libp2p/go-libp2p-kbucket v0.3.1/go.mod h1:oyjT5O7tS9CQurok++ERgc46YLwEpuGoFq9ubvoUOio= -github.com/libp2p/go-libp2p-kbucket v0.4.7/go.mod h1:XyVo99AfQH0foSf176k4jY1xUJ2+jUJIZCSDm7r2YKk= -github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs= -github.com/libp2p/go-libp2p-peerstore v0.2.6/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= -github.com/libp2p/go-libp2p-peerstore v0.8.0/go.mod h1:9geHWmNA3YDlQBjL/uPEJD6vpDK12aDNlUNHJ6kio/s= -github.com/libp2p/go-libp2p-pubsub v0.8.1/go.mod h1:e4kT+DYjzPUYGZeWk4I+oxCSYTXizzXii5LDRRhjKSw= -github.com/libp2p/go-libp2p-record v0.1.2/go.mod h1:pal0eNcT5nqZaTV7UGhqeGqxFgGdsU/9W//C8dqjQDk= -github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= -github.com/libp2p/go-libp2p-routing-helpers v0.2.3/go.mod h1:795bh+9YeoFl99rMASoiVgHdi5bjack0N1+AFAdbvBw= -github.com/libp2p/go-libp2p-testing v0.11.0/go.mod h1:qG4sF27dfKFoK9KlVzK2y52LQKhp0VEmLjV5aDqr1Hg= -github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= -github.com/libp2p/go-libp2p-xor v0.1.0/go.mod h1:LSTM5yRnjGZbWNTA/hRwq2gGFrvRIbQJscoIL/u6InY= -github.com/libp2p/go-maddr-filter v0.1.0/go.mod h1:VzZhTXkMucEGGEOSKddrwGiOv0tUhgnKqNEmIAz/bPU= -github.com/libp2p/go-mplex v0.7.0/go.mod h1:rW8ThnRcYWft/Jb2jeORBmPd6xuG3dGxWN/W168L9EU= -github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= -github.com/libp2p/go-msgio v0.0.6/go.mod h1:4ecVB6d9f4BDSL5fqvPiC4A3KivjWn+Venn/1ALLMWA= -github.com/libp2p/go-msgio v0.2.0/go.mod h1:dBVM1gW3Jk9XqHkU4eKdGvVHdLa51hoGfll6jMJMSlY= -github.com/libp2p/go-nat v0.1.0/go.mod h1:X7teVkwRHNInVNWQiO/tAiAVRwSr5zoRz4YSTC3uRBM= -github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= -github.com/libp2p/go-netroute v0.2.0/go.mod h1:Vio7LTzZ+6hoT4CMZi5/6CpY3Snzh2vgZhWgxMNwlQI= -github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= -github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= -github.com/libp2p/go-openssl v0.0.7/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= -github.com/libp2p/go-openssl v0.1.0/go.mod h1:OiOxwPpL3n4xlenjx2h7AwSGaFSC/KZvf6gNdOBQMtc= -github.com/libp2p/go-reuseport v0.2.0/go.mod h1:bvVho6eLMm6Bz5hmU0LYN3ixd3nPPvtIlaURZZgOY4k= -github.com/libp2p/go-sockaddr v0.0.2/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= -github.com/libp2p/go-yamux/v3 v3.1.2/go.mod h1:jeLEQgLXqE2YqX1ilAClIfCMDY+0uXQUKmmb/qp0gT4= -github.com/libp2p/zeroconf/v2 v2.2.0/go.mod h1:fuJqLnUwZTshS3U/bMRJ3+ow/v9oid1n0DmyYyNO1Xs= -github.com/lucas-clemente/quic-go v0.28.1/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0= -github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= -github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= -github.com/marten-seemann/qtls-go1-17 v0.1.2/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= -github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= -github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= -github.com/marten-seemann/qtls-go1-19 v0.1.0/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= -github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= -github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= -github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= -github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKoFL8DUUmalo2yJJUCxbPKtm8OKfqr2/FTNU= -github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= -github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= -github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= -github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -479,89 +245,31 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= -github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= -github.com/multiformats/go-base32 v0.0.4/go.mod h1:jNLFzjPZtp3aIARHbJRZIaPuspdH0J6q39uUM5pnABM= -github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= -github.com/multiformats/go-multiaddr v0.1.0/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= -github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= -github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= -github.com/multiformats/go-multiaddr v0.2.1/go.mod h1:s/Apk6IyxfvMjDafnhJgJ3/46z7tZ04iMk5wP4QMGGE= -github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u0xW5UouOmQQrn6a3Y= -github.com/multiformats/go-multiaddr v0.3.0/go.mod h1:dF9kph9wfJ+3VLAaeBqo9Of8x4fJxp6ggJGteB8HQTI= -github.com/multiformats/go-multiaddr v0.3.1/go.mod h1:uPbspcUPd5AfaP6ql3ujFY+QWzmBD8uLLL4bXW0XfGc= -github.com/multiformats/go-multiaddr v0.4.1/go.mod h1:3afI9HfVW8csiF8UZqtpYRiDyew8pRX7qLIGHu9FLuM= -github.com/multiformats/go-multiaddr v0.6.0/go.mod h1:F4IpaKZuPP360tOMn2Tpyu0At8w23aRyVqeK0DbFeGM= -github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= -github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= -github.com/multiformats/go-multiaddr-net v0.1.1/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= -github.com/multiformats/go-multiaddr-net v0.1.4/go.mod h1:ilNnaM9HbmVFqsb/qcNysjCu4PVONlrBZpHIrw/qQuA= -github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5rnK7YaOIEOPVDI9tsJbEA= -github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= -github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= -github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= -github.com/multiformats/go-multicodec v0.4.1/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= -github.com/multiformats/go-multicodec v0.5.0/go.mod h1:DiY2HFaEp5EhEXb/iYzVAunmyX/aSFMxq2KMKfWEues= -github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= -github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= -github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= -github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= -github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= -github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= -github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= -github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= -github.com/multiformats/go-multistream v0.3.3/go.mod h1:ODRoqamLUsETKS9BNcII4gcRsJBU5VAwRIv7O39cEXg= -github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= -github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= -github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= -github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= -github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= -github.com/multiversx/mx-chain-communication-go v1.0.3 h1:42LkNQX+PdT4UEDO2/kbOMT/iDpEMfE0ncNldFTyFD4= -github.com/multiversx/mx-chain-communication-go v1.0.3/go.mod h1:7oTI77XfWmRWwVEbCq+pjH5CO3mJ6vEiHGMvQv6vF3Y= -github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk= -github.com/multiversx/mx-chain-core-go v1.2.5/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= -github.com/multiversx/mx-chain-core-go v1.2.12 h1:k8hS/Mu+t8VxP2ynF28ZkGu5qDZmhN8i5gXxqWUI8JM= -github.com/multiversx/mx-chain-core-go v1.2.12/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ= -github.com/multiversx/mx-chain-crypto-go v1.2.6/go.mod h1:rOj0Rr19HTOYt9YTeym7RKxlHt91NXln3LVKjHKVmA0= -github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE= -github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA= -github.com/multiversx/mx-chain-storage-go v1.0.8/go.mod h1:lEkFYFe6taKYxqO1einNgT1esY3K9Pj6vPnoCwV9C3U= -github.com/multiversx/mx-chain-vm-common-go v1.4.1 h1:HHZF9zU4WsMbfLrCarx3ESM95caWUrPBleGHKdsbzgc= -github.com/multiversx/mx-chain-vm-common-go v1.4.1/go.mod h1:K6yCdro8VohzYI6GwjGzTO+fJiPgO5coo2sgQb+zA24= +github.com/multiversx/mx-chain-communication-go v1.0.6-0.20230816102554-567e1f606ee9 h1:5F8Vy5OdcTJDAqdjQ843PHZOlxjZ4mBFA42A7Pd3ssI= +github.com/multiversx/mx-chain-communication-go v1.0.6-0.20230816102554-567e1f606ee9/go.mod h1:tE+3vTDUuhL/sso5Zpx81jmP0m2W92gczAA4sJJCZgw= +github.com/multiversx/mx-chain-core-go v1.2.13-0.20230816090241-133da66033a9 h1:dH8qhDjdURgKSJggqaXyZcQd4bL3S2LW9mjjMivzoUw= +github.com/multiversx/mx-chain-core-go v1.2.13-0.20230816090241-133da66033a9/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= +github.com/multiversx/mx-chain-crypto-go v1.2.8-0.20230816100617-0f47cf084591 h1:fb7qi89NqJ3tdTVis+uNKUg+XJ4BnaQLv7NzJJyiMrw= +github.com/multiversx/mx-chain-logger-go v1.0.13-0.20230816094710-e4e216ffef4e h1:yHTUbBPdd7PSWU2ckgH00r+o0PYUu/Xzq6mGmOsJ0aQ= +github.com/multiversx/mx-chain-logger-go v1.0.13-0.20230816094710-e4e216ffef4e/go.mod h1:dE16PsJFsDubcKtQDV6C5Qk3DeMKWqoX5MhM9oqZlmI= +github.com/multiversx/mx-chain-vm-common-go v1.5.1-0.20230816095353-9830338a04e4 h1:b061kswdx2aImnRfBlN8zMVwkvtGEcNgMgyewEepGyM= +github.com/multiversx/mx-chain-vm-common-go v1.5.1-0.20230816095353-9830338a04e4/go.mod h1:VxN7ZFxFJocIRXso/60MYIKpCF0phQt3NQihpvsJjC0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= -github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= -github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= -github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= -github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -570,8 +278,6 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= -github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -580,76 +286,31 @@ github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrb github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= -github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= -github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= -github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= -github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= -github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= -github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI= -github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU= -github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag= -github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg= -github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= -github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= -github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= -github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= -github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= -github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= -github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0= -github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= -github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= -github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= -github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= -github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= -github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= -github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -657,7 +318,6 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -667,9 +327,7 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= -github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -678,89 +336,32 @@ github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go v1.2.7 h1:qYhyWUUd6WbiM+C6JZAUkIJt/1WrjzNHY9+KCIjVqTo= github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= -github.com/ugorji/go/codec v1.2.9/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.10 h1:p8Fspmz3iTctJstry1PYS3HVdllxnEzTEsgIgtxTrCk= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= -github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= -github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE= -github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= -github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= -github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= -github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= -github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= -go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= -go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U= -go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -775,7 +376,6 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -794,22 +394,14 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -835,38 +427,19 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -877,43 +450,28 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -925,48 +483,26 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -974,29 +510,17 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -1006,9 +530,6 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1035,20 +556,11 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1066,18 +578,12 @@ google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= -google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1106,9 +612,6 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1121,7 +624,6 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1135,8 +637,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1145,23 +647,17 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= -gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1169,11 +665,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= -lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= -sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= From aede91f0c3d306297987c65b8964175f58278368 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 17 Aug 2023 10:23:38 +0300 Subject: [PATCH 160/189] fixes --- process/elasticproc/logsevents/scDeploys.go | 1 - scripts/observers/.env | 2 +- scripts/observers/config.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/process/elasticproc/logsevents/scDeploys.go b/process/elasticproc/logsevents/scDeploys.go index fe255702..03c990ff 100644 --- a/process/elasticproc/logsevents/scDeploys.go +++ b/process/elasticproc/logsevents/scDeploys.go @@ -29,7 +29,6 @@ func newSCDeploysProcessor(pubKeyConverter core.PubkeyConverter) *scDeploysProce } func (sdp *scDeploysProcessor) processEvent(args *argsProcessEvent) argOutputProcessEvent { - eventIdentifier := string(args.event.GetIdentifier()) _, ok := sdp.scDeploysIdentifiers[eventIdentifier] if !ok { diff --git a/scripts/observers/.env b/scripts/observers/.env index 5d154fa8..2797b76e 100644 --- a/scripts/observers/.env +++ b/scripts/observers/.env @@ -4,7 +4,7 @@ MX_CHAIN_PROXY_URL=https://github.com/multiversx/mx-chain-proxy-go NODE_GO_URL="https://github.com/multiversx/mx-chain-go" # TODO change the branch after tests -NODE_GO_BRANCH="indexer-with-metrics" +NODE_GO_BRANCH="extend-sc-deploy-and-upgrade-log" WORKING_DIRECTORY="IndexerObservers" OBSERVER_DIR_PREFIX="observer_shard_" diff --git a/scripts/observers/config.py b/scripts/observers/config.py index e52bd98d..eefd6436 100644 --- a/scripts/observers/config.py +++ b/scripts/observers/config.py @@ -268,7 +268,7 @@ def main(): # build binary mx-chain-go print("building node...") mx_chain_go_folder = working_dir / "mx-chain-go" - flags = '-gcflags="all=-N -l"' + flags = '-gcflags=all=-N -l' subprocess.check_call(["go", "build", flags], cwd=mx_chain_go_folder / "cmd/node") # build binary indexer From 05d1e145a3f66cb3559c816a78ad1094dc1b19e5 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 17 Aug 2023 12:14:43 +0300 Subject: [PATCH 161/189] update readme --- README.md | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c9e05ee8..ad61301e 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,27 @@ -#### mx-chain-es-indexer-go +## mx-chain-es-indexer-go -Is a go module that is used in mx-chain-go repository. +Is a versatile component designed to enhance the data management capabilities of the mx-chain-go repository. +This module can be seamlessly integrated as a Go module or deployed as a separate microservice that communicates with [mx-chain-go](https://github.com/multiversx/mx-chain-go) +over a `WebSocket` connection. +Its core functionality revolves around the efficient interaction with an `Elasticsearch` database. -This module is responsible for the interaction with the Elasticsearch database. +### Overview -mx-chain-es-indexer-go module will prepare all the information in a specific format for the external database. +Modern applications require sophisticated data handling mechanisms to ensure optimal performance, data integrity, and real-time accessibility. +`mx-chain-es-indexer-go` addresses these requirements by facilitating data transformation, indexing, and storage within an Elasticsearch database. +This module's flexibility enables it to be used in two distinct modes: as a Go module or as a separate microservice. + +### Features +- **Data Transformation**: `mx-chain-es-indexer-go` excels at transforming raw data generated by the `mx-chain-go` repository into a structured format suitable for indexing and storage. + +- **Indexing**: The module efficiently indexes the transformed data within Elasticsearch, enhancing query performance and data retrieval. + +- **Microservice Mode**: In microservice mode, mx-chain-es-indexer-go operates as a standalone microservice that communicates with mx-chain-go via WebSocket connections. This architecture promotes modularity and scalability. + +- **Go Module Mode**: In Go module mode, the module can be directly integrated into the mx-chain-go repository, offering seamless data management capabilities without the need for a separate microservice. + + +### Contribution +Contributions to the mx-chain-es-indexer-go module are welcomed. Whether you're interested in improving its features, +extending its capabilities, or addressing issues, your contributions can help the community make the module even more robust. From a4a1a904295b344d6f0d740245ff956139b2ee66 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 17 Aug 2023 12:42:11 +0300 Subject: [PATCH 162/189] continue readme --- README.md | 128 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 127 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ad61301e..2e76c30f 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Modern applications require sophisticated data handling mechanisms to ensure opt This module's flexibility enables it to be used in two distinct modes: as a Go module or as a separate microservice. ### Features + - **Data Transformation**: `mx-chain-es-indexer-go` excels at transforming raw data generated by the `mx-chain-go` repository into a structured format suitable for indexing and storage. - **Indexing**: The module efficiently indexes the transformed data within Elasticsearch, enhancing query performance and data retrieval. @@ -21,7 +22,132 @@ This module's flexibility enables it to be used in two distinct modes: as a Go m - **Go Module Mode**: In Go module mode, the module can be directly integrated into the mx-chain-go repository, offering seamless data management capabilities without the need for a separate microservice. +### Running as a Separate Microservice + +Running the `mx-chain-es-indexer-go` module as a **microservice** allows you to efficiently manage data indexing and storage while maintaining +modularity and scalability. This guide outlines the steps to deploy the module as a separate microservice that communicates with +the `mx-chain-go` repository over `WebSocket` connections. + +### Monitoring Endpoints + +The `mx-chain-es-indexer-go` microservice provides monitoring endpoints that allow you to keep track of its health, performance, and other vital statistics. +These endpoints are essential for maintaining the stability and efficiency of the microservice. + +#### Monitoring Endpoints + +`/status/metrics` + +The endpoint exposes various metrics about the microservice's internal operations. These metrics are formatted in a way that is suitable for consumption +by monitoring and alerting systems. + +HTTP Method: **GET** + +Response: Metrics are presented in JSON format for easy integration with monitoring and alerting systems. + +`/status/prometheus-metrics` + +The /prometheus-metrics endpoint provides Prometheus-compatible metrics in a specific format for easy integration with +Prometheus monitoring systems. + +HTTP Method: **GET** + +Response: Metrics are formatted in a way that Prometheus can scrape and ingest for monitoring and alerting purposes. + + + +### Prerequisites +Before proceeding, ensure you have the following prerequisites: +- Go programming environment set up. +- Access to an Elasticsearch database instance. +- One has to setup one or multiple observers. For running an observing squad, these [docs](https://docs.multiversx.com/integrators/observing-squad/) cover the whole process. +The required configs for launching an observer/s with a driver attached, can be found here. + +The corresponding config section for enabling the driver: + +```toml +[[HostDriversConfig]] + # This flag shall only be used for observer nodes + Enabled = true + # This flag will start the WebSocket connector as server or client (can be "client" or "server") + Mode = "client" + # URL for the WebSocket client/server connection + # This value represents the IP address and port number that the WebSocket client or server will use to establish a connection. + URL = "127.0.0.1:22111" + # After a message will be sent it will wait for an ack message if this flag is enabled + WithAcknowledge = true + # The duration in seconds to wait for an acknowledgment message, after this time passes an error will be returned + AcknowledgeTimeoutInSec = 60 + # Possible values: json, gogo protobuf. Should be compatible with mx-chain-es-indexer-go config + MarshallerType = "json" + # The number of seconds when the client will try again to send the data + RetryDurationInSec = 5 + # Sets if, in case of data payload processing error, we should block or not the advancement to the next processing event. Set this to true if you wish the node to stop processing blocks if the client/server encounters errors while processing requests. + BlockingAckOnError = true + # Set to true to drop messages if there is no active WebSocket connection to send to. + DropMessagesIfNoConnection = false +``` + + +#### Install +Using the cmd/elasticindexer package as root, execute the following commands: +- install go dependencies: `go install` +- build executable: `go build -o elasticindexer` + +#### Launching the elasticindexer + +CLI: run `--help` to get the command line parameters +``` +./elasticindexer --help +``` + +Before launching the `elasticindexer` service, it has to be configured so that it runs with the correct configuration. + +The `prefs.toml` file: + +```toml +[config] + disabled-indices = [] + [config.web-socket] + # URL for the WebSocket client/server connection + # This value represents the IP address and port number that the WebSocket client or server will use to establish a connection. + url = "localhost:22111" + # This flag describes the mode to start the WebSocket connector. Can be "client" or "server" + mode = "server" + # Possible values: json, gogo protobuf. Should be compatible with mx-chain-node outport driver config + data-marshaller-type = "json" + # Retry duration (receive/send ack signal) in seconds + retry-duration-in-seconds = 5 + # Signals if in case of data payload processing error, we should send the ack signal or not + blocking-ack-on-error = true + # After a message will be sent it will wait for an ack message if this flag is enabled + with-acknowledge = true + # The duration in seconds to wait for an acknowledgment message, after this time passes an error will be returned + acknowledge-timeout-in-seconds = 50 + + [config.elastic-cluster] + use-kibana = false + url = "http://localhost:9200" + username = "" + password = "" + bulk-request-max-size-in-bytes = 4194304 # 4MB +``` + +The 'api.toml' file: +```toml +rest-api-interface = ":8080" + +[api-packages] + +[api-packages.status] + routes = [ + { name = "/metrics", open = true }, + { name = "/prometheus-metrics", open = true } + ] +``` + +After the configuration file is set up, the `elasticindexer` instance can be launched. ### Contribution -Contributions to the mx-chain-es-indexer-go module are welcomed. Whether you're interested in improving its features, + +Contributions to the `mx-chain-es-indexer-go` module are welcomed. Whether you're interested in improving its features, extending its capabilities, or addressing issues, your contributions can help the community make the module even more robust. From 68095f5d86e3b8bf050483cf1b2ba3f0391e88db Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 17 Aug 2023 12:43:41 +0300 Subject: [PATCH 163/189] small fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2e76c30f..ad48f3a6 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ The `prefs.toml` file: bulk-request-max-size-in-bytes = 4194304 # 4MB ``` -The 'api.toml' file: +The `api.toml` file: ```toml rest-api-interface = ":8080" From a1c1cd58e760fa800bc938c1c591b5430aedf52f Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 17 Aug 2023 12:44:09 +0300 Subject: [PATCH 164/189] fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ad48f3a6..4e5b76e3 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ The corresponding config section for enabling the driver: #### Install -Using the cmd/elasticindexer package as root, execute the following commands: +Using the `cmd/elasticindexer` package as root, execute the following commands: - install go dependencies: `go install` - build executable: `go build -o elasticindexer` From 84617673e47c37b2ca274ffad8cd5d3f39f1c94a Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 17 Aug 2023 13:18:26 +0300 Subject: [PATCH 165/189] links --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4e5b76e3..5ebb4c5e 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ CLI: run `--help` to get the command line parameters Before launching the `elasticindexer` service, it has to be configured so that it runs with the correct configuration. -The `prefs.toml` file: +The **_[prefs.toml](./cmd/elasticindexer/config/prefs.toml)_** file: ```toml [config] @@ -132,7 +132,7 @@ The `prefs.toml` file: bulk-request-max-size-in-bytes = 4194304 # 4MB ``` -The `api.toml` file: +The _**[api.toml](./cmd/elasticindexer/config/api.toml)**_ file: ```toml rest-api-interface = ":8080" From 25c20447f727a37c135f94cc1fbac8f2742a1405 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 17 Aug 2023 13:39:03 +0300 Subject: [PATCH 166/189] fixes after first review --- README.md | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 5ebb4c5e..7b7bfddb 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,21 @@ ## mx-chain-es-indexer-go -Is a versatile component designed to enhance the data management capabilities of the mx-chain-go repository. -This module can be seamlessly integrated as a Go module or deployed as a separate microservice that communicates with [mx-chain-go](https://github.com/multiversx/mx-chain-go) -over a `WebSocket` connection. -Its core functionality revolves around the efficient interaction with an `Elasticsearch` database. +Is a versatile component designed to enhance the data management capabilities of the `mx-chain-go` repository. ### Overview -Modern applications require sophisticated data handling mechanisms to ensure optimal performance, data integrity, and real-time accessibility. -`mx-chain-es-indexer-go` addresses these requirements by facilitating data transformation, indexing, and storage within an Elasticsearch database. This module's flexibility enables it to be used in two distinct modes: as a Go module or as a separate microservice. ### Features - **Data Transformation**: `mx-chain-es-indexer-go` excels at transforming raw data generated by the `mx-chain-go` repository into a structured format suitable for indexing and storage. -- **Indexing**: The module efficiently indexes the transformed data within Elasticsearch, enhancing query performance and data retrieval. +- **Indexing**: The module indexes the transformed data within Elasticsearch, enhancing query performance and data retrieval. - **Microservice Mode**: In microservice mode, mx-chain-es-indexer-go operates as a standalone microservice that communicates with mx-chain-go via WebSocket connections. This architecture promotes modularity and scalability. -- **Go Module Mode**: In Go module mode, the module can be directly integrated into the mx-chain-go repository, offering seamless data management capabilities without the need for a separate microservice. +- **Go Module Mode**: The `mx-chain-go` repository can include the module directly. This makes it easy to manage data without requiring a separate microservice. ### Running as a Separate Microservice @@ -37,7 +32,7 @@ These endpoints are essential for maintaining the stability and efficiency of th `/status/metrics` -The endpoint exposes various metrics about the microservice's internal operations. These metrics are formatted in a way that is suitable for consumption +This endpoint exposes various metrics about the microservice's internal operations. These metrics are formatted in a way that is suitable for consumption by monitoring and alerting systems. HTTP Method: **GET** @@ -46,7 +41,7 @@ Response: Metrics are presented in JSON format for easy integration with monitor `/status/prometheus-metrics` -The /prometheus-metrics endpoint provides Prometheus-compatible metrics in a specific format for easy integration with +This endpoint provides Prometheus-compatible metrics in a specific format for easy integration with Prometheus monitoring systems. HTTP Method: **GET** From 6d196b12bd1e4fe927ba449599e281b70704ddf9 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 17 Aug 2023 14:11:10 +0300 Subject: [PATCH 167/189] docker file --- Dockerfile | 26 ++++++++++++++++++++++++++ Makefile | 10 ++++++++++ 2 files changed, 36 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..8251fbe7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM golang:1.17.6 as builder + +RUN apt-get update && apt-get install -y + +WORKDIR /multiversx +COPY . . + +WORKDIR /multiversx/cmd/elasticindexer + +RUN go build -o elasticindexer + +# ===== SECOND STAGE ====== +FROM ubuntu:22.04 +RUN apt-get update && apt-get install -y + +RUN useradd -m -u 1000 appuser +USER appuser + +COPY --from=builder /multiversx/cmd/elasticindexer /multiversx + +EXPOSE 22111 + +WORKDIR /multiversx + +ENTRYPOINT ["./elasticindexer"] +CMD ["--log-level", "*:DEBUG"] diff --git a/Makefile b/Makefile index 1dd35db8..a000f850 100644 --- a/Makefile +++ b/Makefile @@ -32,3 +32,13 @@ integration-tests-open-search: go test -v ./integrationtests -tags integrationtests cd scripts && /bin/bash script.sh delete cd scripts && /bin/bash script.sh stop_open_search + +INDEXER_IMAGE_NAME="elasticindexer" +INDEXER_IMAGE_TAG="latest" +DOCKER_FILE=Dockerfile + +docker-build: + docker build \ + -t ${INDEXER_IMAGE_NAME}:${INDEXER_IMAGE_TAG} \ + -f ${DOCKER_FILE} \ + . From 71c696fec35932fa551cdcdc87803902037e499e Mon Sep 17 00:00:00 2001 From: Iulian Pascalau Date: Thu, 17 Aug 2023 15:31:33 +0300 Subject: [PATCH 168/189] - proper releases --- go.mod | 8 ++++---- go.sum | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 34faa2e6..0965383a 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,10 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 - github.com/multiversx/mx-chain-communication-go v1.0.6-0.20230816102554-567e1f606ee9 - github.com/multiversx/mx-chain-core-go v1.2.13-0.20230816090241-133da66033a9 - github.com/multiversx/mx-chain-logger-go v1.0.13-0.20230816094710-e4e216ffef4e - github.com/multiversx/mx-chain-vm-common-go v1.5.1-0.20230816095353-9830338a04e4 + github.com/multiversx/mx-chain-communication-go v1.0.6 + github.com/multiversx/mx-chain-core-go v1.2.13 + github.com/multiversx/mx-chain-logger-go v1.0.13 + github.com/multiversx/mx-chain-vm-common-go v1.5.2 github.com/prometheus/client_model v0.4.0 github.com/prometheus/common v0.37.0 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index 0cc7e50e..5a77910b 100644 --- a/go.sum +++ b/go.sum @@ -247,15 +247,15 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiversx/mx-chain-communication-go v1.0.6-0.20230816102554-567e1f606ee9 h1:5F8Vy5OdcTJDAqdjQ843PHZOlxjZ4mBFA42A7Pd3ssI= -github.com/multiversx/mx-chain-communication-go v1.0.6-0.20230816102554-567e1f606ee9/go.mod h1:tE+3vTDUuhL/sso5Zpx81jmP0m2W92gczAA4sJJCZgw= -github.com/multiversx/mx-chain-core-go v1.2.13-0.20230816090241-133da66033a9 h1:dH8qhDjdURgKSJggqaXyZcQd4bL3S2LW9mjjMivzoUw= -github.com/multiversx/mx-chain-core-go v1.2.13-0.20230816090241-133da66033a9/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= -github.com/multiversx/mx-chain-crypto-go v1.2.8-0.20230816100617-0f47cf084591 h1:fb7qi89NqJ3tdTVis+uNKUg+XJ4BnaQLv7NzJJyiMrw= -github.com/multiversx/mx-chain-logger-go v1.0.13-0.20230816094710-e4e216ffef4e h1:yHTUbBPdd7PSWU2ckgH00r+o0PYUu/Xzq6mGmOsJ0aQ= -github.com/multiversx/mx-chain-logger-go v1.0.13-0.20230816094710-e4e216ffef4e/go.mod h1:dE16PsJFsDubcKtQDV6C5Qk3DeMKWqoX5MhM9oqZlmI= -github.com/multiversx/mx-chain-vm-common-go v1.5.1-0.20230816095353-9830338a04e4 h1:b061kswdx2aImnRfBlN8zMVwkvtGEcNgMgyewEepGyM= -github.com/multiversx/mx-chain-vm-common-go v1.5.1-0.20230816095353-9830338a04e4/go.mod h1:VxN7ZFxFJocIRXso/60MYIKpCF0phQt3NQihpvsJjC0= +github.com/multiversx/mx-chain-communication-go v1.0.6 h1:f2bizRoVuJXBWc32px7pCuzMx4Pgi2tKhUt8BkFV1Fg= +github.com/multiversx/mx-chain-communication-go v1.0.6/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= +github.com/multiversx/mx-chain-core-go v1.2.13 h1:4Svi23hdsoibStFXv0i7lbBWus3kDJPc6CFhrxrKIZ4= +github.com/multiversx/mx-chain-core-go v1.2.13/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= +github.com/multiversx/mx-chain-crypto-go v1.2.8 h1:wOgVlUaO5X4L8iEbFjcQcL8SZvv6WZ7LqH73BiRPhxU= +github.com/multiversx/mx-chain-logger-go v1.0.13 h1:eru/TETo0MkO4ZTnXsQDKf4PBRpAXmqjT02klNT/JnY= +github.com/multiversx/mx-chain-logger-go v1.0.13/go.mod h1:MZJhTAtZTJxT+yK2EHc4ZW3YOHUc1UdjCD0iahRNBZk= +github.com/multiversx/mx-chain-vm-common-go v1.5.2 h1:iRWJNlogjkq9w+pJZIfkVkXQFmMoRxZr6pzCfg2/K68= +github.com/multiversx/mx-chain-vm-common-go v1.5.2/go.mod h1:sqkKMCnwkWl8DURdb9q7pctK8IANghdHY1KJLE0ox2c= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= From f3ae0f190af59bb8995d14c49ec4fedf3883bf8f Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 18 Aug 2023 11:45:59 +0300 Subject: [PATCH 169/189] fixes after review --- Dockerfile | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8251fbe7..d4436ca2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.6 as builder +FROM golang:1.20.5 as builder RUN apt-get update && apt-get install -y diff --git a/README.md b/README.md index 7b7bfddb..68dfade9 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Before proceeding, ensure you have the following prerequisites: - Go programming environment set up. - Access to an Elasticsearch database instance. - One has to setup one or multiple observers. For running an observing squad, these [docs](https://docs.multiversx.com/integrators/observing-squad/) cover the whole process. -The required configs for launching an observer/s with a driver attached, can be found here. +The required configs for launching an observer/s with a driver attached, can be found [here](https://github.com/multiversx/mx-chain-go/blob/master/cmd/node/config/external.toml). The corresponding config section for enabling the driver: From ba37deb1b18eaa09eee241c4321843b84f0e7a85 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 30 Aug 2023 15:12:03 +0300 Subject: [PATCH 170/189] extra fields block structure --- data/block.go | 13 +++++++++++-- process/elasticproc/block/blockProcessor.go | 8 ++++++++ .../elasticproc/block/blockProcessor_test.go | 19 ++++++++++++++++++- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/data/block.go b/data/block.go index b163a623..9c71af27 100644 --- a/data/block.go +++ b/data/block.go @@ -5,8 +5,9 @@ import ( ) // Block is a structure containing all the fields that need -// to be saved for a block. It has all the default fields -// plus some extra information for ease of search and filter +// +// to be saved for a block. It has all the default fields +// plus some extra information for ease of search and filter type Block struct { Nonce uint64 `json:"nonce"` Round uint64 `json:"round"` @@ -37,6 +38,14 @@ type Block struct { MaxGasLimit uint64 `json:"maxGasLimit"` ScheduledData *ScheduledData `json:"scheduledData,omitempty"` EpochStartShardsData []*EpochStartShardData `json:"epochStartShardsData,omitempty"` + RandSeed string `json:"randSeed,omitempty"` + PrevRandSeed string `json:"prevRandSeed,omitempty"` + Signature string `json:"signature,omitempty"` + LeaderSignature string `json:"leaderSignature,omitempty"` + ChainID string `json:"chainID,omitempty"` + SoftwareVersion string `json:"softwareVersion,omitempty"` + ReceiptsHash string `json:"receiptsHash,omitempty"` + Reserved []byte `json:"reserved,omitempty"` } // MiniBlocksDetails is a structure that hold information about mini-blocks execution details diff --git a/process/elasticproc/block/blockProcessor.go b/process/elasticproc/block/blockProcessor.go index 26b60dd0..c23e3e1b 100644 --- a/process/elasticproc/block/blockProcessor.go +++ b/process/elasticproc/block/blockProcessor.go @@ -103,6 +103,14 @@ func (bp *blockProcessor) PrepareBlockForDB(obh *outport.OutportBlockWithHeader) MaxGasLimit: obh.HeaderGasConsumption.MaxGasPerBlock, AccumulatedFees: converters.BigIntToString(obh.Header.GetAccumulatedFees()), DeveloperFees: converters.BigIntToString(obh.Header.GetDeveloperFees()), + RandSeed: hex.EncodeToString(obh.Header.GetRandSeed()), + PrevRandSeed: hex.EncodeToString(obh.Header.GetPrevRandSeed()), + Signature: hex.EncodeToString(obh.Header.GetSignature()), + LeaderSignature: hex.EncodeToString(obh.Header.GetLeaderSignature()), + ChainID: string(obh.Header.GetChainID()), + SoftwareVersion: hex.EncodeToString(obh.Header.GetSoftwareVersion()), + ReceiptsHash: hex.EncodeToString(obh.Header.GetReceiptsHash()), + Reserved: obh.Header.GetReserved(), } additionalData := obh.Header.GetAdditionalData() diff --git a/process/elasticproc/block/blockProcessor_test.go b/process/elasticproc/block/blockProcessor_test.go index b162aaf4..d727fc1f 100644 --- a/process/elasticproc/block/blockProcessor_test.go +++ b/process/elasticproc/block/blockProcessor_test.go @@ -63,7 +63,16 @@ func TestBlockProcessor_PrepareBlockForDBShouldWork(t *testing.T) { bp, _ := NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) outportBlockWithHeader := &outport.OutportBlockWithHeader{ - Header: &dataBlock.Header{}, + Header: &dataBlock.Header{ + RandSeed: []byte("randSeed"), + PrevRandSeed: []byte("prevRandSeed"), + Signature: []byte("signature"), + LeaderSignature: []byte("leaderSignature"), + ChainID: []byte("1"), + SoftwareVersion: []byte("1"), + ReceiptsHash: []byte("hash"), + Reserved: []byte("reserved"), + }, OutportBlock: &outport.OutportBlock{ BlockData: &outport.BlockData{ HeaderHash: []byte("hash"), @@ -96,6 +105,14 @@ func TestBlockProcessor_PrepareBlockForDBShouldWork(t *testing.T) { Size: 114, AccumulatedFees: "0", DeveloperFees: "0", + RandSeed: "72616e6453656564", + PrevRandSeed: "7072657652616e6453656564", + Signature: "7369676e6174757265", + LeaderSignature: "6c65616465725369676e6174757265", + ChainID: "1", + SoftwareVersion: "31", + ReceiptsHash: "68617368", + Reserved: []byte("reserved"), } require.Equal(t, expectedBlock, dbBlock) } From 74aa83edb43fcdd2a08bc6bd1b4a45af26adc26a Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 30 Aug 2023 15:22:00 +0300 Subject: [PATCH 171/189] fix linter issue --- api/gin/webServer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/gin/webServer.go b/api/gin/webServer.go index 656ad5f4..048442e6 100644 --- a/api/gin/webServer.go +++ b/api/gin/webServer.go @@ -7,6 +7,7 @@ import ( "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" + "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-es-indexer-go/api/groups" "github.com/multiversx/mx-chain-es-indexer-go/api/shared" "github.com/multiversx/mx-chain-es-indexer-go/config" @@ -107,7 +108,7 @@ func (ws *webServer) registerRoutes(ginRouter *gin.Engine) { func (ws *webServer) Close() error { var err error ws.Lock() - if ws.httpServer != nil { + if !check.IfNil(ws.httpServer) { err = ws.httpServer.Close() } ws.Unlock() From eb3f39b772330cc973e7f6d9393e5e97a5292cdd Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 30 Aug 2023 15:26:18 +0300 Subject: [PATCH 172/189] update golangci-lint.yml --- .github/workflows/golangci-lint.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 3c1d0d1b..d0e800f7 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -15,13 +15,14 @@ jobs: steps: - uses: actions/setup-go@v3 with: - go-version: 1.17.6 + go-version: 1.20 + cache: false - uses: actions/checkout@v3 - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.45.2 + version: v1.54 # Optional: working directory, useful for monorepos # working-directory: somedir From f7a8b2310ba8000d7f29ad793fd45b5a721834cc Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 30 Aug 2023 15:29:39 +0300 Subject: [PATCH 173/189] fixes --- .github/workflows/golangci-lint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index d0e800f7..c5142d4c 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -15,14 +15,14 @@ jobs: steps: - uses: actions/setup-go@v3 with: - go-version: 1.20 + go-version: 1.20.5 cache: false - uses: actions/checkout@v3 - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.54 + version: v1.54.2 # Optional: working directory, useful for monorepos # working-directory: somedir From bf42840b1fd9f5bed12e2c5bbd66935b9b968dab Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 30 Aug 2023 16:20:55 +0300 Subject: [PATCH 174/189] extra field accounts indexer --- data/account.go | 47 ++++++++++--------- go.mod | 2 +- go.sum | 4 +- .../elasticproc/accounts/accountsProcessor.go | 21 +++++---- .../accounts/accountsProcessor_test.go | 22 +++++---- .../elasticproc/accounts/serialize_test.go | 16 +++---- 6 files changed, 59 insertions(+), 53 deletions(-) diff --git a/data/account.go b/data/account.go index 8e370b13..f28ff087 100644 --- a/data/account.go +++ b/data/account.go @@ -8,29 +8,30 @@ import ( // AccountInfo holds (serializable) data about an account type AccountInfo struct { - Address string `json:"address,omitempty"` - Nonce uint64 `json:"nonce,omitempty"` - Balance string `json:"balance"` - BalanceNum float64 `json:"balanceNum"` - TokenName string `json:"token,omitempty"` - TokenIdentifier string `json:"identifier,omitempty"` - TokenNonce uint64 `json:"tokenNonce,omitempty"` - Properties string `json:"properties,omitempty"` - Frozen bool `json:"frozen,omitempty"` - TotalBalanceWithStake string `json:"totalBalanceWithStake,omitempty"` - TotalBalanceWithStakeNum float64 `json:"totalBalanceWithStakeNum,omitempty"` - Owner string `json:"owner,omitempty"` - UserName string `json:"userName,omitempty"` - DeveloperRewards string `json:"developerRewards,omitempty"` - DeveloperRewardsNum float64 `json:"developerRewardsNum,omitempty"` - Data *TokenMetaData `json:"data,omitempty"` - Timestamp time.Duration `json:"timestamp,omitempty"` - Type string `json:"type,omitempty"` - CurrentOwner string `json:"currentOwner,omitempty"` - ShardID uint32 `json:"shardID"` - IsSender bool `json:"-"` - IsSmartContract bool `json:"-"` - IsNFTCreate bool `json:"-"` + Address string `json:"address,omitempty"` + Nonce uint64 `json:"nonce,omitempty"` + Balance string `json:"balance"` + BalanceNum float64 `json:"balanceNum"` + TokenName string `json:"token,omitempty"` + TokenIdentifier string `json:"identifier,omitempty"` + TokenNonce uint64 `json:"tokenNonce,omitempty"` + Properties string `json:"properties,omitempty"` + Frozen bool `json:"frozen,omitempty"` + Owner string `json:"owner,omitempty"` + UserName string `json:"userName,omitempty"` + DeveloperRewards string `json:"developerRewards,omitempty"` + DeveloperRewardsNum float64 `json:"developerRewardsNum,omitempty"` + Data *TokenMetaData `json:"data,omitempty"` + Timestamp time.Duration `json:"timestamp,omitempty"` + Type string `json:"type,omitempty"` + CurrentOwner string `json:"currentOwner,omitempty"` + ShardID uint32 `json:"shardID"` + RootHash []byte `json:"rootHash,omitempty"` + CodeHash []byte `json:"codeHash,omitempty"` + CodeMetadata []byte `json:"codeMetadata,omitempty"` + IsSender bool `json:"-"` + IsSmartContract bool `json:"-"` + IsNFTCreate bool `json:"-"` } // TokenMetaData holds data about a token metadata diff --git a/go.mod b/go.mod index 0965383a..7301d680 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 github.com/multiversx/mx-chain-communication-go v1.0.6 - github.com/multiversx/mx-chain-core-go v1.2.13 + github.com/multiversx/mx-chain-core-go v1.2.15-0.20230830125719-028435cb3956 github.com/multiversx/mx-chain-logger-go v1.0.13 github.com/multiversx/mx-chain-vm-common-go v1.5.2 github.com/prometheus/client_model v0.4.0 diff --git a/go.sum b/go.sum index 5a77910b..2e536b74 100644 --- a/go.sum +++ b/go.sum @@ -249,8 +249,8 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-communication-go v1.0.6 h1:f2bizRoVuJXBWc32px7pCuzMx4Pgi2tKhUt8BkFV1Fg= github.com/multiversx/mx-chain-communication-go v1.0.6/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= -github.com/multiversx/mx-chain-core-go v1.2.13 h1:4Svi23hdsoibStFXv0i7lbBWus3kDJPc6CFhrxrKIZ4= -github.com/multiversx/mx-chain-core-go v1.2.13/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= +github.com/multiversx/mx-chain-core-go v1.2.15-0.20230830125719-028435cb3956 h1:RlIWOnzWd5zsv7SDcNMiOkvuHiXEFHcdbzvaDfpM5UU= +github.com/multiversx/mx-chain-core-go v1.2.15-0.20230830125719-028435cb3956/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= github.com/multiversx/mx-chain-crypto-go v1.2.8 h1:wOgVlUaO5X4L8iEbFjcQcL8SZvv6WZ7LqH73BiRPhxU= github.com/multiversx/mx-chain-logger-go v1.0.13 h1:eru/TETo0MkO4ZTnXsQDKf4PBRpAXmqjT02klNT/JnY= github.com/multiversx/mx-chain-logger-go v1.0.13/go.mod h1:MZJhTAtZTJxT+yK2EHc4ZW3YOHUc1UdjCD0iahRNBZk= diff --git a/process/elasticproc/accounts/accountsProcessor.go b/process/elasticproc/accounts/accountsProcessor.go index 4d90b02d..1c24bfea 100644 --- a/process/elasticproc/accounts/accountsProcessor.go +++ b/process/elasticproc/accounts/accountsProcessor.go @@ -123,16 +123,14 @@ func (ap *accountsProcessor) PrepareRegularAccountsMap(timestamp uint64, account } acc := &data.AccountInfo{ - Address: address, - Nonce: userAccount.UserAccount.Nonce, - Balance: converters.BigIntToString(balance), - BalanceNum: balanceAsFloat, - IsSender: userAccount.IsSender, - IsSmartContract: core.IsSmartContractAddress(addressBytes), - TotalBalanceWithStake: converters.BigIntToString(balance), - TotalBalanceWithStakeNum: balanceAsFloat, - Timestamp: time.Duration(timestamp), - ShardID: shardID, + Address: address, + Nonce: userAccount.UserAccount.Nonce, + Balance: converters.BigIntToString(balance), + BalanceNum: balanceAsFloat, + IsSender: userAccount.IsSender, + IsSmartContract: core.IsSmartContractAddress(addressBytes), + Timestamp: time.Duration(timestamp), + ShardID: shardID, } ap.addAdditionalDataInAccount(userAccount.UserAccount.AdditionalData, acc) @@ -150,6 +148,9 @@ func (ap *accountsProcessor) addAdditionalDataInAccount(additionalData *alteredA account.UserName = additionalData.UserName account.CurrentOwner = additionalData.CurrentOwner + account.RootHash = additionalData.RootHash + account.CodeHash = additionalData.CodeHash + account.CodeMetadata = additionalData.CodeMetadata ap.addDeveloperRewardsInAccount(additionalData, account) } diff --git a/process/elasticproc/accounts/accountsProcessor_test.go b/process/elasticproc/accounts/accountsProcessor_test.go index 71b1657b..b9c8b7bd 100644 --- a/process/elasticproc/accounts/accountsProcessor_test.go +++ b/process/elasticproc/accounts/accountsProcessor_test.go @@ -268,6 +268,11 @@ func TestAccountsProcessor_PrepareAccountsMapEGLD(t *testing.T) { Address: addr, Balance: "1000", Nonce: 1, + AdditionalData: &alteredAccount.AdditionalAccountData{ + CodeHash: []byte("code"), + CodeMetadata: []byte("metadata"), + RootHash: []byte("root"), + }, } egldAccount := &data.Account{ @@ -282,14 +287,15 @@ func TestAccountsProcessor_PrepareAccountsMapEGLD(t *testing.T) { res := ap.PrepareRegularAccountsMap(123, []*data.Account{egldAccount}, 0) require.Equal(t, &data.AccountInfo{ - Address: addr, - Nonce: 1, - Balance: "1000", - BalanceNum: balanceNum, - TotalBalanceWithStake: "1000", - TotalBalanceWithStakeNum: balanceNum, - IsSmartContract: true, - Timestamp: time.Duration(123), + Address: addr, + Nonce: 1, + Balance: "1000", + BalanceNum: balanceNum, + IsSmartContract: true, + Timestamp: time.Duration(123), + CodeHash: []byte("code"), + CodeMetadata: []byte("metadata"), + RootHash: []byte("root"), }, res[addr]) } diff --git a/process/elasticproc/accounts/serialize_test.go b/process/elasticproc/accounts/serialize_test.go index 3a1eb965..52b0dbf4 100644 --- a/process/elasticproc/accounts/serialize_test.go +++ b/process/elasticproc/accounts/serialize_test.go @@ -39,14 +39,12 @@ func TestSerializeAccounts(t *testing.T) { accs := map[string]*data.AccountInfo{ "addr1": { - Address: "addr1", - Nonce: 1, - Balance: "50", - BalanceNum: 0.1, - TotalBalanceWithStake: "50", - TotalBalanceWithStakeNum: 0.1, - IsSmartContract: true, - IsSender: true, + Address: "addr1", + Nonce: 1, + Balance: "50", + BalanceNum: 0.1, + IsSmartContract: true, + IsSender: true, }, } @@ -56,7 +54,7 @@ func TestSerializeAccounts(t *testing.T) { require.Equal(t, 1, len(buffSlice.Buffers())) expectedRes := `{ "update" : {"_index": "accounts", "_id" : "addr1" } } -{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.account} else {if (ctx._source.containsKey('timestamp')) {if (ctx._source.timestamp <= params.account.timestamp) {ctx._source = params.account}} else {ctx._source = params.account}}","lang": "painless","params": { "account": {"address":"addr1","nonce":1,"balance":"50","balanceNum":0.1,"totalBalanceWithStake":"50","totalBalanceWithStakeNum":0.1,"shardID":0} }},"upsert": {}} +{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx._source = params.account} else {if (ctx._source.containsKey('timestamp')) {if (ctx._source.timestamp <= params.account.timestamp) {ctx._source = params.account}} else {ctx._source = params.account}}","lang": "painless","params": { "account": {"address":"addr1","nonce":1,"balance":"50","balanceNum":0.1,"shardID":0} }},"upsert": {}} ` require.Equal(t, expectedRes, buffSlice.Buffers()[0].String()) } From f13b0ab4f67a249cc6012086d65059d33a1140be Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 30 Aug 2023 16:25:50 +0300 Subject: [PATCH 175/189] fix integration tests --- .../account-balance-first-update.json | 1 - .../account-balance-second-update.json | 1 - 2 files changed, 2 deletions(-) diff --git a/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-first-update.json b/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-first-update.json index 43990f8e..97a80077 100644 --- a/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-first-update.json +++ b/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-first-update.json @@ -2,7 +2,6 @@ "address": "erd17umc0uvel62ng30k5uprqcxh3ue33hq608njejaqljuqzqlxtzuqeuzlcv", "balance": "0", "balanceNum": 0, - "totalBalanceWithStake": "0", "timestamp": 5600, "shardID": 2 } diff --git a/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-second-update.json b/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-second-update.json index 06d0c7d4..3ebc00ce 100644 --- a/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-second-update.json +++ b/integrationtests/testdata/accountsBalanceWithLowerTimestamp/account-balance-second-update.json @@ -3,6 +3,5 @@ "balance": "2000", "balanceNum": 0, "timestamp": 6000, - "totalBalanceWithStake": "2000", "shardID": 2 } From cfe0d548b1eb93d63dac40c778f8dddcd31af83a Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 6 Sep 2023 10:50:31 +0300 Subject: [PATCH 176/189] proper tag --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7301d680..bf04516e 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 github.com/multiversx/mx-chain-communication-go v1.0.6 - github.com/multiversx/mx-chain-core-go v1.2.15-0.20230830125719-028435cb3956 + github.com/multiversx/mx-chain-core-go v1.2.15 github.com/multiversx/mx-chain-logger-go v1.0.13 github.com/multiversx/mx-chain-vm-common-go v1.5.2 github.com/prometheus/client_model v0.4.0 diff --git a/go.sum b/go.sum index 2e536b74..33566c33 100644 --- a/go.sum +++ b/go.sum @@ -249,8 +249,8 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-communication-go v1.0.6 h1:f2bizRoVuJXBWc32px7pCuzMx4Pgi2tKhUt8BkFV1Fg= github.com/multiversx/mx-chain-communication-go v1.0.6/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= -github.com/multiversx/mx-chain-core-go v1.2.15-0.20230830125719-028435cb3956 h1:RlIWOnzWd5zsv7SDcNMiOkvuHiXEFHcdbzvaDfpM5UU= -github.com/multiversx/mx-chain-core-go v1.2.15-0.20230830125719-028435cb3956/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= +github.com/multiversx/mx-chain-core-go v1.2.15 h1:2qbcGP9yHi9CFeLF9xTDnDPJjvafvTmwEkitfI0wWME= +github.com/multiversx/mx-chain-core-go v1.2.15/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= github.com/multiversx/mx-chain-crypto-go v1.2.8 h1:wOgVlUaO5X4L8iEbFjcQcL8SZvv6WZ7LqH73BiRPhxU= github.com/multiversx/mx-chain-logger-go v1.0.13 h1:eru/TETo0MkO4ZTnXsQDKf4PBRpAXmqjT02klNT/JnY= github.com/multiversx/mx-chain-logger-go v1.0.13/go.mod h1:MZJhTAtZTJxT+yK2EHc4ZW3YOHUc1UdjCD0iahRNBZk= From b348fc9baeed1aabbaaef11ab017d06adcef6153 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 13 Sep 2023 11:54:33 +0300 Subject: [PATCH 177/189] extra checks tags index --- process/elasticproc/converters/tags.go | 3 +++ process/elasticproc/tags/serialize.go | 4 ++++ process/elasticproc/tags/serialize_test.go | 24 ++++++++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/process/elasticproc/converters/tags.go b/process/elasticproc/converters/tags.go index 1bdb119b..3af2baa0 100644 --- a/process/elasticproc/converters/tags.go +++ b/process/elasticproc/converters/tags.go @@ -5,6 +5,9 @@ import ( ) const ( + // MaxIDSize is the maximum size of a document id + MaxIDSize = 512 + attributesSeparator = ";" keyValuesSeparator = ":" valuesSeparator = "," diff --git a/process/elasticproc/tags/serialize.go b/process/elasticproc/tags/serialize.go index be13563f..1de2f129 100644 --- a/process/elasticproc/tags/serialize.go +++ b/process/elasticproc/tags/serialize.go @@ -16,12 +16,16 @@ func (tc *tagsCount) Serialize(buffSlice *data.BufferSlice, index string) error } base64Tag := base64.StdEncoding.EncodeToString([]byte(tag)) + if len(base64Tag) > converters.MaxIDSize { + base64Tag = base64Tag[:converters.MaxIDSize] + } meta := []byte(fmt.Sprintf(`{ "update" : {"_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(base64Tag), "\n")) codeToExecute := ` ctx._source.count += params.count; ctx._source.tag = params.tag ` + serializedDataStr := fmt.Sprintf(`{"script": {"source": "%s","lang": "painless","params": {"count": %d, "tag": "%s"}},"upsert": {"count": %d, "tag":"%s"}}`, converters.FormatPainlessSource(codeToExecute), count, converters.JsonEscape(tag), count, converters.JsonEscape(tag), ) diff --git a/process/elasticproc/tags/serialize_test.go b/process/elasticproc/tags/serialize_test.go index 80e69d7d..ffeeadf5 100644 --- a/process/elasticproc/tags/serialize_test.go +++ b/process/elasticproc/tags/serialize_test.go @@ -1,9 +1,13 @@ package tags import ( + "crypto/rand" + "encoding/base64" + "fmt" "testing" "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" "github.com/stretchr/testify/require" ) @@ -24,3 +28,23 @@ func TestTagsCount_Serialize(t *testing.T) { ` require.Equal(t, expected, buffSlice.Buffers()[0].String()) } + +func TestTagsCount_TruncateID(t *testing.T) { + t.Parallel() + + tagsC := NewTagsCount() + + randomBytes := make([]byte, 600) + _, _ = rand.Read(randomBytes) + + tagsC.ParseTags([]string{string(randomBytes)}) + + buffSlice := data.NewBufferSlice(data.DefaultMaxBulkSize) + err := tagsC.Serialize(buffSlice, "tags") + require.Nil(t, err) + + expected := fmt.Sprintf(`{ "update" : {"_index":"tags", "_id" : "%s" } } +{"script": {"source": "ctx._source.count += params.count; ctx._source.tag = params.tag","lang": "painless","params": {"count": 1, "tag": "%s"}},"upsert": {"count": 1, "tag":"%s"}} +`, base64.StdEncoding.EncodeToString(randomBytes)[:converters.MaxIDSize], converters.JsonEscape(string(randomBytes)), converters.JsonEscape(string(randomBytes))) + require.Equal(t, expected, buffSlice.Buffers()[0].String()) +} From cfc78b4f3cfe8cb1f3a2a47520376a4e07f02ef7 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 14 Sep 2023 13:23:14 +0300 Subject: [PATCH 178/189] skip indexing --- process/elasticproc/miniblocks/miniblocksProcessor.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/process/elasticproc/miniblocks/miniblocksProcessor.go b/process/elasticproc/miniblocks/miniblocksProcessor.go index db8405e8..03fed657 100644 --- a/process/elasticproc/miniblocks/miniblocksProcessor.go +++ b/process/elasticproc/miniblocks/miniblocksProcessor.go @@ -48,8 +48,15 @@ func (mp *miniblocksProcessor) PrepareDBMiniblocks(header coreData.HeaderHandler return nil } + selfShard := header.GetShardID() dbMiniblocks := make([]*data.Miniblock, 0) for mbIndex, miniBlock := range miniBlocks { + + if miniBlock.ReceiverShardID == core.AllShardId && selfShard != core.MetachainShardId { + // will not index the miniblock on the destination if is for all shards + continue + } + dbMiniBlock, errPrepareMiniBlock := mp.prepareMiniblockForDB(mbIndex, miniBlock, header, headerHash) if errPrepareMiniBlock != nil { log.Warn("miniblocksProcessor.PrepareDBMiniBlocks cannot prepare miniblock", "error", errPrepareMiniBlock) From 102898a65ff5087bcc2c924a7b2416839af66ac8 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 14 Sep 2023 13:23:35 +0300 Subject: [PATCH 179/189] remove empty line --- process/elasticproc/miniblocks/miniblocksProcessor.go | 1 - 1 file changed, 1 deletion(-) diff --git a/process/elasticproc/miniblocks/miniblocksProcessor.go b/process/elasticproc/miniblocks/miniblocksProcessor.go index 03fed657..c4c1c061 100644 --- a/process/elasticproc/miniblocks/miniblocksProcessor.go +++ b/process/elasticproc/miniblocks/miniblocksProcessor.go @@ -51,7 +51,6 @@ func (mp *miniblocksProcessor) PrepareDBMiniblocks(header coreData.HeaderHandler selfShard := header.GetShardID() dbMiniblocks := make([]*data.Miniblock, 0) for mbIndex, miniBlock := range miniBlocks { - if miniBlock.ReceiverShardID == core.AllShardId && selfShard != core.MetachainShardId { // will not index the miniblock on the destination if is for all shards continue From 00b43f6605ecbdbc77075de967cedfe1189a9c8f Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Thu, 21 Sep 2023 11:32:55 +0300 Subject: [PATCH 180/189] latest version of mx-chain-communication-go --- data/logs.go | 11 +++++----- go.mod | 4 ++-- go.sum | 8 +++---- .../logsevents/logsAndEventsProcessor.go | 22 +++++++++---------- .../logsevents/logsAndEventsProcessor_test.go | 14 +++++++----- process/wsindexer/indexer.go | 6 ++++- 6 files changed, 35 insertions(+), 30 deletions(-) diff --git a/data/logs.go b/data/logs.go index 6d7058a3..d27fd466 100644 --- a/data/logs.go +++ b/data/logs.go @@ -18,11 +18,12 @@ type Logs struct { // Event holds all the fields needed for an event structure type Event struct { - Address string `json:"address"` - Identifier string `json:"identifier"` - Topics [][]byte `json:"topics"` - Data []byte `json:"data"` - Order int `json:"order"` + Address string `json:"address"` + Identifier string `json:"identifier"` + Topics [][]byte `json:"topics"` + Data []byte `json:"data"` + AdditionalData [][]byte `json:"additionalData,omitempty"` + Order int `json:"order"` } // PreparedLogsResults is the DTO that holds all the results after processing diff --git a/go.mod b/go.mod index bf04516e..a349014b 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 - github.com/multiversx/mx-chain-communication-go v1.0.6 - github.com/multiversx/mx-chain-core-go v1.2.15 + github.com/multiversx/mx-chain-communication-go v1.0.7-0.20230920140934-77ca04b5a631 + github.com/multiversx/mx-chain-core-go v1.2.16 github.com/multiversx/mx-chain-logger-go v1.0.13 github.com/multiversx/mx-chain-vm-common-go v1.5.2 github.com/prometheus/client_model v0.4.0 diff --git a/go.sum b/go.sum index 33566c33..9b3f29bd 100644 --- a/go.sum +++ b/go.sum @@ -247,10 +247,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiversx/mx-chain-communication-go v1.0.6 h1:f2bizRoVuJXBWc32px7pCuzMx4Pgi2tKhUt8BkFV1Fg= -github.com/multiversx/mx-chain-communication-go v1.0.6/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= -github.com/multiversx/mx-chain-core-go v1.2.15 h1:2qbcGP9yHi9CFeLF9xTDnDPJjvafvTmwEkitfI0wWME= -github.com/multiversx/mx-chain-core-go v1.2.15/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= +github.com/multiversx/mx-chain-communication-go v1.0.7-0.20230920140934-77ca04b5a631 h1:E3k9OgTMWWTm/Aytj4ON3Rbg8Bf/S9ZKtCk5EWXQIC4= +github.com/multiversx/mx-chain-communication-go v1.0.7-0.20230920140934-77ca04b5a631/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= +github.com/multiversx/mx-chain-core-go v1.2.16 h1:m0hUNmZQjGJxKDLQOHoM9jSaeDfVTbyd+mqiS8+NckE= +github.com/multiversx/mx-chain-core-go v1.2.16/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= github.com/multiversx/mx-chain-crypto-go v1.2.8 h1:wOgVlUaO5X4L8iEbFjcQcL8SZvv6WZ7LqH73BiRPhxU= github.com/multiversx/mx-chain-logger-go v1.0.13 h1:eru/TETo0MkO4ZTnXsQDKf4PBRpAXmqjT02klNT/JnY= github.com/multiversx/mx-chain-logger-go v1.0.13/go.mod h1:MZJhTAtZTJxT+yK2EHc4ZW3YOHUc1UdjCD0iahRNBZk= diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor.go b/process/elasticproc/logsevents/logsAndEventsProcessor.go index 04f0cbca..f633de3c 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor.go @@ -203,7 +203,7 @@ func (lep *logsAndEventsProcessor) PrepareLogsForDB( func (lep *logsAndEventsProcessor) prepareLogsForDB( logHashHex string, - logHandler coreData.LogHandler, + eventLogs *transaction.Log, timestamp uint64, ) *data.Logs { originalTxHash := "" @@ -212,19 +212,16 @@ func (lep *logsAndEventsProcessor) prepareLogsForDB( originalTxHash = scr.OriginalTxHash } - events := logHandler.GetLogEvents() - - encodedAddr := lep.pubKeyConverter.SilentEncode(logHandler.GetAddress(), log) - + encodedAddr := lep.pubKeyConverter.SilentEncode(eventLogs.GetAddress(), log) logsDB := &data.Logs{ ID: logHashHex, OriginalTxHash: originalTxHash, Address: encodedAddr, Timestamp: time.Duration(timestamp), - Events: make([]*data.Event, 0, len(events)), + Events: make([]*data.Event, 0, len(eventLogs.Events)), } - for idx, event := range events { + for idx, event := range eventLogs.Events { if check.IfNil(event) { continue } @@ -232,11 +229,12 @@ func (lep *logsAndEventsProcessor) prepareLogsForDB( encodedAddress := lep.pubKeyConverter.SilentEncode(event.GetAddress(), log) logsDB.Events = append(logsDB.Events, &data.Event{ - Address: encodedAddress, - Identifier: string(event.GetIdentifier()), - Topics: event.GetTopics(), - Data: event.GetData(), - Order: idx, + Address: encodedAddress, + Identifier: string(event.GetIdentifier()), + Topics: event.GetTopics(), + Data: event.GetData(), + AdditionalData: event.GetAdditionalData(), + Order: idx, }) } diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go index 68ed2fdc..31e4ea90 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go @@ -216,9 +216,10 @@ func TestLogsAndEventsProcessor_PrepareLogsForDB(t *testing.T) { Address: []byte("address"), Events: []*transaction.Event{ { - Address: []byte("addr"), - Identifier: []byte(core.BuiltInFunctionESDTNFTTransfer), - Topics: [][]byte{[]byte("my-token"), big.NewInt(0).SetUint64(1).Bytes(), []byte("receiver")}, + Address: []byte("addr"), + Identifier: []byte(core.BuiltInFunctionESDTNFTTransfer), + Topics: [][]byte{[]byte("my-token"), big.NewInt(0).SetUint64(1).Bytes(), []byte("receiver")}, + AdditionalData: [][]byte{[]byte("something")}, }, }, }, @@ -243,9 +244,10 @@ func TestLogsAndEventsProcessor_PrepareLogsForDB(t *testing.T) { Timestamp: time.Duration(1234), Events: []*data.Event{ { - Address: "61646472", - Identifier: core.BuiltInFunctionESDTNFTTransfer, - Topics: [][]byte{[]byte("my-token"), big.NewInt(0).SetUint64(1).Bytes(), []byte("receiver")}, + Address: "61646472", + Identifier: core.BuiltInFunctionESDTNFTTransfer, + Topics: [][]byte{[]byte("my-token"), big.NewInt(0).SetUint64(1).Bytes(), []byte("receiver")}, + AdditionalData: [][]byte{[]byte("something")}, }, }, }, logsDB[0]) diff --git a/process/wsindexer/indexer.go b/process/wsindexer/indexer.go index 98a4a2d0..42dcfac5 100644 --- a/process/wsindexer/indexer.go +++ b/process/wsindexer/indexer.go @@ -70,7 +70,11 @@ func (i *indexer) initActionsMap() { } // ProcessPayload will proces the provided payload based on the topic -func (i *indexer) ProcessPayload(payload []byte, topic string) error { +func (i *indexer) ProcessPayload(payload []byte, topic string, version uint32) error { + if version != 1 { + log.Warn("received a payload with a different version", "version", version) + } + payloadTypeAction, ok := i.actions[topic] if !ok { log.Warn("invalid payload type", "topic", topic) From 49011b9c0a4c83d520ac7f79341296f2be267aa2 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Fri, 22 Sep 2023 14:23:30 +0300 Subject: [PATCH 181/189] proper release for communication go --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a349014b..9fbf9d92 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 - github.com/multiversx/mx-chain-communication-go v1.0.7-0.20230920140934-77ca04b5a631 + github.com/multiversx/mx-chain-communication-go v1.0.7 github.com/multiversx/mx-chain-core-go v1.2.16 github.com/multiversx/mx-chain-logger-go v1.0.13 github.com/multiversx/mx-chain-vm-common-go v1.5.2 diff --git a/go.sum b/go.sum index 9b3f29bd..4f913b00 100644 --- a/go.sum +++ b/go.sum @@ -247,8 +247,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiversx/mx-chain-communication-go v1.0.7-0.20230920140934-77ca04b5a631 h1:E3k9OgTMWWTm/Aytj4ON3Rbg8Bf/S9ZKtCk5EWXQIC4= -github.com/multiversx/mx-chain-communication-go v1.0.7-0.20230920140934-77ca04b5a631/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= +github.com/multiversx/mx-chain-communication-go v1.0.7 h1:7qeDBcqmGYYhSqFcpwv0qKkR3ahOfIMbRwXYEnOt/do= +github.com/multiversx/mx-chain-communication-go v1.0.7/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= github.com/multiversx/mx-chain-core-go v1.2.16 h1:m0hUNmZQjGJxKDLQOHoM9jSaeDfVTbyd+mqiS8+NckE= github.com/multiversx/mx-chain-core-go v1.2.16/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= github.com/multiversx/mx-chain-crypto-go v1.2.8 h1:wOgVlUaO5X4L8iEbFjcQcL8SZvv6WZ7LqH73BiRPhxU= From 2f139c0ad7ebb3069bdb28a8fb5d499b5954bb57 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 6 Nov 2023 13:24:06 +0200 Subject: [PATCH 182/189] extrac check metrics response --- client/transport/transport.go | 8 +++++++- client/transport/transport_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/client/transport/transport.go b/client/transport/transport.go index 931f59b6..3b8542e8 100644 --- a/client/transport/transport.go +++ b/client/transport/transport.go @@ -40,7 +40,13 @@ func (m *metricsTransport) RoundTrip(req *http.Request) (*http.Response, error) startTime := time.Now() size := req.ContentLength + + var statusCode int resp, err := m.transport.RoundTrip(req) + if err == nil { + statusCode = resp.StatusCode + } + duration := time.Since(startTime) valueFromCtx := req.Context().Value(request.ContextKey) @@ -50,7 +56,7 @@ func (m *metricsTransport) RoundTrip(req *http.Request) (*http.Response, error) topic := fmt.Sprintf("%s", valueFromCtx) m.statusMetrics.AddIndexingData(metrics.ArgsAddIndexingData{ - StatusCode: resp.StatusCode, + StatusCode: statusCode, GotError: err != nil, MessageLen: uint64(size), Topic: topic, diff --git a/client/transport/transport_test.go b/client/transport/transport_test.go index 31c78ef9..8be7b196 100644 --- a/client/transport/transport_test.go +++ b/client/transport/transport_test.go @@ -3,6 +3,7 @@ package transport import ( "bytes" "context" + "errors" "net/http" "testing" @@ -34,6 +35,30 @@ func TestMetricsTransport_NilRequest(t *testing.T) { require.Equal(t, errNilRequest, err) } +func TestMetricsTransport_RoundTripNilResponseShouldWork(t *testing.T) { + t.Parallel() + + metricsHandler := metrics.NewStatusMetrics() + transportHandler, _ := NewMetricsTransport(metricsHandler) + + testErr := errors.New("test") + transportHandler.transport = &mock.TransportMock{ + Response: nil, + Err: testErr, + } + + testTopic := "test" + contextWithValue := context.WithValue(context.Background(), request.ContextKey, testTopic) + req, _ := http.NewRequestWithContext(contextWithValue, http.MethodGet, "dummy", bytes.NewBuffer([]byte("test"))) + + _, _ = transportHandler.RoundTrip(req) + + metricsMap := metricsHandler.GetMetrics() + require.Equal(t, uint64(1), metricsMap[testTopic].OperationsCount) + require.Equal(t, uint64(1), metricsMap[testTopic].TotalErrorsCount) + require.Equal(t, uint64(4), metricsMap[testTopic].TotalData) +} + func TestMetricsTransport_RoundTrip(t *testing.T) { t.Parallel() From 8fe7b6595c36e0c286952fb6d420bfc4c60585b7 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 7 Nov 2023 11:13:29 +0200 Subject: [PATCH 183/189] update go mod --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9fbf9d92..05b99800 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 - github.com/multiversx/mx-chain-communication-go v1.0.7 + github.com/multiversx/mx-chain-communication-go v1.0.9-0.20231107090642-a5c42bc5f83d github.com/multiversx/mx-chain-core-go v1.2.16 github.com/multiversx/mx-chain-logger-go v1.0.13 github.com/multiversx/mx-chain-vm-common-go v1.5.2 diff --git a/go.sum b/go.sum index 4f913b00..7361c9bb 100644 --- a/go.sum +++ b/go.sum @@ -247,8 +247,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiversx/mx-chain-communication-go v1.0.7 h1:7qeDBcqmGYYhSqFcpwv0qKkR3ahOfIMbRwXYEnOt/do= -github.com/multiversx/mx-chain-communication-go v1.0.7/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= +github.com/multiversx/mx-chain-communication-go v1.0.9-0.20231107090642-a5c42bc5f83d h1:DpTNMqloxH+jqidZ9ZUXH6mqlW8/jKwfhyripPIeU/4= +github.com/multiversx/mx-chain-communication-go v1.0.9-0.20231107090642-a5c42bc5f83d/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= github.com/multiversx/mx-chain-core-go v1.2.16 h1:m0hUNmZQjGJxKDLQOHoM9jSaeDfVTbyd+mqiS8+NckE= github.com/multiversx/mx-chain-core-go v1.2.16/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= github.com/multiversx/mx-chain-crypto-go v1.2.8 h1:wOgVlUaO5X4L8iEbFjcQcL8SZvv6WZ7LqH73BiRPhxU= From c386b4e51dd2ea8b3612ff636aac8822d47fc4bd Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Tue, 7 Nov 2023 12:04:36 +0200 Subject: [PATCH 184/189] change commit hash --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 05b99800..7051c179 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 - github.com/multiversx/mx-chain-communication-go v1.0.9-0.20231107090642-a5c42bc5f83d + github.com/multiversx/mx-chain-communication-go v1.0.9-0.20231107095942-cc2ce30f1d67 github.com/multiversx/mx-chain-core-go v1.2.16 github.com/multiversx/mx-chain-logger-go v1.0.13 github.com/multiversx/mx-chain-vm-common-go v1.5.2 diff --git a/go.sum b/go.sum index 7361c9bb..74376ab4 100644 --- a/go.sum +++ b/go.sum @@ -247,8 +247,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiversx/mx-chain-communication-go v1.0.9-0.20231107090642-a5c42bc5f83d h1:DpTNMqloxH+jqidZ9ZUXH6mqlW8/jKwfhyripPIeU/4= -github.com/multiversx/mx-chain-communication-go v1.0.9-0.20231107090642-a5c42bc5f83d/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= +github.com/multiversx/mx-chain-communication-go v1.0.9-0.20231107095942-cc2ce30f1d67 h1:O51TqeklC6arawKEkDMFkNQdnURKme8x7mzq9S1jTmQ= +github.com/multiversx/mx-chain-communication-go v1.0.9-0.20231107095942-cc2ce30f1d67/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= github.com/multiversx/mx-chain-core-go v1.2.16 h1:m0hUNmZQjGJxKDLQOHoM9jSaeDfVTbyd+mqiS8+NckE= github.com/multiversx/mx-chain-core-go v1.2.16/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= github.com/multiversx/mx-chain-crypto-go v1.2.8 h1:wOgVlUaO5X4L8iEbFjcQcL8SZvv6WZ7LqH73BiRPhxU= From 81815edac0ad6f802b5f1419e7580f0e776b2f45 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 17 Nov 2023 11:58:18 +0200 Subject: [PATCH 185/189] proper tag --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7051c179..3a53b4be 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 - github.com/multiversx/mx-chain-communication-go v1.0.9-0.20231107095942-cc2ce30f1d67 + github.com/multiversx/mx-chain-communication-go v1.0.10 github.com/multiversx/mx-chain-core-go v1.2.16 github.com/multiversx/mx-chain-logger-go v1.0.13 github.com/multiversx/mx-chain-vm-common-go v1.5.2 diff --git a/go.sum b/go.sum index 74376ab4..339cffe8 100644 --- a/go.sum +++ b/go.sum @@ -247,8 +247,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiversx/mx-chain-communication-go v1.0.9-0.20231107095942-cc2ce30f1d67 h1:O51TqeklC6arawKEkDMFkNQdnURKme8x7mzq9S1jTmQ= -github.com/multiversx/mx-chain-communication-go v1.0.9-0.20231107095942-cc2ce30f1d67/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= +github.com/multiversx/mx-chain-communication-go v1.0.10 h1:tvbrhVOBL39ONoBFmIkwhmjhFIpDwIUXnBtXGphmYJc= +github.com/multiversx/mx-chain-communication-go v1.0.10/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= github.com/multiversx/mx-chain-core-go v1.2.16 h1:m0hUNmZQjGJxKDLQOHoM9jSaeDfVTbyd+mqiS8+NckE= github.com/multiversx/mx-chain-core-go v1.2.16/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= github.com/multiversx/mx-chain-crypto-go v1.2.8 h1:wOgVlUaO5X4L8iEbFjcQcL8SZvv6WZ7LqH73BiRPhxU= From 7a25154391d31dbc8fc319db4516c8a8bf7669c3 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Fri, 17 Nov 2023 13:17:11 +0200 Subject: [PATCH 186/189] proper tag --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3a53b4be..46f5f423 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 - github.com/multiversx/mx-chain-communication-go v1.0.10 + github.com/multiversx/mx-chain-communication-go v1.0.11 github.com/multiversx/mx-chain-core-go v1.2.16 github.com/multiversx/mx-chain-logger-go v1.0.13 github.com/multiversx/mx-chain-vm-common-go v1.5.2 diff --git a/go.sum b/go.sum index 339cffe8..04be2f1a 100644 --- a/go.sum +++ b/go.sum @@ -247,8 +247,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiversx/mx-chain-communication-go v1.0.10 h1:tvbrhVOBL39ONoBFmIkwhmjhFIpDwIUXnBtXGphmYJc= -github.com/multiversx/mx-chain-communication-go v1.0.10/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= +github.com/multiversx/mx-chain-communication-go v1.0.11 h1:lmRL7YYXH1vhZQ3/WCbNOD3ojCG/DaE1gDEV22f5DgE= +github.com/multiversx/mx-chain-communication-go v1.0.11/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= github.com/multiversx/mx-chain-core-go v1.2.16 h1:m0hUNmZQjGJxKDLQOHoM9jSaeDfVTbyd+mqiS8+NckE= github.com/multiversx/mx-chain-core-go v1.2.16/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= github.com/multiversx/mx-chain-crypto-go v1.2.8 h1:wOgVlUaO5X4L8iEbFjcQcL8SZvv6WZ7LqH73BiRPhxU= From 2b5996708222df0c48adc8d202b22ec9d59e3865 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Mon, 20 Nov 2023 10:37:47 +0200 Subject: [PATCH 187/189] proper tag --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 46f5f423..2dd2ccb9 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 - github.com/multiversx/mx-chain-communication-go v1.0.11 + github.com/multiversx/mx-chain-communication-go v1.0.12 github.com/multiversx/mx-chain-core-go v1.2.16 github.com/multiversx/mx-chain-logger-go v1.0.13 github.com/multiversx/mx-chain-vm-common-go v1.5.2 diff --git a/go.sum b/go.sum index 04be2f1a..87bf48e2 100644 --- a/go.sum +++ b/go.sum @@ -247,8 +247,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiversx/mx-chain-communication-go v1.0.11 h1:lmRL7YYXH1vhZQ3/WCbNOD3ojCG/DaE1gDEV22f5DgE= -github.com/multiversx/mx-chain-communication-go v1.0.11/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= +github.com/multiversx/mx-chain-communication-go v1.0.12 h1:67WOaf87gpwouydD1AAOHw5LMGZh7NfITrp/KqFY3Tw= +github.com/multiversx/mx-chain-communication-go v1.0.12/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= github.com/multiversx/mx-chain-core-go v1.2.16 h1:m0hUNmZQjGJxKDLQOHoM9jSaeDfVTbyd+mqiS8+NckE= github.com/multiversx/mx-chain-core-go v1.2.16/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= github.com/multiversx/mx-chain-crypto-go v1.2.8 h1:wOgVlUaO5X4L8iEbFjcQcL8SZvv6WZ7LqH73BiRPhxU= From 4694690a43a99bfc2bf1133c5ecc8ceb44b8d23c Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 29 Nov 2023 09:33:59 +0200 Subject: [PATCH 188/189] integrate burn for all --- .../logsevents/esdtPropertiesProcessor.go | 21 +++++++++++++------ process/elasticproc/logsevents/serialize.go | 5 ++++- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/process/elasticproc/logsevents/esdtPropertiesProcessor.go b/process/elasticproc/logsevents/esdtPropertiesProcessor.go index 1b1802ec..004f9790 100644 --- a/process/elasticproc/logsevents/esdtPropertiesProcessor.go +++ b/process/elasticproc/logsevents/esdtPropertiesProcessor.go @@ -4,6 +4,7 @@ import ( "unicode" "github.com/multiversx/mx-chain-core-go/core" + vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) const ( @@ -23,10 +24,12 @@ func newEsdtPropertiesProcessor(pubKeyConverter core.PubkeyConverter) *esdtPrope return &esdtPropertiesProc{ pubKeyConverter: pubKeyConverter, rolesOperationsIdentifiers: map[string]struct{}{ - core.BuiltInFunctionSetESDTRole: {}, - core.BuiltInFunctionUnSetESDTRole: {}, - core.BuiltInFunctionESDTNFTCreateRoleTransfer: {}, - upgradePropertiesEvent: {}, + core.BuiltInFunctionSetESDTRole: {}, + core.BuiltInFunctionUnSetESDTRole: {}, + core.BuiltInFunctionESDTNFTCreateRoleTransfer: {}, + upgradePropertiesEvent: {}, + vmcommon.BuiltInFunctionESDTUnSetBurnRoleForAll: {}, + vmcommon.BuiltInFunctionESDTSetBurnRoleForAll: {}, }, } } @@ -67,10 +70,16 @@ func (epp *esdtPropertiesProc) processEvent(args *argsProcessEvent) argOutputPro } } - shouldAddRole := identifier == core.BuiltInFunctionSetESDTRole + shouldAddRole := identifier == core.BuiltInFunctionSetESDTRole || identifier == vmcommon.BuiltInFunctionESDTSetBurnRoleForAll + addrBech := epp.pubKeyConverter.SilentEncode(args.event.GetAddress(), log) for _, roleBytes := range rolesBytes { - args.tokenRolesAndProperties.AddRole(string(topics[tokenTopicsIndex]), addrBech, string(roleBytes), shouldAddRole) + addr := addrBech + if string(roleBytes) == vmcommon.ESDTRoleBurnForAll { + addr = "" + } + + args.tokenRolesAndProperties.AddRole(string(topics[tokenTopicsIndex]), addr, string(roleBytes), shouldAddRole) } return argOutputProcessEvent{ diff --git a/process/elasticproc/logsevents/serialize.go b/process/elasticproc/logsevents/serialize.go index c9db9274..488b1282 100644 --- a/process/elasticproc/logsevents/serialize.go +++ b/process/elasticproc/logsevents/serialize.go @@ -299,7 +299,10 @@ func serializeRoleData(buffSlice *data.BufferSlice, rd *tokeninfo.RoleData, role codeToExecute := ` if (ctx._source.containsKey('roles')) { if (ctx._source.roles.containsKey(params.role)) { - ctx._source.roles.get(params.role).removeIf(p -> p.equals(params.address)) + ctx._source.roles.get(params.role).removeIf(p -> p.equals(params.address)); + if (ctx._source.roles.get(params.role).length == 0) { + ctx._source.roles.remove(params.role) + } } } ` From 70ab075169efd4c8a8c3cde58374f99df02287a8 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 29 Nov 2023 09:39:59 +0200 Subject: [PATCH 189/189] fix tests --- .../testdata/issueTokenAndSetRoles/token-after-unset-role.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integrationtests/testdata/issueTokenAndSetRoles/token-after-unset-role.json b/integrationtests/testdata/issueTokenAndSetRoles/token-after-unset-role.json index f0fd19fd..42c45bfe 100644 --- a/integrationtests/testdata/issueTokenAndSetRoles/token-after-unset-role.json +++ b/integrationtests/testdata/issueTokenAndSetRoles/token-after-unset-role.json @@ -15,8 +15,7 @@ "roles": { "ESDTRoleNFTCreate": [ "erd1suhxyflu4w4pqdxmushpxzc6a3qszr89m8uswzqcvyh0mh9mzxwqdwkm0x" - ], - "ESDTRoleNFTBurn": [] + ] }, "properties": { "canMint": false,