From 41c6486469676eb88af1131c65dbbd73751458bb Mon Sep 17 00:00:00 2001 From: Nazarii Denha Date: Mon, 2 Sep 2024 17:17:34 +0200 Subject: [PATCH] feat: add decoding methods (#10) * add decoding mehods * add tests for codecv0 and ccodecv1 * decompressing * add decompressing for codecv2 * fix * change zstd library from c binding to full go port * handle error * sync with main * add v3 decoding * refactor: make DAChunkRawTx an alias * address comments * comment * comment * address comments * fix test * support v4 * address renaming nit-picks --------- Co-authored-by: jonastheis <4181434+jonastheis@users.noreply.github.com> --- encoding/bitmap.go | 27 +++++++ encoding/codecv0/codecv0.go | 67 ++++++++++++++++ encoding/codecv0/codecv0_test.go | 48 ++++++++++- encoding/codecv1/codecv1.go | 133 +++++++++++++++++++++++++++++++ encoding/codecv1/codecv1_test.go | 59 +++++++++++++- encoding/codecv2/codecv2.go | 22 +++++ encoding/codecv2/codecv2_test.go | 86 +++++++++++++++++++- encoding/codecv3/codecv3.go | 13 +++ encoding/codecv3/codecv3_test.go | 5 +- encoding/codecv4/codecv4.go | 26 ++++++ encoding/codecv4/codecv4_test.go | 5 +- encoding/da.go | 41 ++++++++++ encoding/da_test.go | 3 +- go.mod | 1 + go.sum | 2 + 15 files changed, 529 insertions(+), 9 deletions(-) diff --git a/encoding/bitmap.go b/encoding/bitmap.go index 7ada6d6..5631983 100644 --- a/encoding/bitmap.go +++ b/encoding/bitmap.go @@ -63,3 +63,30 @@ func ConstructSkippedBitmap(batchIndex uint64, chunks []*Chunk, totalL1MessagePo return bitmapBytes, nextIndex, nil } + +// DecodeBitmap decodes skipped L1 message bitmap of the batch from bytes to big.Int's +func DecodeBitmap(skippedL1MessageBitmap []byte, totalL1MessagePopped int) ([]*big.Int, error) { + length := len(skippedL1MessageBitmap) + if length%32 != 0 { + return nil, fmt.Errorf("skippedL1MessageBitmap length doesn't match, skippedL1MessageBitmap length should be equal 0 modulo 32, length of skippedL1MessageBitmap: %v", length) + } + if length*8 < totalL1MessagePopped { + return nil, fmt.Errorf("skippedL1MessageBitmap length is too small, skippedL1MessageBitmap length should be at least %v, length of skippedL1MessageBitmap: %v", (totalL1MessagePopped+7)/8, length) + } + var skippedBitmap []*big.Int + for index := 0; index < length/32; index++ { + bitmap := big.NewInt(0).SetBytes(skippedL1MessageBitmap[index*32 : index*32+32]) + skippedBitmap = append(skippedBitmap, bitmap) + } + return skippedBitmap, nil +} + +// IsL1MessageSkipped checks if index is skipped in bitmap +func IsL1MessageSkipped(skippedBitmap []*big.Int, index uint64) bool { + if index > uint64(len(skippedBitmap))*256 { + return false + } + quo := index / 256 + rem := index % 256 + return skippedBitmap[quo].Bit(int(rem)) != 0 +} diff --git a/encoding/codecv0/codecv0.go b/encoding/codecv0/codecv0.go index f757a93..2cc8e8e 100644 --- a/encoding/codecv0/codecv0.go +++ b/encoding/codecv0/codecv0.go @@ -16,6 +16,9 @@ import ( "github.com/scroll-tech/da-codec/encoding" ) +const BlockContextByteSize = 60 +const TxLenByteSize = 4 + // DABlock represents a Data Availability Block. type DABlock struct { BlockNumber uint64 @@ -32,6 +35,12 @@ type DAChunk struct { Transactions [][]*types.TransactionData } +// DAChunkRawTx groups consecutive DABlocks with their L2 transactions, L1 msgs are loaded in another place. +type DAChunkRawTx struct { + Blocks []*DABlock + Transactions []types.Transactions +} + // DABatch contains metadata about a batch of DAChunks. type DABatch struct { Version uint8 @@ -179,6 +188,64 @@ func (c *DAChunk) Encode() ([]byte, error) { return chunkBytes, nil } +// DecodeDAChunksRawTx takes a byte slice and decodes it into a []*DAChunkRawTx. +func DecodeDAChunksRawTx(bytes [][]byte) ([]*DAChunkRawTx, error) { + var chunks []*DAChunkRawTx + for _, chunk := range bytes { + if len(chunk) < 1 { + return nil, fmt.Errorf("invalid chunk, length is less than 1") + } + + numBlocks := int(chunk[0]) + if len(chunk) < 1+numBlocks*BlockContextByteSize { + return nil, fmt.Errorf("chunk size doesn't match with numBlocks, byte length of chunk: %v, expected length: %v", len(chunk), 1+numBlocks*BlockContextByteSize) + } + + blocks := make([]*DABlock, numBlocks) + for i := 0; i < numBlocks; i++ { + startIdx := 1 + i*BlockContextByteSize // add 1 to skip numBlocks byte + endIdx := startIdx + BlockContextByteSize + blocks[i] = &DABlock{} + err := blocks[i].Decode(chunk[startIdx:endIdx]) + if err != nil { + return nil, err + } + } + + var transactions []types.Transactions + currentIndex := 1 + numBlocks*BlockContextByteSize + for _, block := range blocks { + var blockTransactions types.Transactions + // ignore L1 msg transactions from the block, consider only L2 transactions + txNum := int(block.NumTransactions - block.NumL1Messages) + for i := 0; i < txNum; i++ { + if len(chunk) < currentIndex+TxLenByteSize { + return nil, fmt.Errorf("chunk size doesn't match, next tx size is less then 4, byte length of chunk: %v, expected minimum length: %v, txNum without l1 msgs: %d", len(chunk), currentIndex+TxLenByteSize, i) + } + txLen := int(binary.BigEndian.Uint32(chunk[currentIndex : currentIndex+TxLenByteSize])) + if len(chunk) < currentIndex+TxLenByteSize+txLen { + return nil, fmt.Errorf("chunk size doesn't match with next tx length, byte length of chunk: %v, expected minimum length: %v, txNum without l1 msgs: %d", len(chunk), currentIndex+TxLenByteSize+txLen, i) + } + txData := chunk[currentIndex+TxLenByteSize : currentIndex+TxLenByteSize+txLen] + tx := &types.Transaction{} + err := tx.UnmarshalBinary(txData) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal tx, pos of tx in chunk bytes: %d. tx num without l1 msgs: %d, err: %w", currentIndex, i, err) + } + blockTransactions = append(blockTransactions, tx) + currentIndex += TxLenByteSize + txLen + } + transactions = append(transactions, blockTransactions) + } + + chunks = append(chunks, &DAChunkRawTx{ + Blocks: blocks, + Transactions: transactions, + }) + } + return chunks, nil +} + // Hash computes the hash of the DAChunk data. func (c *DAChunk) Hash() (common.Hash, error) { chunkBytes, err := c.Encode() diff --git a/encoding/codecv0/codecv0_test.go b/encoding/codecv0/codecv0_test.go index 330a826..0a5b514 100644 --- a/encoding/codecv0/codecv0_test.go +++ b/encoding/codecv0/codecv0_test.go @@ -7,9 +7,10 @@ import ( "os" "testing" + "github.com/stretchr/testify/assert" + "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/log" - "github.com/stretchr/testify/assert" "github.com/scroll-tech/da-codec/encoding" ) @@ -264,6 +265,38 @@ func TestCodecV0(t *testing.T) { assert.NoError(t, err) assert.Equal(t, 61, len(chunkBytes2)) + daChunksRawTx, err := DecodeDAChunksRawTx([][]byte{chunkBytes1, chunkBytes2}) + assert.NoError(t, err) + // assert number of chunks + assert.Equal(t, 2, len(daChunksRawTx)) + + // assert block in first chunk + assert.Equal(t, 3, len(daChunksRawTx[0].Blocks)) + assert.Equal(t, daChunk1.Blocks[0], daChunksRawTx[0].Blocks[0]) + assert.Equal(t, daChunk1.Blocks[1], daChunksRawTx[0].Blocks[1]) + daChunksRawTx[0].Blocks[2].BaseFee = nil + assert.Equal(t, daChunk1.Blocks[2], daChunksRawTx[0].Blocks[2]) + + // assert block in second chunk + assert.Equal(t, 1, len(daChunksRawTx[1].Blocks)) + daChunksRawTx[1].Blocks[0].BaseFee = nil + assert.Equal(t, daChunk2.Blocks[0], daChunksRawTx[1].Blocks[0]) + + // assert transactions in first chunk + assert.Equal(t, 3, len(daChunksRawTx[0].Transactions)) + // here number of transactions in encoded and decoded chunks may be different, because decodec chunks doesn't contain l1msgs + assert.Equal(t, 2, len(daChunksRawTx[0].Transactions[0])) + assert.Equal(t, 1, len(daChunksRawTx[0].Transactions[1])) + assert.Equal(t, 1, len(daChunksRawTx[0].Transactions[2])) + + assert.EqualValues(t, daChunk1.Transactions[0][0].TxHash, daChunksRawTx[0].Transactions[0][0].Hash().String()) + assert.EqualValues(t, daChunk1.Transactions[0][1].TxHash, daChunksRawTx[0].Transactions[0][1].Hash().String()) + + // assert transactions in second chunk + assert.Equal(t, 1, len(daChunksRawTx[1].Transactions)) + // here number of transactions in encoded and decoded chunks may be different, because decodec chunks doesn't contain l1msgs + assert.Equal(t, 0, len(daChunksRawTx[1].Transactions[0])) + batch = &encoding.Batch{ Index: 1, TotalL1MessagePoppedBefore: 0, @@ -297,6 +330,19 @@ func TestCodecV0(t *testing.T) { decodedBatchHexString = hex.EncodeToString(decodedBatchBytes) assert.Equal(t, batchHexString, decodedBatchHexString) + decodedBitmap, err := encoding.DecodeBitmap(decodedDABatch.SkippedL1MessageBitmap, int(decodedDABatch.L1MessagePopped)) + assert.NoError(t, err) + assert.True(t, encoding.IsL1MessageSkipped(decodedBitmap, 0)) + assert.True(t, encoding.IsL1MessageSkipped(decodedBitmap, 9)) + assert.False(t, encoding.IsL1MessageSkipped(decodedBitmap, 10)) + assert.True(t, encoding.IsL1MessageSkipped(decodedBitmap, 11)) + assert.True(t, encoding.IsL1MessageSkipped(decodedBitmap, 36)) + assert.False(t, encoding.IsL1MessageSkipped(decodedBitmap, 37)) + assert.False(t, encoding.IsL1MessageSkipped(decodedBitmap, 38)) + assert.False(t, encoding.IsL1MessageSkipped(decodedBitmap, 39)) + assert.False(t, encoding.IsL1MessageSkipped(decodedBitmap, 40)) + assert.False(t, encoding.IsL1MessageSkipped(decodedBitmap, 41)) + // Test case: many consecutive L1 Msgs in 1 bitmap, no leading skipped msgs. chunk = &encoding.Chunk{ Blocks: []*encoding.Block{block4}, diff --git a/encoding/codecv1/codecv1.go b/encoding/codecv1/codecv1.go index 72b0a0e..25c6798 100644 --- a/encoding/codecv1/codecv1.go +++ b/encoding/codecv1/codecv1.go @@ -21,12 +21,17 @@ import ( // MaxNumChunks is the maximum number of chunks that a batch can contain. const MaxNumChunks = 15 +const BlockContextByteSize = codecv0.BlockContextByteSize + // DABlock represents a Data Availability Block. type DABlock = codecv0.DABlock // DAChunk groups consecutive DABlocks with their transactions. type DAChunk codecv0.DAChunk +// DAChunkRawTx groups consecutive DABlocks with their L2 transactions, L1 msgs are loaded in another place. +type DAChunkRawTx = codecv0.DAChunkRawTx + // DABatch contains metadata about a batch of DAChunks. type DABatch struct { // header @@ -93,6 +98,41 @@ func (c *DAChunk) Encode() []byte { return chunkBytes } +// DecodeDAChunksRawTx takes a byte slice and decodes it into a []*DAChunkRawTx. +// Beginning from codecv1 tx data posted to blobs, not to chunk bytes in calldata +func DecodeDAChunksRawTx(bytes [][]byte) ([]*DAChunkRawTx, error) { + var chunks []*DAChunkRawTx + for _, chunk := range bytes { + if len(chunk) < 1 { + return nil, fmt.Errorf("invalid chunk, length is less than 1") + } + + numBlocks := int(chunk[0]) + if len(chunk) < 1+numBlocks*BlockContextByteSize { + return nil, fmt.Errorf("chunk size doesn't match with numBlocks, byte length of chunk: %v, expected length: %v", len(chunk), 1+numBlocks*BlockContextByteSize) + } + + blocks := make([]*DABlock, numBlocks) + for i := 0; i < numBlocks; i++ { + startIdx := 1 + i*BlockContextByteSize // add 1 to skip numBlocks byte + endIdx := startIdx + BlockContextByteSize + blocks[i] = &DABlock{} + err := blocks[i].Decode(chunk[startIdx:endIdx]) + if err != nil { + return nil, err + } + } + + var transactions []types.Transactions + + chunks = append(chunks, &DAChunkRawTx{ + Blocks: blocks, + Transactions: transactions, // Transactions field is still empty in the phase of DecodeDAChunksRawTx, because txs moved to bobs and filled in DecodeTxsFromBlob method. + }) + } + return chunks, nil +} + // Hash computes the hash of the DAChunk data. func (c *DAChunk) Hash() (common.Hash, error) { var dataBytes []byte @@ -286,6 +326,99 @@ func constructBlobPayload(chunks []*encoding.Chunk, useMockTxData bool) (*kzg484 return blob, blobVersionedHash, &z, nil } +// DecodeTxsFromBytes decodes txs from blob bytes and writes to chunks +func DecodeTxsFromBytes(blobBytes []byte, chunks []*DAChunkRawTx, maxNumChunks int) error { + numChunks := int(binary.BigEndian.Uint16(blobBytes[0:2])) + if numChunks != len(chunks) { + return fmt.Errorf("blob chunk number is not same as calldata, blob num chunks: %d, calldata num chunks: %d", numChunks, len(chunks)) + } + index := 2 + maxNumChunks*4 + for chunkID, chunk := range chunks { + var transactions []types.Transactions + chunkSize := int(binary.BigEndian.Uint32(blobBytes[2+4*chunkID : 2+4*chunkID+4])) + + chunkBytes := blobBytes[index : index+chunkSize] + curIndex := 0 + for _, block := range chunk.Blocks { + var blockTransactions types.Transactions + txNum := int(block.NumTransactions - block.NumL1Messages) + for i := 0; i < txNum; i++ { + tx, nextIndex, err := GetNextTx(chunkBytes, curIndex) + if err != nil { + return fmt.Errorf("couldn't decode next tx from blob bytes: %w, index: %d", err, index+curIndex+4) + } + curIndex = nextIndex + blockTransactions = append(blockTransactions, tx) + } + transactions = append(transactions, blockTransactions) + } + chunk.Transactions = transactions + index += chunkSize + } + return nil +} + +// DecodeTxsFromBlob decodes txs from blob bytes and writes to chunks +func DecodeTxsFromBlob(blob *kzg4844.Blob, chunks []*DAChunkRawTx) error { + batchBytes := encoding.BytesFromBlobCanonical(blob) + return DecodeTxsFromBytes(batchBytes[:], chunks, MaxNumChunks) +} + +var errSmallLength error = fmt.Errorf("length of blob bytes is too small") + +// GetNextTx parses blob bytes to find length of payload of next Tx and decode it +func GetNextTx(bytes []byte, index int) (*types.Transaction, int, error) { + var nextIndex int + length := len(bytes) + if length < index+1 { + return nil, 0, errSmallLength + } + var txBytes []byte + if bytes[index] <= 0x7f { + // the first byte is transaction type, rlp encoding begins from next byte + txBytes = append(txBytes, bytes[index]) + index++ + } + if length < index+1 { + return nil, 0, errSmallLength + } + if bytes[index] >= 0xc0 && bytes[index] <= 0xf7 { + // length of payload is simply bytes[index] - 0xc0 + payloadLen := int(bytes[index] - 0xc0) + if length < index+1+payloadLen { + return nil, 0, errSmallLength + } + txBytes = append(txBytes, bytes[index:index+1+payloadLen]...) + nextIndex = index + 1 + payloadLen + } else if bytes[index] > 0xf7 { + // the length of payload is encoded in next bytes[index] - 0xf7 bytes + // length of bytes representation of length of payload + lenPayloadLen := int(bytes[index] - 0xf7) + if length < index+1+lenPayloadLen { + return nil, 0, errSmallLength + } + lenBytes := bytes[index+1 : index+1+lenPayloadLen] + for len(lenBytes) < 8 { + lenBytes = append([]byte{0x0}, lenBytes...) + } + payloadLen := binary.BigEndian.Uint64(lenBytes) + + if length < index+1+lenPayloadLen+int(payloadLen) { + return nil, 0, errSmallLength + } + txBytes = append(txBytes, bytes[index:index+1+lenPayloadLen+int(payloadLen)]...) + nextIndex = index + 1 + lenPayloadLen + int(payloadLen) + } else { + return nil, 0, fmt.Errorf("incorrect format of rlp encoding") + } + tx := &types.Transaction{} + err := tx.UnmarshalBinary(txBytes) + if err != nil { + return nil, 0, fmt.Errorf("failed to unmarshal tx, err: %w", err) + } + return tx, nextIndex, nil +} + // NewDABatchFromBytes decodes the given byte slice into a DABatch. // Note: This function only populates the batch header, it leaves the blob-related fields empty. func NewDABatchFromBytes(data []byte) (*DABatch, error) { diff --git a/encoding/codecv1/codecv1_test.go b/encoding/codecv1/codecv1_test.go index b914ed6..6522c59 100644 --- a/encoding/codecv1/codecv1_test.go +++ b/encoding/codecv1/codecv1_test.go @@ -7,10 +7,11 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/core/types" "github.com/scroll-tech/go-ethereum/crypto/kzg4844" - "github.com/stretchr/testify/assert" "github.com/scroll-tech/da-codec/encoding" "github.com/scroll-tech/da-codec/encoding/codecv0" @@ -473,6 +474,62 @@ func TestCodecV1BatchBlob(t *testing.T) { assert.Equal(t, "0x01b63f87bdd2caa8d43500d47ee59204f61af95339483c62ff436c6beabf47bf", batch.BlobVersionedHash.Hex()) } +func TestCodecV1Decode(t *testing.T) { + trace0 := readBlockFromJSON(t, "../testdata/blockTrace_02.json") + trace1 := readBlockFromJSON(t, "../testdata/blockTrace_03.json") + chunk0 := &encoding.Chunk{Blocks: []*encoding.Block{trace0, trace1}} + daChunk0, err := NewDAChunk(chunk0, 0) + assert.NoError(t, err) + chunkBytes0 := daChunk0.Encode() + + trace2 := readBlockFromJSON(t, "../testdata/blockTrace_04.json") + trace3 := readBlockFromJSON(t, "../testdata/blockTrace_05.json") + chunk1 := &encoding.Chunk{Blocks: []*encoding.Block{trace2, trace3}} + daChunk1, err := NewDAChunk(chunk1, 0) + assert.NoError(t, err) + chunkBytes1 := daChunk1.Encode() + + originalBatch := &encoding.Batch{Chunks: []*encoding.Chunk{chunk0, chunk1}} + batch, err := NewDABatch(originalBatch) + assert.NoError(t, err) + + daChunksRawTx, err := DecodeDAChunksRawTx([][]byte{chunkBytes0, chunkBytes1}) + assert.NoError(t, err) + // assert number of chunks + assert.Equal(t, 2, len(daChunksRawTx)) + + // assert block in first chunk + assert.Equal(t, 2, len(daChunksRawTx[0].Blocks)) + assert.Equal(t, daChunk0.Blocks[0], daChunksRawTx[0].Blocks[0]) + assert.Equal(t, daChunk0.Blocks[1], daChunksRawTx[0].Blocks[1]) + + // assert block in second chunk + assert.Equal(t, 2, len(daChunksRawTx[1].Blocks)) + daChunksRawTx[1].Blocks[0].BaseFee = nil + assert.Equal(t, daChunk1.Blocks[0], daChunksRawTx[1].Blocks[0]) + daChunksRawTx[1].Blocks[1].BaseFee = nil + assert.Equal(t, daChunk1.Blocks[1], daChunksRawTx[1].Blocks[1]) + + blob := batch.Blob() + err = DecodeTxsFromBlob(blob, daChunksRawTx) + assert.NoError(t, err) + + // assert transactions in first chunk + assert.Equal(t, 2, len(daChunksRawTx[0].Transactions)) + // here number of transactions in encoded and decoded chunks may be different, because decodec chunks doesn't contain l1msgs + assert.Equal(t, 2, len(daChunksRawTx[0].Transactions[0])) + assert.Equal(t, 1, len(daChunksRawTx[0].Transactions[1])) + + assert.EqualValues(t, daChunk0.Transactions[0][0].TxHash, daChunksRawTx[0].Transactions[0][0].Hash().String()) + assert.EqualValues(t, daChunk0.Transactions[0][1].TxHash, daChunksRawTx[0].Transactions[0][1].Hash().String()) + + // assert transactions in second chunk + assert.Equal(t, 2, len(daChunksRawTx[1].Transactions)) + // here number of transactions in encoded and decoded chunks may be different, because decodec chunks doesn't contain l1msgs + assert.Equal(t, 1, len(daChunksRawTx[1].Transactions[0])) + assert.Equal(t, 0, len(daChunksRawTx[1].Transactions[1])) +} + func TestCodecV1BatchChallenge(t *testing.T) { trace2 := readBlockFromJSON(t, "../testdata/blockTrace_02.json") chunk2 := &encoding.Chunk{Blocks: []*encoding.Block{trace2}} diff --git a/encoding/codecv2/codecv2.go b/encoding/codecv2/codecv2.go index a7a2607..dd00dc9 100644 --- a/encoding/codecv2/codecv2.go +++ b/encoding/codecv2/codecv2.go @@ -22,12 +22,17 @@ import ( // MaxNumChunks is the maximum number of chunks that a batch can contain. const MaxNumChunks = 45 +const BlockContextByteSize = codecv1.BlockContextByteSize + // DABlock represents a Data Availability Block. type DABlock = codecv1.DABlock // DAChunk groups consecutive DABlocks with their transactions. type DAChunk = codecv1.DAChunk +// DAChunkRawTx groups consecutive DABlocks with their L2 transactions, L1 msgs are loaded in another place. +type DAChunkRawTx = codecv1.DAChunkRawTx + // DABatch contains metadata about a batch of DAChunks. type DABatch struct { // header @@ -55,6 +60,11 @@ func NewDAChunk(chunk *encoding.Chunk, totalL1MessagePoppedBefore uint64) (*DACh return codecv1.NewDAChunk(chunk, totalL1MessagePoppedBefore) } +// DecodeDAChunksRawTx takes a byte slice and decodes it into a []*DAChunkRawTx. +func DecodeDAChunksRawTx(bytes [][]byte) ([]*DAChunkRawTx, error) { + return codecv1.DecodeDAChunksRawTx(bytes) +} + // NewDABatch creates a DABatch from the provided encoding.Batch. func NewDABatch(batch *encoding.Batch) (*DABatch, error) { // this encoding can only support a fixed number of chunks per batch @@ -217,6 +227,18 @@ func ConstructBlobPayload(chunks []*encoding.Chunk, useMockTxData bool) (*kzg484 return blob, blobVersionedHash, &z, blobBytes, nil } +// DecodeTxsFromBlob decodes txs from blob bytes and writes to chunks +func DecodeTxsFromBlob(blob *kzg4844.Blob, chunks []*DAChunkRawTx) error { + compressedBytes := encoding.BytesFromBlobCanonical(blob) + magics := []byte{0x28, 0xb5, 0x2f, 0xfd} + + batchBytes, err := encoding.DecompressScrollBlobToBatch(append(magics, compressedBytes[:]...)) + if err != nil { + return err + } + return codecv1.DecodeTxsFromBytes(batchBytes, chunks, MaxNumChunks) +} + // NewDABatchFromBytes decodes the given byte slice into a DABatch. // Note: This function only populates the batch header, it leaves the blob-related fields empty. func NewDABatchFromBytes(data []byte) (*DABatch, error) { diff --git a/encoding/codecv2/codecv2_test.go b/encoding/codecv2/codecv2_test.go index c34f608..69713d5 100644 --- a/encoding/codecv2/codecv2_test.go +++ b/encoding/codecv2/codecv2_test.go @@ -7,15 +7,17 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/core/types" "github.com/scroll-tech/go-ethereum/crypto" "github.com/scroll-tech/go-ethereum/crypto/kzg4844" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/scroll-tech/da-codec/encoding" "github.com/scroll-tech/da-codec/encoding/codecv0" + "github.com/scroll-tech/da-codec/encoding/zstd" ) func TestCodecV2BlockEncode(t *testing.T) { @@ -393,6 +395,86 @@ func TestCodecV2BatchDataHash(t *testing.T) { assert.Equal(t, "0x9b0f37c563d27d9717ab16d47075df996c54fe110130df6b11bfd7230e134767", batch.DataHash.Hex()) } +func TestCodecV2CompressDecompress(t *testing.T) { + blobString := "00" + "0001" + "000000e6" + "00000000" + "00000000" + "00000000" + "00000000" + "00000000" + "00000000" + "00" + "00" + "000000" + "00000000" + "00000000" + "00000000" + "00000000" + "00000000" + "00000000" + "00000000" + + // tx payload + "00f87180843b9aec2e8307a12094c0c4c8baea3f6acb49b6e1fb9e2adeceeacb000ca28a152d02c7e14af60000008083019ecea0ab07ae99c67aa78e7ba5cf670081e90cc32b219b1de102513d56548a41e86df514a034cbd19feacd73e8ce6400d00c4d1996b9b5243c578fd7f51bfaec288bbaf42a8bf87101843b9aec2e830007a1209401bae6bf68e9a03fb2bc0615b1bf0d69ce9411ed8a152d02c7e14a00f60000008083019ecea0f039985866d8256f10c1be4f7b2cace28d8f20bde2007e2604393eb095b7f77316a05a3e6e81065f2b4604bcec5bd4aba68483599600fc3f879380aac1c09c6eed32f1" + blobBytes, err := hex.DecodeString(blobString) + assert.NoError(t, err) + + compressed, err := zstd.CompressScrollBatchBytes(blobBytes) + assert.NoError(t, err) + + blob, err := encoding.MakeBlobCanonical(compressed) + assert.NoError(t, err) + + res := encoding.BytesFromBlobCanonical(blob) + compressedBytes := res[:] + magics := []byte{0x28, 0xb5, 0x2f, 0xfd} + compressedBytes = append(magics, compressedBytes...) + + decompressedBlobBytes, err := encoding.DecompressScrollBlobToBatch(compressedBytes) + + assert.NoError(t, err) + assert.Equal(t, blobBytes, decompressedBlobBytes) +} + +func TestCodecV2Decode(t *testing.T) { + trace0 := readBlockFromJSON(t, "../testdata/blockTrace_02.json") + trace1 := readBlockFromJSON(t, "../testdata/blockTrace_03.json") + chunk0 := &encoding.Chunk{Blocks: []*encoding.Block{trace0, trace1}} + daChunk0, err := NewDAChunk(chunk0, 0) + assert.NoError(t, err) + chunkBytes0 := daChunk0.Encode() + + trace2 := readBlockFromJSON(t, "../testdata/blockTrace_04.json") + trace3 := readBlockFromJSON(t, "../testdata/blockTrace_05.json") + chunk1 := &encoding.Chunk{Blocks: []*encoding.Block{trace2, trace3}} + daChunk1, err := NewDAChunk(chunk1, 0) + assert.NoError(t, err) + chunkBytes1 := daChunk1.Encode() + + originalBatch := &encoding.Batch{Chunks: []*encoding.Chunk{chunk0, chunk1}} + batch, err := NewDABatch(originalBatch) + assert.NoError(t, err) + + daChunksRawTx, err := DecodeDAChunksRawTx([][]byte{chunkBytes0, chunkBytes1}) + assert.NoError(t, err) + // assert number of chunks + assert.Equal(t, 2, len(daChunksRawTx)) + + // assert block in first chunk + assert.Equal(t, 2, len(daChunksRawTx[0].Blocks)) + assert.Equal(t, daChunk0.Blocks[0], daChunksRawTx[0].Blocks[0]) + assert.Equal(t, daChunk0.Blocks[1], daChunksRawTx[0].Blocks[1]) + + // assert block in second chunk + assert.Equal(t, 2, len(daChunksRawTx[1].Blocks)) + daChunksRawTx[1].Blocks[0].BaseFee = nil + assert.Equal(t, daChunk1.Blocks[0], daChunksRawTx[1].Blocks[0]) + daChunksRawTx[1].Blocks[1].BaseFee = nil + assert.Equal(t, daChunk1.Blocks[1], daChunksRawTx[1].Blocks[1]) + + blob := batch.Blob() + err = DecodeTxsFromBlob(blob, daChunksRawTx) + assert.NoError(t, err) + + // assert transactions in first chunk + assert.Equal(t, 2, len(daChunksRawTx[0].Transactions)) + // here number of transactions in encoded and decoded chunks may be different, because decodec chunks doesn't contain l1msgs + assert.Equal(t, 2, len(daChunksRawTx[0].Transactions[0])) + assert.Equal(t, 1, len(daChunksRawTx[0].Transactions[1])) + + assert.EqualValues(t, daChunk0.Transactions[0][0].TxHash, daChunksRawTx[0].Transactions[0][0].Hash().String()) + assert.EqualValues(t, daChunk0.Transactions[0][1].TxHash, daChunksRawTx[0].Transactions[0][1].Hash().String()) + + // assert transactions in second chunk + assert.Equal(t, 2, len(daChunksRawTx[1].Transactions)) + // here number of transactions in encoded and decoded chunks may be different, because decodec chunks doesn't contain l1msgs + assert.Equal(t, 1, len(daChunksRawTx[1].Transactions[0])) + assert.Equal(t, 0, len(daChunksRawTx[1].Transactions[1])) +} + func TestCodecV2BatchBlob(t *testing.T) { trace2 := readBlockFromJSON(t, "../testdata/blockTrace_02.json") chunk2 := &encoding.Chunk{Blocks: []*encoding.Block{trace2}} diff --git a/encoding/codecv3/codecv3.go b/encoding/codecv3/codecv3.go index 819378a..da184ea 100644 --- a/encoding/codecv3/codecv3.go +++ b/encoding/codecv3/codecv3.go @@ -23,6 +23,9 @@ type DABlock = codecv2.DABlock // DAChunk groups consecutive DABlocks with their transactions. type DAChunk = codecv2.DAChunk +// DAChunkRawTx groups consecutive DABlocks with their L2 transactions, L1 msgs are loaded in another place. +type DAChunkRawTx = codecv2.DAChunkRawTx + // DABatch contains metadata about a batch of DAChunks. type DABatch struct { // header @@ -54,6 +57,11 @@ func NewDAChunk(chunk *encoding.Chunk, totalL1MessagePoppedBefore uint64) (*DACh return codecv2.NewDAChunk(chunk, totalL1MessagePoppedBefore) } +// DecodeDAChunksRawTx takes a byte slice and decodes it into a []*DAChunkRawTx. +func DecodeDAChunksRawTx(bytes [][]byte) ([]*DAChunkRawTx, error) { + return codecv2.DecodeDAChunksRawTx(bytes) +} + // NewDABatch creates a DABatch from the provided encoding.Batch. func NewDABatch(batch *encoding.Batch) (*DABatch, error) { // this encoding can only support a fixed number of chunks per batch @@ -125,6 +133,11 @@ func ConstructBlobPayload(chunks []*encoding.Chunk, useMockTxData bool) (*kzg484 return codecv2.ConstructBlobPayload(chunks, useMockTxData) } +// DecodeTxsFromBlob decodes txs from blob bytes and writes to chunks +func DecodeTxsFromBlob(blob *kzg4844.Blob, chunks []*DAChunkRawTx) error { + return codecv2.DecodeTxsFromBlob(blob, chunks) +} + // NewDABatchFromBytes decodes the given byte slice into a DABatch. // Note: This function only populates the batch header, it leaves the blob-related fields empty. func NewDABatchFromBytes(data []byte) (*DABatch, error) { diff --git a/encoding/codecv3/codecv3_test.go b/encoding/codecv3/codecv3_test.go index fef0c12..2a917fd 100644 --- a/encoding/codecv3/codecv3_test.go +++ b/encoding/codecv3/codecv3_test.go @@ -7,12 +7,13 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/core/types" "github.com/scroll-tech/go-ethereum/crypto" "github.com/scroll-tech/go-ethereum/crypto/kzg4844" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/scroll-tech/da-codec/encoding" "github.com/scroll-tech/da-codec/encoding/codecv0" diff --git a/encoding/codecv4/codecv4.go b/encoding/codecv4/codecv4.go index 4c43ff0..d1aa48c 100644 --- a/encoding/codecv4/codecv4.go +++ b/encoding/codecv4/codecv4.go @@ -15,6 +15,7 @@ import ( "github.com/scroll-tech/go-ethereum/log" "github.com/scroll-tech/da-codec/encoding" + "github.com/scroll-tech/da-codec/encoding/codecv1" "github.com/scroll-tech/da-codec/encoding/codecv3" "github.com/scroll-tech/da-codec/encoding/zstd" ) @@ -28,6 +29,9 @@ type DABlock = codecv3.DABlock // DAChunk groups consecutive DABlocks with their transactions. type DAChunk = codecv3.DAChunk +// DAChunkRawTx groups consecutive DABlocks with their L2 transactions, L1 msgs are loaded in another place. +type DAChunkRawTx = codecv3.DAChunkRawTx + // DABatch contains metadata about a batch of DAChunks. type DABatch struct { // header @@ -59,6 +63,11 @@ func NewDAChunk(chunk *encoding.Chunk, totalL1MessagePoppedBefore uint64) (*DACh return codecv3.NewDAChunk(chunk, totalL1MessagePoppedBefore) } +// DecodeDAChunksRawTx takes a byte slice and decodes it into a []*DAChunkRawTx. +func DecodeDAChunksRawTx(bytes [][]byte) ([]*DAChunkRawTx, error) { + return codecv3.DecodeDAChunksRawTx(bytes) +} + // NewDABatch creates a DABatch from the provided encoding.Batch. func NewDABatch(batch *encoding.Batch, enableCompress bool) (*DABatch, error) { // this encoding can only support a fixed number of chunks per batch @@ -239,6 +248,23 @@ func ConstructBlobPayload(chunks []*encoding.Chunk, enableCompress bool, useMock return blob, blobVersionedHash, &z, blobBytes, nil } +// DecodeTxsFromBlob decodes txs from blob bytes and writes to chunks +func DecodeTxsFromBlob(blob *kzg4844.Blob, chunks []*DAChunkRawTx) error { + rawBytes := encoding.BytesFromBlobCanonical(blob) + + // if first byte is 1 - data compressed, 0 - not compressed + if rawBytes[0] == 0x1 { + magics := []byte{0x28, 0xb5, 0x2f, 0xfd} + batchBytes, err := encoding.DecompressScrollBlobToBatch(append(magics, rawBytes[1:]...)) + if err != nil { + return err + } + return codecv1.DecodeTxsFromBytes(batchBytes, chunks, MaxNumChunks) + } else { + return codecv1.DecodeTxsFromBytes(rawBytes[1:], chunks, MaxNumChunks) + } +} + // NewDABatchFromBytes decodes the given byte slice into a DABatch. // Note: This function only populates the batch header, it leaves the blob-related fields empty. func NewDABatchFromBytes(data []byte) (*DABatch, error) { diff --git a/encoding/codecv4/codecv4_test.go b/encoding/codecv4/codecv4_test.go index fa1eee0..a1b13cf 100644 --- a/encoding/codecv4/codecv4_test.go +++ b/encoding/codecv4/codecv4_test.go @@ -6,12 +6,13 @@ import ( "os" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/core/types" "github.com/scroll-tech/go-ethereum/crypto" "github.com/scroll-tech/go-ethereum/crypto/kzg4844" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/scroll-tech/da-codec/encoding" "github.com/scroll-tech/da-codec/encoding/codecv0" diff --git a/encoding/da.go b/encoding/da.go index 3adf08c..8ce6c35 100644 --- a/encoding/da.go +++ b/encoding/da.go @@ -1,10 +1,13 @@ package encoding import ( + "bytes" "encoding/binary" "fmt" "math/big" + "github.com/klauspost/compress/zstd" + "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/common/hexutil" "github.com/scroll-tech/go-ethereum/core/types" @@ -354,6 +357,44 @@ func MakeBlobCanonical(blobBytes []byte) (*kzg4844.Blob, error) { return &blob, nil } +// BytesFromBlobCanonical converts the canonical blob representation into the raw blob data +func BytesFromBlobCanonical(blob *kzg4844.Blob) [126976]byte { + var blobBytes [126976]byte + for from := 0; from < len(blob); from += 32 { + copy(blobBytes[from/32*31:], blob[from+1:from+32]) + } + return blobBytes +} + +// DecompressScrollBlobToBatch decompresses the given blob bytes into scroll batch bytes +func DecompressScrollBlobToBatch(compressedBytes []byte) ([]byte, error) { + // decompress data in stream and in batches of bytes, because we don't know actual length of compressed data + var res []byte + readBatchSize := 131072 + batchOfBytes := make([]byte, readBatchSize) + + r := bytes.NewReader(compressedBytes) + zr, err := zstd.NewReader(r) + if err != nil { + return nil, err + } + defer zr.Close() + + for { + i, err := zr.Read(batchOfBytes) + res = append(res, batchOfBytes[:i]...) // append already decoded bytes even if we meet error + // the error here is supposed to be EOF or similar that indicates that buffer has been read until the end + // we should return all data that read by this moment + if i < readBatchSize || err != nil { + break + } + } + if len(res) == 0 { + return nil, fmt.Errorf("failed to decompress blob bytes") + } + return res, nil +} + // CalculatePaddedBlobSize calculates the required size on blob storage // where every 32 bytes can store only 31 bytes of actual data, with the first byte being zero. func CalculatePaddedBlobSize(dataSize uint64) uint64 { diff --git a/encoding/da_test.go b/encoding/da_test.go index 1665b43..0481597 100644 --- a/encoding/da_test.go +++ b/encoding/da_test.go @@ -5,10 +5,11 @@ import ( "os" "testing" + "github.com/stretchr/testify/assert" + "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/core/types" "github.com/scroll-tech/go-ethereum/log" - "github.com/stretchr/testify/assert" ) func TestMain(m *testing.M) { diff --git a/go.mod b/go.mod index 8d5696e..0a84dd2 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/go-stack/stack v1.8.1 // indirect github.com/holiman/uint256 v1.2.4 // indirect github.com/iden3/go-iden3-crypto v0.0.15 // indirect + github.com/klauspost/compress v1.17.9 github.com/kr/text v0.2.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 29a3574..0a2e1b6 100644 --- a/go.sum +++ b/go.sum @@ -46,6 +46,8 @@ github.com/iden3/go-iden3-crypto v0.0.15/go.mod h1:dLpM4vEPJ3nDHzhWFXDjzkn1qHoBe github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=