From 582d09483ef2291b020e67628b4c0362502b0bac Mon Sep 17 00:00:00 2001 From: classicalliu Date: Fri, 19 Aug 2022 16:01:38 +0800 Subject: [PATCH 1/2] fix: log.transaction_index always be 0 --- crates/indexer/src/indexer.rs | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/crates/indexer/src/indexer.rs b/crates/indexer/src/indexer.rs index 55bd8870..f8db53c3 100644 --- a/crates/indexer/src/indexer.rs +++ b/crates/indexer/src/indexer.rs @@ -104,7 +104,7 @@ impl Web3Indexer { Ok(row.and_then(|(n,)| n.to_u64())) } - // NOTE: remember to update `tx_index`, `cumulative_gas_used` + // NOTE: remember to update `tx_index`, `cumulative_gas_used`, `log.transaction_index` fn filter_single_transaction( &self, l2_transaction: L2Transaction, @@ -199,8 +199,7 @@ impl Web3Indexer { let input = polyjuice_args.input.clone().unwrap_or_default(); // read logs - let tx_receipt: TxReceipt = - self.get_transaction_receipt(gw_tx_hash, block_number, mock_tx_index)?; + let tx_receipt: TxReceipt = self.get_transaction_receipt(gw_tx_hash, block_number)?; let log_item_vec = tx_receipt.logs(); // read polyjuice system log @@ -350,7 +349,7 @@ impl Web3Indexer { let nonce: u32 = l2_transaction.raw().nonce().unpack(); let tx_receipt: TxReceipt = - self.get_transaction_receipt(gw_tx_hash, block_number, mock_tx_index)?; + self.get_transaction_receipt(gw_tx_hash, block_number)?; let exit_code: u8 = tx_receipt.exit_code().into(); let web3_transaction = Web3Transaction::new( @@ -478,7 +477,17 @@ impl Web3Indexer { .flatten() .enumerate() .map(|(tx_index, mut tx)| { - tx.tx.transaction_index = tx_index as u32 + tx_index_cursor; + let transaction_index = tx_index as u32 + tx_index_cursor; + tx.tx.transaction_index = transaction_index; + // update log.transaction_index too + tx.logs = tx + .logs + .into_iter() + .map(|mut log| { + log.transaction_index = transaction_index; + log + }) + .collect(); cumulative_gas_used += tx.tx.gas_used; tx.tx.cumulative_gas_used = cumulative_gas_used; @@ -514,7 +523,6 @@ impl Web3Indexer { &self, gw_tx_hash: gw_common::H256, block_number: u64, - tx_index: u32, ) -> Result { let tx_hash = ckb_types::H256::from_slice(gw_tx_hash.as_slice())?; let tx_hash_hex = hex(tx_hash.as_bytes()) @@ -526,10 +534,9 @@ impl Web3Indexer { .get_transaction_receipt(&tx_hash)? .ok_or_else(|| { anyhow!( - "tx receipt not found by tx_hash: ({}) of block: {}, index: {}", + "tx receipt not found by tx_hash: ({}) of block: {}", tx_hash_hex, block_number, - tx_index ) })? .into(); From edfb4cde3e3f53f26fc03d1b47679af33e0ddb35 Mon Sep 17 00:00:00 2001 From: classicalliu Date: Fri, 19 Aug 2022 16:57:34 +0800 Subject: [PATCH 2/2] fix: Add cli to fix logs.transaction_index --- packages/api-server/cli/README.md | 21 +++- .../cli/fix-log-transaction-index.ts | 95 +++++++++++++++++++ packages/api-server/cli/index.ts | 20 ++++ 3 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 packages/api-server/cli/fix-log-transaction-index.ts diff --git a/packages/api-server/cli/README.md b/packages/api-server/cli/README.md index 2d0baaff..0bab8063 100644 --- a/packages/api-server/cli/README.md +++ b/packages/api-server/cli/README.md @@ -1,6 +1,6 @@ # CLI -### Fix wrong data +### Fix wrong transaction.eth_tx_hash This problem fixed in `v1.5.1-rc1`, so data indexed by version `>= v1.5.1-rc1` is ok. @@ -20,3 +20,22 @@ yarn run cli list-wrong-eth-tx-hashes --help // for more info yarn run cli fix-eth-tx-hash -d -c yarn run cli fix-eth-tx-hash --help // for more info ``` + +### Fix wrong log.transaction_index + +Run `wrong-log-transaction-index-count` to see how many wrong data in database. + +`log.transaction_index` always be zero when indexed after version `v1.6.0-rc1`. + +Run once after `web3-indexer` updated is enough. + +``` +// Get count, database-url can also read from env +yarn run cli wrong-log-transaction-index-count -d +yarn run cli wrong-log-transaction-index-count --help // for more info + +// Fix wrong data +// database-url can also read from env +yarn run cli fix-log-transaction-index -d +yarn run cli fix-log-transaction-index --help // for more info +``` diff --git a/packages/api-server/cli/fix-log-transaction-index.ts b/packages/api-server/cli/fix-log-transaction-index.ts new file mode 100644 index 00000000..79bf35bc --- /dev/null +++ b/packages/api-server/cli/fix-log-transaction-index.ts @@ -0,0 +1,95 @@ +import Knex, { Knex as KnexType } from "knex"; +import { DBLog } from "../src/db/types"; +import commander from "commander"; +import dotenv from "dotenv"; + +export async function fixLogTransactionIndexRun(program: commander.Command) { + try { + let databaseUrl = (program as any).databaseUrl; + if (databaseUrl == null) { + dotenv.config({ path: "./.env" }); + databaseUrl = process.env.DATABASE_URL; + } + + if (databaseUrl == null) { + throw new Error("Please provide --database-url"); + } + + await fixLogTransactionIndex(databaseUrl); + process.exit(0); + } catch (e) { + console.error(e); + process.exit(1); + } +} + +export async function wrongLogTransactionIndexCountRun( + program: commander.Command +): Promise { + try { + let databaseUrl = (program as any).databaseUrl; + if (databaseUrl == null) { + dotenv.config({ path: "./.env" }); + databaseUrl = process.env.DATABASE_URL; + } + if (databaseUrl == null) { + throw new Error("Please provide --database-url"); + } + await wrongLogTransactionIndexCount(databaseUrl); + process.exit(0); + } catch (e) { + console.error(e); + process.exit(1); + } +} + +// fix for leading zeros +export async function fixLogTransactionIndex( + databaseUrl: string +): Promise { + const knex = getKnex(databaseUrl); + + const query = knex("logs") + .whereRaw( + "transaction_index <> (select transaction_index from transactions where hash = logs.transaction_hash)" + ) + .count(); + const sql = query.toSQL(); + console.log("Query SQL:", sql.sql); + const wrongLogsCount = await query; + console.log(`Found ${wrongLogsCount[0].count} wrong logs`); + + const _updateQuery = await knex.raw( + "update logs set transaction_index = subquery.transaction_index from (select transaction_index, hash from transactions) as subquery where logs.transaction_hash = subquery.hash and logs.transaction_index <> subquery.transaction_index;" + ); + + console.log(`All logs updated!`); +} + +async function wrongLogTransactionIndexCount( + databaseUrl: string +): Promise { + const knex = getKnex(databaseUrl); + + const query = knex("logs") + .whereRaw( + "transaction_index <> (select transaction_index from transactions where hash = logs.transaction_hash)" + ) + .count(); + const sql = query.toSQL(); + console.log("Query SQL:", sql.sql); + const logsCount = await query; + console.log(`Found ${logsCount[0].count} wrong logs`); +} + +function getKnex(databaseUrl: string): KnexType { + const knex = Knex({ + client: "postgresql", + connection: { + connectionString: databaseUrl, + keepAlive: true, + }, + pool: { min: 2, max: 20 }, + }); + return knex; +} diff --git a/packages/api-server/cli/index.ts b/packages/api-server/cli/index.ts index c1d102b3..d890fa02 100644 --- a/packages/api-server/cli/index.ts +++ b/packages/api-server/cli/index.ts @@ -1,6 +1,10 @@ import { Command } from "commander"; import { fixEthTxHashRun, listWrongEthTxHashesRun } from "./fix-eth-tx-hash"; import { version as packageVersion } from "../package.json"; +import { + fixLogTransactionIndexRun, + wrongLogTransactionIndexCountRun, +} from "./fix-log-transaction-index"; const program = new Command(); program.version(packageVersion); @@ -29,4 +33,20 @@ program .option("-d, --database-url ", "database url", undefined) .action(listWrongEthTxHashesRun); +program + .command("fix-log-transaction-index") + .description("Fix wrong log's transaction_index") + .option( + "-d, --database-url ", + "If not provide, will use env `DATABASE_URL`, throw error if not provided too", + undefined + ) + .action(fixLogTransactionIndexRun); + +program + .command("wrong-log-transaction-index-count") + .description("Get log's count which transaction_index is wrong") + .option("-d, --database-url ", "database url", undefined) + .action(wrongLogTransactionIndexCountRun); + program.parse();