From 4c9e58a8a9741c8de7bb6f4cb8aff14c5b94f814 Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Wed, 15 Sep 2021 15:27:03 +0000 Subject: [PATCH] Updates for `pprof`. --- Cargo.lock | 257 ++++++++++++++++++++++++++++++++-- Cargo.toml | 1 + config/default.yaml | 2 +- examples/js/stress.ts | 24 ++-- src/matchengine/controller.rs | 4 +- src/matchengine/market/mod.rs | 23 +++ 6 files changed, 282 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d3f22731..7e048e05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -277,6 +277,15 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" +[[package]] +name = "arrayvec" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" +dependencies = [ + "nodrop", +] + [[package]] name = "arrayvec" version = "0.5.2" @@ -324,6 +333,17 @@ dependencies = [ "num-traits 0.2.14", ] +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.0.1" @@ -470,6 +490,12 @@ version = "3.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" +[[package]] +name = "bytemuck" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72957246c41db82b8ef88a5486143830adeb8227ef9837740bdec67724cf2c5b" + [[package]] name = "byteorder" version = "1.4.2" @@ -579,6 +605,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "cpp_demangle" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea47428dc9d2237f3c6bc134472edfd63ebba0af932e783506dcfd66f10d18a" +dependencies = [ + "cfg-if", +] + [[package]] name = "cpuid-bool" version = "0.1.2" @@ -650,6 +685,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "debugid" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91cf5a8c2f2097e2a32627123508635d47ce10563d999ec1a95addf08b502ba" +dependencies = [ + "uuid", +] + [[package]] name = "derivative" version = "2.2.0" @@ -704,13 +748,14 @@ dependencies = [ "humantime", "humantime-serde", "hyper", - "itertools", + "itertools 0.10.0", "lazy_static", "log", "nix", "num_enum", "orchestra", "paperclip", + "pprof", "qstring", "rand 0.8.3", "serde 1.0.124", @@ -1171,6 +1216,24 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inferno" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa5bd9a10b38bf5f3c670f9d75c194adbecd2b1573f737668ab8599f41edc87" +dependencies = [ + "ahash 0.7.2", + "atty", + "indexmap", + "itoa", + "lazy_static", + "log", + "num-format", + "quick-xml", + "rgb", + "str_stack", +] + [[package]] name = "instant" version = "0.1.9" @@ -1180,6 +1243,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.10.0" @@ -1222,7 +1294,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21f866863575d0e1d654fbeeabdc927292fdf862873dc3c96c6f753357e13374" dependencies = [ - "arrayvec", + "arrayvec 0.5.2", "bitflags", "cfg-if", "ryu", @@ -1326,6 +1398,16 @@ version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +[[package]] +name = "memmap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "mime" version = "0.3.16" @@ -1383,6 +1465,12 @@ dependencies = [ "libc", ] +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" + [[package]] name = "nom" version = "5.1.2" @@ -1462,6 +1550,16 @@ dependencies = [ "num-traits 0.2.14", ] +[[package]] +name = "num-format" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465" +dependencies = [ + "arrayvec 0.4.12", + "itoa", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -1568,7 +1666,7 @@ name = "orchestra" version = "0.1.0" source = "git+https://github.com/fluidex/orchestra.git?branch=master#3938acc11e605cb381b62467a9b9cd23abed86b7" dependencies = [ - "prost", + "prost 0.8.0", "serde 1.0.124", "tonic", "tonic-build", @@ -1580,7 +1678,7 @@ version = "0.5.0" source = "git+https://github.com/fluidex/paperclip.git#374e30b0bbef538f536f896999313c03f995a69a" dependencies = [ "anyhow", - "itertools", + "itertools 0.10.0", "once_cell", "paperclip-actix", "paperclip-core", @@ -1796,6 +1894,27 @@ dependencies = [ "postgres-protocol", ] +[[package]] +name = "pprof" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c7600124d694d855283caf9f333befe2abce090833bb638009aeddd9e156dee" +dependencies = [ + "backtrace", + "inferno", + "lazy_static", + "libc", + "log", + "nix", + "parking_lot", + "prost 0.7.0", + "prost-build 0.7.0", + "prost-derive 0.7.0", + "symbolic-demangle", + "tempfile", + "thiserror", +] + [[package]] name = "ppv-lite86" version = "0.2.10" @@ -1856,6 +1975,16 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "prost" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e6984d2f1a23009bd270b8bb56d0926810a3d483f59c987d77969e9d8e840b2" +dependencies = [ + "bytes", + "prost-derive 0.7.0", +] + [[package]] name = "prost" version = "0.8.0" @@ -1863,7 +1992,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.8.0", +] + +[[package]] +name = "prost-build" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32d3ebd75ac2679c2af3a92246639f9fcc8a442ee420719cc4fe195b98dd5fa3" +dependencies = [ + "bytes", + "heck", + "itertools 0.9.0", + "log", + "multimap", + "petgraph", + "prost 0.7.0", + "prost-types 0.7.0", + "tempfile", + "which", ] [[package]] @@ -1874,16 +2021,29 @@ checksum = "355f634b43cdd80724ee7848f95770e7e70eefa6dcf14fea676216573b8fd603" dependencies = [ "bytes", "heck", - "itertools", + "itertools 0.10.0", "log", "multimap", "petgraph", - "prost", - "prost-types", + "prost 0.8.0", + "prost-types 0.8.0", "tempfile", "which", ] +[[package]] +name = "prost-derive" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "169a15f3008ecb5160cba7d37bcd690a7601b6d30cfb87a117d45e59d52af5d4" +dependencies = [ + "anyhow", + "itertools 0.9.0", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "prost-derive" version = "0.8.0" @@ -1891,12 +2051,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "600d2f334aa05acb02a755e217ef1ab6dea4d51b58b7846588b747edec04efba" dependencies = [ "anyhow", - "itertools", + "itertools 0.10.0", "proc-macro2", "quote", "syn", ] +[[package]] +name = "prost-types" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b518d7cdd93dab1d1122cf07fa9a60771836c668dde9d9e2a139f957f0d9f1bb" +dependencies = [ + "bytes", + "prost 0.7.0", +] + [[package]] name = "prost-types" version = "0.8.0" @@ -1904,7 +2074,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b" dependencies = [ "bytes", - "prost", + "prost 0.8.0", ] [[package]] @@ -1916,6 +2086,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "quick-xml" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8533f14c8382aaad0d592c812ac3b826162128b65662331e1127b45c3d18536b" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.9" @@ -2087,6 +2266,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "rgb" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fddb3b23626145d1776addfc307e1a1851f60ef6ca64f376bcb889697144cf0" +dependencies = [ + "bytemuck", +] + [[package]] name = "ring" version = "0.16.20" @@ -2114,7 +2302,7 @@ version = "1.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01127cb8617e5e21bcf2e19b5eb48317735ca677f1d0a94833c21c331c446582" dependencies = [ - "arrayvec", + "arrayvec 0.5.2", "byteorder", "bytes", "num-traits 0.2.14", @@ -2477,6 +2665,12 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "standback" version = "0.2.15" @@ -2541,6 +2735,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" +[[package]] +name = "str_stack" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" + [[package]] name = "stringprep" version = "0.1.2" @@ -2575,6 +2775,29 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" +[[package]] +name = "symbolic-common" +version = "8.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "348885c332e7d0784d661844b13b198464144a5ebcd3bfc047a6c441867ea467" +dependencies = [ + "debugid", + "memmap", + "stable_deref_trait", + "uuid", +] + +[[package]] +name = "symbolic-demangle" +version = "8.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6780c62bfbd609bffaa13d6959715850578aa43caaae7aee14f1f24ceb64f433" +dependencies = [ + "cpp_demangle", + "rustc-demangle", + "symbolic-common", +] + [[package]] name = "syn" version = "1.0.63" @@ -2836,8 +3059,8 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost", - "prost-derive", + "prost 0.8.0", + "prost-derive 0.8.0", "tokio", "tokio-stream", "tokio-util", @@ -2855,7 +3078,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12b52d07035516c2b74337d2ac7746075e7dcae7643816c1b12c5ff8a7484c08" dependencies = [ "proc-macro2", - "prost-build", + "prost-build 0.8.0", "quote", "syn", ] @@ -3064,6 +3287,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" + [[package]] name = "vcpkg" version = "0.2.11" diff --git a/Cargo.toml b/Cargo.toml index 5ebba245..7444cc95 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ tracing = "0.1" tracing-appender = "0.1" tracing-subscriber = "0.2" ttl_cache = "0.5.1" +pprof = { version = "0.4", features = [ "flamegraph", "protobuf" ] } [[bin]] name = "restapi" diff --git a/config/default.yaml b/config/default.yaml index 06525de3..5f805c43 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -3,4 +3,4 @@ slice_keeptime: 259200 disable_self_trade: true disable_market_order: true check_eddsa_signatue: auto -user_order_num_limit: 800000 +user_order_num_limit: 90000000 diff --git a/examples/js/stress.ts b/examples/js/stress.ts index 4f1bb851..adf45014 100644 --- a/examples/js/stress.ts +++ b/examples/js/stress.ts @@ -10,7 +10,7 @@ import { putRandOrder, sleep, decimalAdd, - decimalEqual, + decimalEqual } from "./util"; async function stressTest({ parallel, interval, repeat }) { @@ -20,10 +20,10 @@ async function stressTest({ parallel, interval, repeat }) { console.log(await client.orderCancelAll(userId2, market)); await depositAssets({ USDT: "10000000", ETH: "10000" }, userId1); await depositAssets({ USDT: "10000000", ETH: "10000" }, userId2); - const USDTBefore1 = await client.balanceQueryByAsset(userId1, "USDT"); - const ETHBefore1 = await client.balanceQueryByAsset(userId1, "ETH"); - const USDTBefore2 = await client.balanceQueryByAsset(userId2, "USDT"); - const ETHBefore2 = await client.balanceQueryByAsset(userId2, "ETH"); + /* + const USDTBefore = await client.balanceQueryByAsset(userId, "USDT"); + const ETHBefore = await client.balanceQueryByAsset(userId, "ETH"); + */ await printBalance(); const startTime = Date.now(); function elapsedSecs() { @@ -55,18 +55,18 @@ async function stressTest({ parallel, interval, repeat }) { } const totalTime = elapsedSecs(); await printBalance(); - const USDTAfter1 = await client.balanceQueryByAsset(userId1, "USDT"); - const ETHAfter1 = await client.balanceQueryByAsset(userId1, "ETH"); - const USDTAfter2 = await client.balanceQueryByAsset(userId2, "USDT"); - const ETHAfter2 = await client.balanceQueryByAsset(userId2, "ETH"); - /* TODO: Needs to validate the all balances for each asset. + /* + const USDTAfter = await client.balanceQueryByAsset(userId, "USDT"); + const ETHAfter = await client.balanceQueryByAsset(userId, "ETH"); decimalEqual(USDTAfter.available, USDTBefore.available); decimalEqual(USDTAfter.frozen, USDTBefore.frozen); decimalEqual(USDTAfter.total, USDTBefore.total); */ + const result = await client.marketSummary(market); + console.log(result); const tradeCountAfter = (await client.marketSummary(market)).trade_count; - console.log(tradeCountBefore); console.log(tradeCountAfter); + console.log(tradeCountBefore); console.log("avg orders/s:", (parallel * repeat) / totalTime); console.log( "avg trades/s:", @@ -77,7 +77,7 @@ async function stressTest({ parallel, interval, repeat }) { async function main() { try { - await stressTest({ parallel: 120, interval: 100, repeat: 230 }); + await stressTest({ parallel: 150, interval: 100, repeat: 150 }); // await stressTest({ parallel: 1, interval: 500, repeat: 0 }); } catch (error) { console.error("Caught error:", error); diff --git a/src/matchengine/controller.rs b/src/matchengine/controller.rs index 0b6cf88e..5d417d79 100644 --- a/src/matchengine/controller.rs +++ b/src/matchengine/controller.rs @@ -45,8 +45,8 @@ impl OperationLogConsumer for OperationLogSender { // TODO: reuse pool of two dbs when they are same? fn create_persistor(settings: &config::Settings) -> Box { - let persist_to_mq = true; - let persist_to_mq_full_order = true; + let persist_to_mq = false; + let persist_to_mq_full_order = false; let persist_to_db = false; let persist_to_file = false; let mut persistor = Box::new(CompositePersistor::default()); diff --git a/src/matchengine/market/mod.rs b/src/matchengine/market/mod.rs index 5a559c13..cdce80f1 100644 --- a/src/matchengine/market/mod.rs +++ b/src/matchengine/market/mod.rs @@ -16,6 +16,9 @@ use fluidex_common::rust_decimal::{Decimal, RoundingStrategy}; use itertools::Itertools; use serde::{Deserialize, Serialize}; +use std::fs::File; +use {pprof::protos::Message, std::io::Write}; + pub use types::{OrderSide, OrderType}; mod order; @@ -45,6 +48,8 @@ pub struct Market { pub disable_self_trade: bool, pub disable_market_order: bool, pub check_eddsa_signatue: OrderSignatrueCheck, + + wrapped_profiler: Option>, } pub struct BalanceManagerWrapper<'a> { @@ -124,6 +129,7 @@ impl Market { disable_self_trade: global_settings.disable_self_trade, disable_market_order: global_settings.disable_market_order, check_eddsa_signatue: global_settings.check_eddsa_signatue, + wrapped_profiler: None, }; Ok(market) } @@ -308,6 +314,7 @@ impl Market { // TODO: find a more elegant way to handle this let mut need_cancel = false; + for maker_ref in counter_orders { // Step1: get ask and bid let mut maker = maker_ref.borrow_mut(); @@ -401,6 +408,22 @@ impl Market { }; #[cfg(feature = "emit_state_diff")] let state_before = Self::get_trade_state(ask_order, bid_order, balance_manager, &self.base, &self.quote); + + if self.trade_count % 10000 == 0 { + if let Some(profiler) = &self.wrapped_profiler { + if let Ok(report) = profiler.report().build() { + let mut file = File::create(format!("profile_trade_{}.pb", self.trade_count)).unwrap(); + let profile = report.pprof().unwrap(); + + let mut content = Vec::new(); + profile.encode(&mut content).unwrap(); + file.write_all(&content).unwrap(); + } + } + + self.wrapped_profiler = Some(pprof::ProfilerGuard::new(100).unwrap()); + } + self.trade_count += 1; if self.disable_self_trade { debug_assert_ne!(trade.ask_user_id, trade.bid_user_id);