diff --git a/Cargo.lock b/Cargo.lock index 589bae6ef79..2715290dd16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -380,6 +380,7 @@ name = "benchmarks-module" version = "0.1.0" dependencies = [ "anyhow", + "fake", "spacetimedb", ] @@ -1403,6 +1404,12 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "deunicode" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339544cc9e2c4dc3fc7149fd630c5f22263a4fdf18a98afd0075784968b5cf00" + [[package]] name = "dialoguer" version = "0.11.0" @@ -1670,6 +1677,16 @@ dependencies = [ "serde", ] +[[package]] +name = "fake" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef603df4ba9adbca6a332db7da6f614f21eafefbaf8e087844e452fdec152d0" +dependencies = [ + "deunicode", + "rand 0.8.5", +] + [[package]] name = "fallible-iterator" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 3d81c1b29e6..4b125e307dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -157,6 +157,7 @@ enum-as-inner = "0.6" enum-map = "2.6.3" env_logger = "0.10" ethnum = { version = "1.5.0", features = ["serde"] } +fake = "3.0.1" flate2 = "1.0.24" foldhash = "0.1.4" fs-err = "2.9.0" diff --git a/crates/bench/load.py b/crates/bench/load.py new file mode 100644 index 00000000000..aceb18d6c15 --- /dev/null +++ b/crates/bench/load.py @@ -0,0 +1,109 @@ +# Mini-tool for executing load testing and call reducer functions. +import argparse +import subprocess +import sys +import time +from datetime import datetime, timedelta + + +class ProgressBar: + def __init__(self, total: int, label: str, size: int): + """ + Initialize the progress bar. + + Args: + total (int): The total number of steps/items. + label (str): Label for the progress bar. + size (int): The width of the progress bar. + """ + self.total = total + self.label = label + self.size = size + self.current = 0 + self.suffix = "" + + def show(self): + progress = int(self.size * self.current / self.total) + bar = "█" * progress + "." * (self.size - progress) + print(f"{self.label} {bar} {self.current}/{self.total} {self.suffix}", end="\r", flush=True) + + def step(self, steps: int = 1): + self.current = min(self.current + steps, self.total) + self.show() + + def finish(self): + self.current = self.total + self.show() + print() + + +def _run(progress: ProgressBar, title: str, cli: str, database: str, cmd: list): + for reducer in cmd: + progress.label = title + progress.suffix = f' {reducer}' + progress.show() + subprocess.check_call(f'{cli} call {database} {reducer}', shell=True) + progress.step() + + +def run(cli: str, database: str, init: list, load: list, frequency: float, duration: float): + print(f'Running load testing for database: {database}') + print(f" Frequency: {frequency} calls/second, Duration: {duration} seconds") + print() + if init: + progress = ProgressBar(len(init), label="Processing...", size=20) + _run(progress, f'Init reducers for database: {database}', cli, database, init) + progress.finish() + + if load: + start_time = datetime.now() + end_time = start_time + timedelta(seconds=duration) + current_time = start_time + interval = 1.0 / frequency + + progress = ProgressBar(int(duration * frequency) * len(load), label="Processing...", size=20) + while current_time < end_time: + _run(progress, f'Load reducers for database: {database}', cli, database, load) + time.sleep(interval) + current_time = datetime.now() + progress.finish() + + print(f'Load testing for database: {database} finished.') + + +if __name__ == '__main__': + """ + Usage: + python load.py -d -i -l [--no-cli] [-f ] [-s ] + + Example: + python load.py -d quickstart -f 2 -s 10 -i "insert_bulk_small_rows 100" -l "queries 'small, inserts:10,query:10,deletes:10';" + """ + parser = argparse.ArgumentParser() + + parser.add_argument('-d', '--database', type=str, help='Database name', required=True) + parser.add_argument('-i', '--init', type=str, help='Init reducers, separated by ;') + parser.add_argument('-l', '--load', type=str, help='Load reducers, separated by ;') + parser.add_argument('-f', '--frequency', type=float, default=1.0, + help="Frequency (calls per second)") + parser.add_argument('-s', '--seconds', type=float, default=1.0, help="Duration (in seconds)") + parser.add_argument('--no-cli', action='store_false', dest='cli', + help='Disable spacetime-cli if true, run `cargo run...` instead', + default=True) + + args = vars(parser.parse_args()) + + database = args['database'] + cli = args['cli'] + frequency = args['frequency'] + duration = args['seconds'] + + init = [x.strip() for x in (args['init'] or '').split(';') if x.strip()] + load = [x.strip() for x in (args['load'] or '').split(';') if x.strip()] + + if cli: + cli = '/Users/mamcx/.cargo/bin/spacetimedb-cli' + else: + cli = 'cargo run -p spacetimedb-cli --bin spacetimedb-cli' + + run(cli, database, init, load, frequency, duration) diff --git a/modules/benchmarks/Cargo.toml b/modules/benchmarks/Cargo.toml index f545a81bea4..acdbf9ac356 100644 --- a/modules/benchmarks/Cargo.toml +++ b/modules/benchmarks/Cargo.toml @@ -12,3 +12,4 @@ crate-type = ["cdylib"] spacetimedb = { path = "../../crates/bindings" } anyhow.workspace = true +fake.workspace = true \ No newline at end of file diff --git a/modules/benchmarks/src/synthetic.rs b/modules/benchmarks/src/synthetic.rs index 10f2999c8d7..282063d7996 100644 --- a/modules/benchmarks/src/synthetic.rs +++ b/modules/benchmarks/src/synthetic.rs @@ -23,9 +23,16 @@ //! Obviously more could be added... #![allow(non_camel_case_types)] #![allow(clippy::too_many_arguments)] -use spacetimedb::{log, ReducerContext, Table}; +use fake::faker::address::raw::{CityName, CountryName, StateName, ZipCode}; +use fake::faker::internet::raw::{Password, SafeEmail}; +use fake::faker::lorem::raw::{Paragraph, Words}; +use fake::faker::name::raw::*; +use fake::faker::phone_number::raw::CellNumber; +use fake::locales::EN; +use fake::{Fake, Faker}; +use spacetimedb::rand::Rng; +use spacetimedb::{log, ConnectionId, Identity, ReducerContext, SpacetimeType, StdbRng, Table}; use std::hint::black_box; - // ---------- schemas ---------- #[spacetimedb::table(name = unique_0_u32_u64_str)] @@ -78,6 +85,151 @@ pub struct btree_each_column_u32_u64_u64_t { y: u64, } +// Tables for data generation loading tests + +#[derive(SpacetimeType)] +pub enum Load { + Tiny, + Small, + Medium, + Large, +} + +#[derive(SpacetimeType)] +pub enum Index { + One, + Many, +} + +#[spacetimedb::table(name = tiny_rows)] +pub struct tiny_rows_t { + #[index(btree)] + id: u8, +} + +#[spacetimedb::table(name = small_rows)] +pub struct small_rows_t { + #[index(btree)] + id: u64, + x: u64, + y: u64, +} + +#[spacetimedb::table(name = small_btree_each_column_rows)] +pub struct small_rows_btree_each_column_t { + #[index(btree)] + id: u64, + #[index(btree)] + x: u64, + #[index(btree)] + y: u64, +} + +#[spacetimedb::table(name = medium_var_rows)] +pub struct medium_var_rows_t { + #[index(btree)] + id: u64, + name: String, + email: String, + password: String, + identity: Identity, + connection: ConnectionId, + pos: Vec, +} + +#[spacetimedb::table(name = medium_var_rows_btree_each_column)] +pub struct medium_var_rows_btree_each_column_t { + #[index(btree)] + id: u64, + #[index(btree)] + name: String, + #[index(btree)] + email: String, + #[index(btree)] + password: String, + #[index(btree)] + identity: Identity, + #[index(btree)] + connection: ConnectionId, + #[index(btree)] + pos: Vec, +} + +#[spacetimedb::table(name = large_var_rows)] +pub struct large_var_rows_t { + #[index(btree)] + id: u128, + invoice_code: String, + status: String, + customer: Identity, + company: ConnectionId, + user_name: String, + + price: f64, + cost: f64, + discount: f64, + taxes: Vec, + tax_total: f64, + sub_total: f64, + total: f64, + + country: String, + state: String, + city: String, + zip_code: Option, + phone: String, + + notes: String, + tags: Option>, +} + +#[spacetimedb::table(name = large_var_rows_btree_each_column)] +pub struct large_var_rows_btree_each_column_t { + #[index(btree)] + id: u128, + #[index(btree)] + invoice_code: String, + #[index(btree)] + status: String, + #[index(btree)] + customer: Identity, + #[index(btree)] + company: ConnectionId, + #[index(btree)] + user_name: String, + + #[index(btree)] + price: f64, + #[index(btree)] + cost: f64, + #[index(btree)] + discount: f64, + #[index(btree)] + taxes: Vec, + #[index(btree)] + tax_total: f64, + #[index(btree)] + sub_total: f64, + #[index(btree)] + total: f64, + + #[index(btree)] + country: String, + #[index(btree)] + state: String, + #[index(btree)] + city: String, + #[index(btree)] + zip_code: Option, + #[index(btree)] + phone: String, + + #[index(btree)] + notes: String, + #[index(btree)] + tags: Option>, +} + // ---------- empty ---------- #[spacetimedb::reducer] @@ -170,6 +322,296 @@ pub fn insert_bulk_btree_each_column_u32_u64_str(ctx: &ReducerContext, people: V } } +fn rand_connection_id(rng: &mut &StdbRng) -> ConnectionId { + ConnectionId::from(Faker.fake_with_rng::(rng)) +} + +fn rand_identity(rng: &mut &StdbRng) -> Identity { + Identity::from_u256(Faker.fake_with_rng::(rng).into()) +} + +#[spacetimedb::reducer] +pub fn insert_bulk_tiny_rows(ctx: &ReducerContext, rows: u8) { + for id in 0..rows { + ctx.db.tiny_rows().insert(tiny_rows_t { id }); + } + log::info!("Inserted on tiny_rows: {} rows", rows); +} + +#[spacetimedb::reducer] +pub fn insert_bulk_small_rows(ctx: &ReducerContext, rows: u64) { + let mut rng = ctx.rng(); + for id in 0..rows { + ctx.db.small_rows().insert(small_rows_t { + id, + x: rng.gen(), + y: rng.gen(), + }); + } + log::info!("Inserted on small_rows: {} rows", rows); +} + +#[spacetimedb::reducer] +pub fn insert_bulk_small_btree_each_column_rows(ctx: &ReducerContext, rows: u64) { + let mut rng = ctx.rng(); + for id in 0..rows { + ctx.db + .small_btree_each_column_rows() + .insert(small_rows_btree_each_column_t { + id, + x: rng.gen(), + y: rng.gen(), + }); + } + log::info!("Inserted on small_btree_each_column_rows: {} rows", rows); +} + +#[spacetimedb::reducer] +pub fn insert_bulk_medium_var_rows(ctx: &ReducerContext, rows: u64) { + let mut rng = ctx.rng(); + for id in 0..rows { + ctx.db.medium_var_rows().insert(medium_var_rows_t { + id, + name: Name(EN).fake_with_rng(&mut rng), + email: SafeEmail(EN).fake_with_rng(&mut rng), + password: Password(EN, 6..10).fake_with_rng(&mut rng), + identity: rand_identity(&mut rng), + connection: rand_connection_id(&mut rng), + pos: Faker.fake_with_rng(&mut rng), + }); + } + log::info!("Inserted on medium_var_rows: {} rows", rows); +} + +#[spacetimedb::reducer] +pub fn insert_bulk_medium_var_rows_btree_each_column(ctx: &ReducerContext, rows: u64) { + let mut rng = ctx.rng(); + for id in 0..rows { + ctx.db + .medium_var_rows_btree_each_column() + .insert(medium_var_rows_btree_each_column_t { + id, + name: Name(EN).fake_with_rng(&mut rng), + email: SafeEmail(EN).fake_with_rng(&mut rng), + password: Password(EN, 6..10).fake_with_rng(&mut rng), + identity: rand_identity(&mut rng), + connection: rand_connection_id(&mut rng), + pos: Faker.fake_with_rng(&mut rng), + }); + } + log::info!("Inserted on medium_var_rows_btree_each_column: {} rows", rows); +} + +#[spacetimedb::reducer] +pub fn insert_bulk_large_var_rows(ctx: &ReducerContext, rows: u64) { + let mut rng = ctx.rng(); + for id in 0..(rows as u128) { + ctx.db.large_var_rows().insert(large_var_rows_t { + id, + invoice_code: Faker.fake_with_rng(&mut rng), + status: Faker.fake_with_rng(&mut rng), + customer: rand_identity(&mut rng), + company: rand_connection_id(&mut rng), + user_name: Faker.fake_with_rng(&mut rng), + + price: Faker.fake_with_rng(&mut rng), + cost: Faker.fake_with_rng(&mut rng), + discount: Faker.fake_with_rng(&mut rng), + taxes: Faker.fake_with_rng(&mut rng), + tax_total: Faker.fake_with_rng(&mut rng), + sub_total: Faker.fake_with_rng(&mut rng), + total: Faker.fake_with_rng(&mut rng), + + country: CountryName(EN).fake_with_rng(&mut rng), + state: StateName(EN).fake_with_rng(&mut rng), + city: CityName(EN).fake_with_rng(&mut rng), + zip_code: ZipCode(EN).fake_with_rng(&mut rng), + phone: CellNumber(EN).fake_with_rng(&mut rng), + + notes: Paragraph(EN, 0..3).fake_with_rng(&mut rng), + tags: Words(EN, 0..3).fake_with_rng(&mut rng), + }); + } + log::info!("Inserted on large_var_rows: {} rows", rows); +} + +#[spacetimedb::reducer] +pub fn insert_bulk_large_var_rows_btree_each_column(ctx: &ReducerContext, rows: u64) { + let mut rng = ctx.rng(); + for id in 0..(rows as u128) { + ctx.db + .large_var_rows_btree_each_column() + .insert(large_var_rows_btree_each_column_t { + id, + invoice_code: Faker.fake_with_rng(&mut rng), + status: Faker.fake_with_rng(&mut rng), + customer: rand_identity(&mut rng), + company: rand_connection_id(&mut rng), + user_name: Faker.fake_with_rng(&mut rng), + + price: Faker.fake_with_rng(&mut rng), + cost: Faker.fake_with_rng(&mut rng), + discount: Faker.fake_with_rng(&mut rng), + taxes: Faker.fake_with_rng(&mut rng), + tax_total: Faker.fake_with_rng(&mut rng), + sub_total: Faker.fake_with_rng(&mut rng), + total: Faker.fake_with_rng(&mut rng), + + country: CountryName(EN).fake_with_rng(&mut rng), + state: StateName(EN).fake_with_rng(&mut rng), + city: CityName(EN).fake_with_rng(&mut rng), + zip_code: ZipCode(EN).fake_with_rng(&mut rng), + phone: CellNumber(EN).fake_with_rng(&mut rng), + + notes: Paragraph(EN, 0..3).fake_with_rng(&mut rng), + tags: Words(EN, 0..3).fake_with_rng(&mut rng), + }); + } + log::info!("Inserted on large_var_rows_btree_each_column: {} rows", rows); +} + +/// This reducer is used to load synthetic data into the database for benchmarking purposes. +/// +/// The input is a string with the following format: +/// +/// `load_type`: [`Load`], `index_type`: [`Index`], `row_count`: `u32` +#[spacetimedb::reducer] +pub fn load(ctx: &ReducerContext, input: String) -> Result<(), String> { + let args = input.split(',').map(|x| x.trim().to_lowercase()).collect::>(); + if args.len() != 3 { + return Err(format!("Expected 3 arguments, got {}", args.len())); + } + let load = match args[0].as_str() { + "tiny" => Load::Tiny, + "small" => Load::Small, + "medium" => Load::Medium, + "large" => Load::Large, + x => { + return Err(format!( + "Invalid load type: '{x}', expected: tiny, small, medium, or large" + )) + } + }; + let index = match args[1].as_str() { + "one" => Index::One, + "many" => Index::Many, + x => return Err(format!("Invalid index type: '{x}', expected: one, or many")), + }; + let rows = args[2] + .parse::() + .map_err(|e| format!("Invalid row count: {}", e))?; + + match (load, index) { + (Load::Tiny, Index::One | Index::Many) => insert_bulk_tiny_rows(ctx, rows as u8), + (Load::Small, Index::One) => insert_bulk_small_rows(ctx, rows), + (Load::Small, Index::Many) => insert_bulk_small_btree_each_column_rows(ctx, rows), + (Load::Medium, Index::One) => insert_bulk_medium_var_rows(ctx, rows), + (Load::Medium, Index::Many) => insert_bulk_medium_var_rows_btree_each_column(ctx, rows), + (Load::Large, Index::One) => insert_bulk_large_var_rows(ctx, rows), + (Load::Large, Index::Many) => insert_bulk_large_var_rows_btree_each_column(ctx, rows), + } + + Ok(()) +} + +/// Used to execute a series of reducers in sequence for benchmarking purposes. +/// +/// The input is a string with the following format: +/// +/// `load_type`: [`Load`], `inserts`: `u32`, `query`: `u32`, `deletes`: `u32` +/// +/// The order of the `inserts`, `query`, and `deletes` can be changed and will be executed in that order. +#[spacetimedb::reducer] +pub fn queries(ctx: &ReducerContext, input: String) -> Result<(), String> { + let args = input.split(',').map(|x| x.trim().to_lowercase()).collect::>(); + if args.len() < 2 { + return Err(format!("Expected at least 2 arguments, got {}", args.len())); + } + let load = match args[0].as_str() { + "tiny" => Load::Tiny, + "small" => Load::Small, + "medium" => Load::Medium, + "large" => Load::Large, + x => { + return Err(format!( + "Invalid load type: '{x}', expected: tiny, small, medium, or large" + )) + } + }; + + let mut inserts = 0u64; + let mut queries = 0u64; + let mut deletes = 0u64; + + for arg in args.iter().skip(1) { + let parts = arg.split(':').map(|x| x.trim()).collect::>(); + if parts.len() != 2 { + return Err(format!("Invalid argument: '{arg}', expected: 'operation:count'")); + } + let count = parts[1].parse::().map_err(|e| format!("Invalid count: {}", e))?; + match parts[0] { + "inserts" => inserts = count, + "query" => queries = count, + "deletes" => deletes = count, + x => { + return Err(format!( + "Invalid operation: '{x}', expected: inserts, query, or deletes" + )) + } + } + } + + log::info!("Executing queries: inserts: {inserts}, query: {queries}, deletes: {deletes}"); + // To allow to insert duplicate rows, the `ids` not use `[unique]` attribute, causing to not be able to use `update` method + match load { + Load::Tiny => { + if inserts > 0 { + insert_bulk_tiny_rows(ctx, inserts as u8); + } + for id in 0..queries { + filter_tiny_rows_by_id(ctx, id as u8); + } + for id in 0..deletes { + delete_tiny_rows_by_id(ctx, id as u8); + } + } + Load::Small => { + if inserts > 0 { + insert_bulk_small_rows(ctx, inserts); + } + for id in 0..queries { + filter_small_rows_by_id(ctx, id); + } + for id in 0..deletes { + delete_small_rows_by_id(ctx, id); + } + } + Load::Medium => { + if inserts > 0 { + insert_bulk_medium_var_rows(ctx, inserts); + } + for id in 0..queries { + filter_medium_var_rows_by_id(ctx, id); + } + for id in 0..deletes { + delete_medium_var_rows_by_id(ctx, id); + } + } + Load::Large => { + if inserts > 0 { + insert_bulk_large_var_rows(ctx, inserts); + } + for id in 0..queries { + filter_large_var_rows_by_id(ctx, id as u128); + } + for id in 0..deletes { + delete_large_var_rows_by_id(ctx, id as u128); + } + } + } + + Ok(()) +} // ---------- update ---------- #[spacetimedb::reducer] @@ -322,6 +764,55 @@ pub fn filter_btree_each_column_u32_u64_u64_by_y(ctx: &ReducerContext, y: u64) { } } +#[spacetimedb::reducer] +pub fn filter_tiny_rows_by_id(ctx: &ReducerContext, id: u8) { + for row in ctx.db.tiny_rows().iter().filter(|r| r.id == id) { + black_box(row); + } +} + +#[spacetimedb::reducer] +pub fn filter_small_rows_by_id(ctx: &ReducerContext, id: u64) { + for row in ctx.db.small_rows().iter().filter(|r| r.id == id) { + black_box(row); + } +} + +#[spacetimedb::reducer] +pub fn filter_small_btree_each_column_rows_by_id(ctx: &ReducerContext, id: u64) { + for row in ctx.db.small_btree_each_column_rows().iter().filter(|r| r.id == id) { + black_box(row); + } +} + +#[spacetimedb::reducer] +pub fn filter_medium_var_rows_by_id(ctx: &ReducerContext, id: u64) { + for row in ctx.db.medium_var_rows().iter().filter(|r| r.id == id) { + black_box(row); + } +} + +#[spacetimedb::reducer] +pub fn filter_medium_var_rows_btree_each_column_by_id(ctx: &ReducerContext, id: u64) { + for row in ctx.db.medium_var_rows_btree_each_column().iter().filter(|r| r.id == id) { + black_box(row); + } +} + +#[spacetimedb::reducer] +pub fn filter_large_var_rows_by_id(ctx: &ReducerContext, id: u128) { + for row in ctx.db.large_var_rows().iter().filter(|r| r.id == id) { + black_box(row); + } +} + +#[spacetimedb::reducer] +pub fn filter_large_var_rows_btree_each_column_by_id(ctx: &ReducerContext, id: u128) { + for row in ctx.db.large_var_rows_btree_each_column().iter().filter(|r| r.id == id) { + black_box(row); + } +} + // ---------- delete ---------- // FIXME: current nonunique delete interface is UNUSABLE!!!! @@ -336,6 +827,41 @@ pub fn delete_unique_0_u32_u64_u64_by_id(ctx: &ReducerContext, id: u32) { ctx.db.unique_0_u32_u64_u64().id().delete(id); } +#[spacetimedb::reducer] +pub fn delete_tiny_rows_by_id(ctx: &ReducerContext, id: u8) { + ctx.db.tiny_rows().id().delete(id); +} + +#[spacetimedb::reducer] +pub fn delete_small_rows_by_id(ctx: &ReducerContext, id: u64) { + ctx.db.small_rows().id().delete(id); +} + +#[spacetimedb::reducer] +pub fn delete_small_btree_each_column_rows_by_id(ctx: &ReducerContext, id: u64) { + ctx.db.small_btree_each_column_rows().id().delete(id); +} + +#[spacetimedb::reducer] +pub fn delete_medium_var_rows_by_id(ctx: &ReducerContext, id: u64) { + ctx.db.medium_var_rows().id().delete(id); +} + +#[spacetimedb::reducer] +pub fn delete_medium_var_rows_btree_each_column_by_id(ctx: &ReducerContext, id: u64) { + ctx.db.medium_var_rows_btree_each_column().id().delete(id); +} + +#[spacetimedb::reducer] +pub fn delete_large_var_rows_by_id(ctx: &ReducerContext, id: u128) { + ctx.db.large_var_rows().id().delete(id); +} + +#[spacetimedb::reducer] +pub fn delete_large_var_rows_btree_each_column_by_id(ctx: &ReducerContext, id: u128) { + ctx.db.large_var_rows_btree_each_column().id().delete(id); +} + // ---------- clear table ---------- #[spacetimedb::reducer] pub fn clear_table_unique_0_u32_u64_str(_ctx: &ReducerContext) {