From e22c6c9a4f8c79d8ee2475d10e18ddfbe0076a27 Mon Sep 17 00:00:00 2001 From: Jack May Date: Thu, 20 Sep 2018 23:34:28 -0700 Subject: [PATCH 1/7] Integration of native dynamic programs --- Cargo.toml | 23 +++- programs/move_funds/Cargo.toml | 24 ++++ programs/move_funds/src/lib.rs | 48 ++++++++ programs/noop/Cargo.toml | 22 ++++ programs/noop/src/lib.rs | 6 + programs/print/Cargo.toml | 22 ++++ programs/print/src/lib.rs | 9 ++ src/bank.rs | 39 ++++++- src/dynamic_program.rs | 204 +++++++++++++++++++++++++++++++++ src/lib.rs | 2 + src/system_program.rs | 95 +++++++++++++-- src/transaction.rs | 18 +++ 12 files changed, 498 insertions(+), 14 deletions(-) create mode 100644 programs/move_funds/Cargo.toml create mode 100644 programs/move_funds/src/lib.rs create mode 100644 programs/noop/Cargo.toml create mode 100644 programs/noop/src/lib.rs create mode 100644 programs/print/Cargo.toml create mode 100644 programs/print/src/lib.rs create mode 100644 src/dynamic_program.rs diff --git a/Cargo.toml b/Cargo.toml index d9138fc4daed84..c6514abe3438b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,6 +89,8 @@ jsonrpc-http-server = { git = "https://github.com/paritytech/jsonrpc", rev = "4b jsonrpc-macros = { git = "https://github.com/paritytech/jsonrpc", rev = "4b6060b" } ipnetwork = "0.12.7" itertools = "0.7.8" +libc = "0.2.43" +libloading = "0.5.0" log = "0.4.2" matches = "0.1.6" nix = "0.11.0" @@ -106,7 +108,11 @@ sys-info = "0.5.6" tokio = "0.1" tokio-codec = "0.1" untrusted = "0.6.2" -libc = "0.2.43" + +[dev-dependencies] +noop = { path = "programs/noop" } +print = { path = "programs/print" } +move_funds = { path = "programs/move_funds" } [[bench]] name = "bank" @@ -122,3 +128,18 @@ name = "signature" [[bench]] name = "sigverify" + +[workspace] +members = [ + ".", + "programs/noop", + "programs/print", + "programs/move_funds", +] +default-members = [ + ".", + "programs/noop", + "programs/print", + "programs/move_funds", +] + diff --git a/programs/move_funds/Cargo.toml b/programs/move_funds/Cargo.toml new file mode 100644 index 00000000000000..fd66239e04f691 --- /dev/null +++ b/programs/move_funds/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "move_funds" +version = "0.1.0" +authors = [ + "Anatoly Yakovenko ", + "Greg Fitzgerald ", + "Stephen Akridge ", + "Michael Vines ", + "Rob Walker ", + "Pankaj Garg ", + "Tyera Eulberg ", + "Jack May ", +] + +[dependencies] +bincode = "1.0.0" +generic-array = { version = "0.12.0", default-features = false, features = ["serde"] } +libloading = "0.5.0" +solana = { path = "../.." } + +[lib] +name = "move_funds" +crate-type = ["dylib"] + diff --git a/programs/move_funds/src/lib.rs b/programs/move_funds/src/lib.rs new file mode 100644 index 00000000000000..6efcd3c85ea1dc --- /dev/null +++ b/programs/move_funds/src/lib.rs @@ -0,0 +1,48 @@ +extern crate bincode; +extern crate solana; + +use bincode::deserialize; +use solana::dynamic_program::KeyedAccount; + +#[no_mangle] +pub extern "C" fn process(infos: &mut Vec, data: &[u8]) { + let tokens: i64 = deserialize(data).unwrap(); + if infos[0].account.tokens >= tokens { + infos[0].account.tokens -= tokens; + infos[1].account.tokens += tokens; + } else { + println!( + "Insufficient funds, asked {}, only had {}", + tokens, infos[0].account.tokens + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bincode::serialize; + use solana::bank::Account; + use solana::signature::Pubkey; + + #[test] + fn test_move_funds() { + let tokens: i64 = 100; + let data: Vec = serialize(&tokens).unwrap(); + let keys = vec![Pubkey::default(); 2]; + let mut accounts = vec![Account::default(), Account::default()]; + accounts[0].tokens = 100; + accounts[1].tokens = 1; + + { + let mut infos: Vec = Vec::new(); + for (key, account) in keys.iter().zip(&mut accounts).collect::>() { + infos.push(KeyedAccount { key, account }); + } + + process(&mut infos, &data); + } + assert_eq!(0, accounts[0].tokens); + assert_eq!(101, accounts[1].tokens); + } +} diff --git a/programs/noop/Cargo.toml b/programs/noop/Cargo.toml new file mode 100644 index 00000000000000..2cc53a8b3f9a37 --- /dev/null +++ b/programs/noop/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "noop" +version = "0.1.0" +authors = [ + "Anatoly Yakovenko ", + "Greg Fitzgerald ", + "Stephen Akridge ", + "Michael Vines ", + "Rob Walker ", + "Pankaj Garg ", + "Tyera Eulberg ", + "Jack May ", +] + +[dependencies] +libloading = "0.5.0" +solana = { path = "../.." } + +[lib] +name = "noop" +crate-type = ["dylib"] + diff --git a/programs/noop/src/lib.rs b/programs/noop/src/lib.rs new file mode 100644 index 00000000000000..470ee953abcdb2 --- /dev/null +++ b/programs/noop/src/lib.rs @@ -0,0 +1,6 @@ +extern crate solana; + +use solana::dynamic_program::KeyedAccount; + +#[no_mangle] +pub extern "C" fn process(_infos: &mut Vec, _data: &[u8]) {} diff --git a/programs/print/Cargo.toml b/programs/print/Cargo.toml new file mode 100644 index 00000000000000..25322b1c586e2d --- /dev/null +++ b/programs/print/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "print" +version = "0.1.0" +authors = [ + "Anatoly Yakovenko ", + "Greg Fitzgerald ", + "Stephen Akridge ", + "Michael Vines ", + "Rob Walker ", + "Pankaj Garg ", + "Tyera Eulberg ", + "Jack May ", +] + +[dependencies] +libloading = "0.5.0" +solana = { path = "../.." } + +[lib] +name = "print" +crate-type = ["dylib"] + diff --git a/programs/print/src/lib.rs b/programs/print/src/lib.rs new file mode 100644 index 00000000000000..f2315daaec3cf4 --- /dev/null +++ b/programs/print/src/lib.rs @@ -0,0 +1,9 @@ +extern crate solana; + +use solana::dynamic_program::KeyedAccount; + +#[no_mangle] +pub extern "C" fn process(infos: &mut Vec, _data: &[u8]) { + println!("AccountInfos: {:#?}", infos); + //println!("data: {:#?}", data); +} diff --git a/src/bank.rs b/src/bank.rs index 3645c953d3c362..119d173ac3f358 100644 --- a/src/bank.rs +++ b/src/bank.rs @@ -7,6 +7,7 @@ use bincode::deserialize; use bincode::serialize; use budget_program::BudgetState; use counter::Counter; +use dynamic_program::{DynamicProgram, KeyedAccount}; use entry::Entry; use hash::{hash, Hash}; use itertools::Itertools; @@ -136,6 +137,9 @@ pub struct Bank { // The latest finality time for the network finality_time: AtomicUsize, + + // loaded contracts hashed by program_id + loaded_contracts: RwLock>, } impl Default for Bank { @@ -147,6 +151,7 @@ impl Default for Bank { transaction_count: AtomicUsize::new(0), is_leader: true, finality_time: AtomicUsize::new(std::usize::MAX), + loaded_contracts: RwLock::new(HashMap::new()), } } } @@ -307,6 +312,7 @@ impl Bank { Ok(called_accounts) } } + fn load_accounts( &self, txs: &[Transaction], @@ -317,6 +323,7 @@ impl Bank { .map(|tx| self.load_account(tx, accounts, error_counters)) .collect() } + pub fn verify_transaction( tx: &Transaction, pre_program_id: &Pubkey, @@ -341,11 +348,33 @@ impl Bank { } Ok(()) } + + fn loaded_contract(&self, tx: &Transaction, accounts: &mut [Account]) -> bool { + let loaded_contracts = self.loaded_contracts.write().unwrap(); + match loaded_contracts.get(&tx.program_id) { + Some(dc) => { + let mut infos: Vec<_> = (&tx.keys) + .into_iter() + .zip(accounts) + .map(|(key, account)| KeyedAccount { key, account }) + .collect(); + + dc.call(&mut infos, &tx.userdata); + true + } + None => false, + } + } + /// Execute a transaction. /// This method calls the contract's process_transaction method and verifies that the result of /// the contract does not violate the bank's accounting rules. /// The accounts are committed back to the bank only if this function returns Ok(_). - fn execute_transaction(tx: Transaction, accounts: &mut [Account]) -> Result { + fn execute_transaction( + &self, + tx: Transaction, + accounts: &mut [Account], + ) -> Result { let pre_total: i64 = accounts.iter().map(|a| a.tokens).sum(); let pre_data: Vec<_> = accounts .iter_mut() @@ -355,13 +384,17 @@ impl Bank { // Call the contract method // It's up to the contract to implement its own rules on moving funds if SystemProgram::check_id(&tx.program_id) { - SystemProgram::process_transaction(&tx, accounts) + SystemProgram::process_transaction(&tx, accounts, &self.loaded_contracts) } else if BudgetState::check_id(&tx.program_id) { // TODO: the runtime should be checking read/write access to memory // we are trusting the hard coded contracts not to clobber or allocate BudgetState::process_transaction(&tx, accounts) +<<<<<<< HEAD } else if StorageProgram::check_id(&tx.program_id) { StorageProgram::process_transaction(&tx, accounts) +======= + } else if self.loaded_contract(&tx, accounts) { +>>>>>>> Integration of native dynamic programs } else { return Err(BankError::UnknownContractId(tx.program_id)); } @@ -418,7 +451,7 @@ impl Bank { .zip(txs.into_iter()) .map(|(acc, tx)| match acc { Err(e) => Err(e.clone()), - Ok(ref mut accounts) => Self::execute_transaction(tx, accounts), + Ok(ref mut accounts) => self.execute_transaction(tx, accounts), }).collect(); let execution_elapsed = now.elapsed(); let now = Instant::now(); diff --git a/src/dynamic_program.rs b/src/dynamic_program.rs new file mode 100644 index 00000000000000..322c2de9b095cf --- /dev/null +++ b/src/dynamic_program.rs @@ -0,0 +1,204 @@ +extern crate bincode; +extern crate generic_array; + +use bank::Account; +use libloading::{Library, Symbol}; +use signature::Pubkey; +use std::path::PathBuf; + +#[cfg(debug_assertions)] +const CARGO_PROFILE: &str = "debug"; + +#[cfg(not(debug_assertions))] +const CARGO_PROFILE: &str = "release"; + +/// Dynamic link library prefix +#[cfg(unix)] +const PLATFORM_FILE_PREFIX: &str = "lib"; +/// Dynamic link library prefix +#[cfg(windows)] +const PLATFORM_FILE_PREFIX: &str = ""; +/// Dynamic link library file extension specific to the platform +#[cfg(any(target_os = "macos", target_os = "ios"))] +const PLATFORM_FILE_EXTENSION: &str = "dylib"; +/// Dynamic link library file extension specific to the platform +#[cfg(all(unix, not(any(target_os = "macos", target_os = "ios"))))] +const PLATFORM_FILE_EXTENSION: &str = "so"; +/// Dynamic link library file extension specific to the platform +#[cfg(windows)] +const PLATFORM_FILE_EXTENSION: &str = "dll"; + +/// Creates a platform-specific file path +fn create_library_path(name: &str) -> PathBuf { + let mut path = PathBuf::new(); + path.push("target"); + path.push(CARGO_PROFILE); + path.push("deps"); + path.push(PLATFORM_FILE_PREFIX.to_string() + name); + path.set_extension(PLATFORM_FILE_EXTENSION); + path +} + +#[derive(Debug)] +pub struct KeyedAccount<'a> { + pub key: &'a Pubkey, + pub account: &'a mut Account, +} + +// All programs export a symbol named process() +const ENTRYPOINT: &str = "process"; +type Entrypoint = unsafe extern "C" fn(infos: &mut Vec, data: &[u8]); + +#[derive(Debug)] +pub enum DynamicProgram { + /// Native program + /// * Transaction::keys[0..] - program dependent + /// * name - name of the program, translated to a file path of the program module + /// * userdata - program specific user data + Native { name: String, library: Library }, + /// Bpf program + /// * Transaction::keys[0..] - program dependent + /// * TODO BPF specific stuff + /// * userdata - program specific user data + Bpf { userdata: Vec }, +} + +impl DynamicProgram { + pub fn new(name: String) -> Self { + // TODO determine what kind of module to load + // create native program + println!("loading {}", name); + let path = create_library_path(&name); + let library = Library::new(&path).expect("Failed to load library"); + DynamicProgram::Native { name, library } + } + + pub fn call(&self, infos: &mut Vec, data: &[u8]) { + match self { + DynamicProgram::Native { name, library } => unsafe { + let entrypoint: Symbol = match library.get(ENTRYPOINT.as_bytes()) { + Ok(s) => s, + Err(e) => panic!( + "{:?} Unable to find {:?} in program {}", + e, ENTRYPOINT, name + ), + }; + entrypoint(infos, data); + }, + DynamicProgram::Bpf { .. } => { + // TODO BPF + println!{"Bpf program not supported"} + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bank::Account; + use bincode::serialize; + use signature::Pubkey; + use std::path::Path; + + #[test] + fn test_create_library_path_1() { + let path = create_library_path("noop"); + assert_eq!(true, Path::new(&path).exists()); + let path = create_library_path("print"); + assert_eq!(true, Path::new(&path).exists()); + let path = create_library_path("move_funds"); + assert_eq!(true, Path::new(&path).exists()); + } + + #[test] + fn test_program_noop() { + let data: Vec = vec![0]; + let keys = vec![Pubkey::default(); 2]; + let mut accounts = vec![Account::default(), Account::default()]; + accounts[0].tokens = 100; + accounts[1].tokens = 1; + + { + let mut infos: Vec<_> = (&keys) + .into_iter() + .zip(&mut accounts) + .map(|(key, account)| KeyedAccount { key, account }) + .collect(); + + let dp = DynamicProgram::new("noop".to_string()); + dp.call(&mut infos, &data); + } + } + + #[test] + #[ignore] + fn test_program_print() { + let data: Vec = vec![0]; + let keys = vec![Pubkey::default(); 2]; + let mut accounts = vec![Account::default(), Account::default()]; + accounts[0].tokens = 100; + accounts[1].tokens = 1; + + { + let mut infos: Vec<_> = (&keys) + .into_iter() + .zip(&mut accounts) + .map(|(key, account)| KeyedAccount { key, account }) + .collect(); + + let dp = DynamicProgram::new("print".to_string()); + dp.call(&mut infos, &data); + } + } + + #[test] + fn test_program_move_funds_success() { + let tokens: i64 = 100; + let data: Vec = serialize(&tokens).unwrap(); + let keys = vec![Pubkey::default(); 2]; + let mut accounts = vec![Account::default(), Account::default()]; + accounts[0].tokens = 100; + accounts[1].tokens = 1; + + { + let mut infos: Vec<_> = (&keys) + .into_iter() + .zip(&mut accounts) + .map(|(key, account)| KeyedAccount { key, account }) + .collect(); + + let dp = DynamicProgram::new("move_funds".to_string()); + dp.call(&mut infos, &data); + } + assert_eq!(0, accounts[0].tokens); + assert_eq!(101, accounts[1].tokens); + } + + #[test] + fn test_program_move_funds_insufficient_funds() { + let tokens: i64 = 100; + let data: Vec = serialize(&tokens).unwrap(); + let keys = vec![Pubkey::default(); 2]; + let mut accounts = vec![Account::default(), Account::default()]; + accounts[0].tokens = 10; + accounts[1].tokens = 1; + + { + let mut infos: Vec<_> = (&keys) + .into_iter() + .zip(&mut accounts) + .map(|(key, account)| KeyedAccount { key, account }) + .collect(); + + let dp = DynamicProgram::new("move_funds".to_string()); + dp.call(&mut infos, &data); + } + assert_eq!(10, accounts[0].tokens); + assert_eq!(1, accounts[1].tokens); + } + + // TODO add more tests to validate the Userdata and Account data is + // moving across the boundary correctly + +} diff --git a/src/lib.rs b/src/lib.rs index 3c487d3641fdad..3d918d24e37ce1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,6 +21,7 @@ pub mod instruction; pub mod crdt; pub mod budget_program; pub mod drone; +pub mod dynamic_program; pub mod entry; pub mod entry_writer; #[cfg(feature = "erasure")] @@ -82,6 +83,7 @@ extern crate jsonrpc_core; extern crate jsonrpc_macros; extern crate jsonrpc_http_server; extern crate libc; +extern crate libloading; #[macro_use] extern crate log; extern crate nix; diff --git a/src/system_program.rs b/src/system_program.rs index 6f6d88465f24d7..e69bfb031414f2 100644 --- a/src/system_program.rs +++ b/src/system_program.rs @@ -2,7 +2,10 @@ use bank::Account; use bincode::deserialize; +use dynamic_program::DynamicProgram; use signature::Pubkey; +use std::collections::HashMap; +use std::sync::RwLock; use transaction::Transaction; #[derive(Serialize, Deserialize, Debug, Clone)] @@ -25,6 +28,10 @@ pub enum SystemProgram { /// * Transaction::keys[0] - source /// * Transaction::keys[1] - destination Move { tokens: i64 }, + /// Load a program + /// programn_id - id to associate this program + /// nanme - file path of the program to load + Load { program_id: Pubkey, name: String }, } pub const SYSTEM_PROGRAM_ID: [u8; 32] = [0u8; 32]; @@ -40,7 +47,11 @@ impl SystemProgram { pub fn get_balance(account: &Account) -> i64 { account.tokens } - pub fn process_transaction(tx: &Transaction, accounts: &mut [Account]) { + pub fn process_transaction( + tx: &Transaction, + accounts: &mut [Account], + loaded_programs: &RwLock>, + ) { if let Ok(syscall) = deserialize(&tx.userdata) { trace!("process_transaction: {:?}", syscall); match syscall { @@ -74,6 +85,11 @@ impl SystemProgram { accounts[0].tokens -= tokens; accounts[1].tokens += tokens; } + SystemProgram::Load { program_id, name } => { + println!("Load program: {}", name); + let mut hashmap = loaded_programs.write().unwrap(); + hashmap.insert(program_id, DynamicProgram::new(name)); + } } } else { info!("Invalid transaction userdata: {:?}", tx.userdata); @@ -83,17 +99,23 @@ impl SystemProgram { #[cfg(test)] mod test { use bank::Account; + use bincode::serialize; + use dynamic_program::KeyedAccount; use hash::Hash; use signature::{Keypair, KeypairUtil, Pubkey}; + use std::collections::HashMap; + use std::sync::RwLock; use system_program::SystemProgram; use transaction::Transaction; + #[test] fn test_create_noop() { let from = Keypair::new(); let to = Keypair::new(); let mut accounts = vec![Account::default(), Account::default()]; let tx = Transaction::system_new(&from, to.pubkey(), 0, Hash::default()); - SystemProgram::process_transaction(&tx, &mut accounts); + let hash = RwLock::new(HashMap::new()); + SystemProgram::process_transaction(&tx, &mut accounts, &hash); assert_eq!(accounts[0].tokens, 0); assert_eq!(accounts[1].tokens, 0); } @@ -104,7 +126,8 @@ mod test { let mut accounts = vec![Account::default(), Account::default()]; accounts[0].tokens = 1; let tx = Transaction::system_new(&from, to.pubkey(), 1, Hash::default()); - SystemProgram::process_transaction(&tx, &mut accounts); + let hash = RwLock::new(HashMap::new()); + SystemProgram::process_transaction(&tx, &mut accounts, &hash); assert_eq!(accounts[0].tokens, 0); assert_eq!(accounts[1].tokens, 1); } @@ -116,7 +139,8 @@ mod test { accounts[0].tokens = 1; accounts[0].program_id = from.pubkey(); let tx = Transaction::system_new(&from, to.pubkey(), 1, Hash::default()); - SystemProgram::process_transaction(&tx, &mut accounts); + let hash = RwLock::new(HashMap::new()); + SystemProgram::process_transaction(&tx, &mut accounts, &hash); assert_eq!(accounts[0].tokens, 1); assert_eq!(accounts[1].tokens, 0); } @@ -127,7 +151,8 @@ mod test { let mut accounts = vec![Account::default(), Account::default()]; let tx = Transaction::system_create(&from, to.pubkey(), Hash::default(), 0, 1, to.pubkey(), 0); - SystemProgram::process_transaction(&tx, &mut accounts); + let hash = RwLock::new(HashMap::new()); + SystemProgram::process_transaction(&tx, &mut accounts, &hash); assert!(accounts[0].userdata.is_empty()); assert_eq!(accounts[1].userdata.len(), 1); assert_eq!(accounts[1].program_id, to.pubkey()); @@ -147,7 +172,8 @@ mod test { Pubkey::default(), 0, ); - SystemProgram::process_transaction(&tx, &mut accounts); + let hash = RwLock::new(HashMap::new()); + SystemProgram::process_transaction(&tx, &mut accounts, &hash); assert!(accounts[1].userdata.is_empty()); } #[test] @@ -165,7 +191,8 @@ mod test { Pubkey::default(), 0, ); - SystemProgram::process_transaction(&tx, &mut accounts); + let hash = RwLock::new(HashMap::new()); + SystemProgram::process_transaction(&tx, &mut accounts, &hash); assert!(accounts[1].userdata.is_empty()); } #[test] @@ -183,16 +210,63 @@ mod test { Pubkey::default(), 0, ); - SystemProgram::process_transaction(&tx, &mut accounts); + let hash = RwLock::new(HashMap::new()); + SystemProgram::process_transaction(&tx, &mut accounts, &hash); assert_eq!(accounts[1].userdata.len(), 3); } #[test] + fn test_load_call() { + // first load the program + let program_id = Pubkey::default(); // same program id for both + let loaded_programs = RwLock::new(HashMap::new()); + { + let from = Keypair::new(); + let mut accounts = vec![Account::default()]; + let tx = Transaction::system_load( + &from, + Hash::default(), + 0, + program_id, + "move_funds".to_string(), + ); + + SystemProgram::process_transaction(&tx, &mut accounts, &loaded_programs); + } + // then call the program + { + let keys = vec![Pubkey::default(), Pubkey::default()]; + let mut accounts = vec![Account::default(), Account::default()]; + accounts[0].tokens = 100; + accounts[1].tokens = 1; + let tokens: i64 = 100; + let data: Vec = serialize(&tokens).unwrap(); + { + let hash = loaded_programs.write().unwrap(); + match hash.get(&program_id) { + Some(dp) => { + let mut infos: Vec<_> = (&keys) + .into_iter() + .zip(&mut accounts) + .map(|(key, account)| KeyedAccount { key, account }) + .collect(); + + dp.call(&mut infos, &data); + } + None => panic!("failed to find program in hash"), + } + } + assert_eq!(0, accounts[0].tokens); + assert_eq!(101, accounts[1].tokens); + } + } + #[test] fn test_create_assign() { let from = Keypair::new(); let program = Keypair::new(); let mut accounts = vec![Account::default()]; let tx = Transaction::system_assign(&from, Hash::default(), program.pubkey(), 0); - SystemProgram::process_transaction(&tx, &mut accounts); + let hash = RwLock::new(HashMap::new()); + SystemProgram::process_transaction(&tx, &mut accounts, &hash); assert_eq!(accounts[0].program_id, program.pubkey()); } #[test] @@ -202,7 +276,8 @@ mod test { let mut accounts = vec![Account::default(), Account::default()]; accounts[0].tokens = 1; let tx = Transaction::new(&from, to.pubkey(), 1, Hash::default()); - SystemProgram::process_transaction(&tx, &mut accounts); + let hash = RwLock::new(HashMap::new()); + SystemProgram::process_transaction(&tx, &mut accounts, &hash); assert_eq!(accounts[0].tokens, 0); assert_eq!(accounts[1].tokens, 1); } diff --git a/src/transaction.rs b/src/transaction.rs index b6d339f873aec5..cf5649fa574322 100644 --- a/src/transaction.rs +++ b/src/transaction.rs @@ -195,6 +195,24 @@ impl Transaction { fee, ) } + /// Create and sign new SystemProgram::Load transaction + pub fn system_load( + from_keypair: &Keypair, + last_id: Hash, + fee: i64, + program_id: Pubkey, + name: String, + ) -> Self { + let load = SystemProgram::Load { program_id, name }; + Transaction::new_with_userdata( + from_keypair, + &[], + SystemProgram::id(), + serialize(&load).unwrap(), + last_id, + fee, + ) + } /// Create and sign new SystemProgram::CreateAccount transaction pub fn system_assign( from_keypair: &Keypair, From 55f4051820f93d9148a9b77f416156e87d1f4475 Mon Sep 17 00:00:00 2001 From: Jack May Date: Thu, 20 Sep 2018 23:58:03 -0700 Subject: [PATCH 2/7] fix order of fn def, test nit --- src/system_program.rs | 3 ++- src/transaction.rs | 36 ++++++++++++++++++------------------ 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/system_program.rs b/src/system_program.rs index e69bfb031414f2..9c5e3d52fda05e 100644 --- a/src/system_program.rs +++ b/src/system_program.rs @@ -217,11 +217,11 @@ mod test { #[test] fn test_load_call() { // first load the program - let program_id = Pubkey::default(); // same program id for both let loaded_programs = RwLock::new(HashMap::new()); { let from = Keypair::new(); let mut accounts = vec![Account::default()]; + let program_id = Pubkey::default(); // same program id for both let tx = Transaction::system_load( &from, Hash::default(), @@ -234,6 +234,7 @@ mod test { } // then call the program { + let program_id = Pubkey::default(); // same program id for both let keys = vec![Pubkey::default(), Pubkey::default()]; let mut accounts = vec![Account::default(), Account::default()]; accounts[0].tokens = 100; diff --git a/src/transaction.rs b/src/transaction.rs index cf5649fa574322..0c0196d717a105 100644 --- a/src/transaction.rs +++ b/src/transaction.rs @@ -195,24 +195,6 @@ impl Transaction { fee, ) } - /// Create and sign new SystemProgram::Load transaction - pub fn system_load( - from_keypair: &Keypair, - last_id: Hash, - fee: i64, - program_id: Pubkey, - name: String, - ) -> Self { - let load = SystemProgram::Load { program_id, name }; - Transaction::new_with_userdata( - from_keypair, - &[], - SystemProgram::id(), - serialize(&load).unwrap(), - last_id, - fee, - ) - } /// Create and sign new SystemProgram::CreateAccount transaction pub fn system_assign( from_keypair: &Keypair, @@ -252,6 +234,24 @@ impl Transaction { fee, ) } + /// Create and sign new SystemProgram::Load transaction + pub fn system_load( + from_keypair: &Keypair, + last_id: Hash, + fee: i64, + program_id: Pubkey, + name: String, + ) -> Self { + let load = SystemProgram::Load { program_id, name }; + Transaction::new_with_userdata( + from_keypair, + &[], + SystemProgram::id(), + serialize(&load).unwrap(), + last_id, + fee, + ) + } /// Create and sign new SystemProgram::Move transaction pub fn new(from_keypair: &Keypair, to: Pubkey, tokens: i64, last_id: Hash) -> Self { Transaction::system_move(from_keypair, to, tokens, last_id, 0) From be78e4d7f2d9ffe780796f95dadc9da326305b19 Mon Sep 17 00:00:00 2001 From: Jack May Date: Fri, 21 Sep 2018 00:28:17 -0700 Subject: [PATCH 3/7] unable to reproduce ci failure, submit debug --- src/system_program.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/system_program.rs b/src/system_program.rs index 9c5e3d52fda05e..ce5e0b498d712f 100644 --- a/src/system_program.rs +++ b/src/system_program.rs @@ -220,7 +220,7 @@ mod test { let loaded_programs = RwLock::new(HashMap::new()); { let from = Keypair::new(); - let mut accounts = vec![Account::default()]; + let mut accounts = vec![Account::default(), Account::default()]; let program_id = Pubkey::default(); // same program id for both let tx = Transaction::system_load( &from, @@ -251,13 +251,13 @@ mod test { .map(|(key, account)| KeyedAccount { key, account }) .collect(); - dp.call(&mut infos, &data); + //dp.call(&mut infos, &data); } None => panic!("failed to find program in hash"), } } - assert_eq!(0, accounts[0].tokens); - assert_eq!(101, accounts[1].tokens); + //assert_eq!(0, accounts[0].tokens); + //assert_eq!(101, accounts[1].tokens); } } #[test] From 6f583dafbd0290cdfee877d987e7c0b841c79ab9 Mon Sep 17 00:00:00 2001 From: Jack May Date: Fri, 21 Sep 2018 00:35:04 -0700 Subject: [PATCH 4/7] compiler warning --- src/system_program.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/system_program.rs b/src/system_program.rs index ce5e0b498d712f..45c76ed72ab076 100644 --- a/src/system_program.rs +++ b/src/system_program.rs @@ -240,12 +240,12 @@ mod test { accounts[0].tokens = 100; accounts[1].tokens = 1; let tokens: i64 = 100; - let data: Vec = serialize(&tokens).unwrap(); + let _data: Vec = serialize(&tokens).unwrap(); { let hash = loaded_programs.write().unwrap(); match hash.get(&program_id) { - Some(dp) => { - let mut infos: Vec<_> = (&keys) + Some(_dp) => { + let mut _infos: Vec<_> = (&keys) .into_iter() .zip(&mut accounts) .map(|(key, account)| KeyedAccount { key, account }) From cc320fbc704dde78eb4186b461e2a0e64d687af4 Mon Sep 17 00:00:00 2001 From: Jack May Date: Sun, 23 Sep 2018 11:08:24 -0700 Subject: [PATCH 5/7] linux bug workaround, never unload native modules --- src/bank.rs | 3 -- src/dynamic_program.rs | 72 ++++++++++++++++++++++++++++++++------- src/system_program.rs | 76 +++++++++++++++++++++++++++++++++++++----- 3 files changed, 128 insertions(+), 23 deletions(-) diff --git a/src/bank.rs b/src/bank.rs index 119d173ac3f358..6ddb43db051db2 100644 --- a/src/bank.rs +++ b/src/bank.rs @@ -389,12 +389,9 @@ impl Bank { // TODO: the runtime should be checking read/write access to memory // we are trusting the hard coded contracts not to clobber or allocate BudgetState::process_transaction(&tx, accounts) -<<<<<<< HEAD } else if StorageProgram::check_id(&tx.program_id) { StorageProgram::process_transaction(&tx, accounts) -======= } else if self.loaded_contract(&tx, accounts) { ->>>>>>> Integration of native dynamic programs } else { return Err(BankError::UnknownContractId(tx.program_id)); } diff --git a/src/dynamic_program.rs b/src/dynamic_program.rs index 322c2de9b095cf..c226bce0132a69 100644 --- a/src/dynamic_program.rs +++ b/src/dynamic_program.rs @@ -2,7 +2,8 @@ extern crate bincode; extern crate generic_array; use bank::Account; -use libloading::{Library, Symbol}; +use libc; +use libloading; use signature::Pubkey; use std::path::PathBuf; @@ -55,7 +56,10 @@ pub enum DynamicProgram { /// * Transaction::keys[0..] - program dependent /// * name - name of the program, translated to a file path of the program module /// * userdata - program specific user data - Native { name: String, library: Library }, + Native { + name: String, + library: libloading::Library, + }, /// Bpf program /// * Transaction::keys[0..] - program dependent /// * TODO BPF specific stuff @@ -66,23 +70,28 @@ pub enum DynamicProgram { impl DynamicProgram { pub fn new(name: String) -> Self { // TODO determine what kind of module to load + // create native program - println!("loading {}", name); let path = create_library_path(&name); - let library = Library::new(&path).expect("Failed to load library"); + // TODO linux tls bug can cause crash on dlclose, workaround by never unloading + let os_lib = + libloading::os::unix::Library::open(Some(path), libc::RTLD_NODELETE | libc::RTLD_NOW) + .unwrap(); + let library = libloading::Library::from(os_lib); DynamicProgram::Native { name, library } } pub fn call(&self, infos: &mut Vec, data: &[u8]) { match self { DynamicProgram::Native { name, library } => unsafe { - let entrypoint: Symbol = match library.get(ENTRYPOINT.as_bytes()) { - Ok(s) => s, - Err(e) => panic!( - "{:?} Unable to find {:?} in program {}", - e, ENTRYPOINT, name - ), - }; + let entrypoint: libloading::Symbol = + match library.get(ENTRYPOINT.as_bytes()) { + Ok(s) => s, + Err(e) => panic!( + "{:?} Unable to find {:?} in program {}", + e, ENTRYPOINT, name + ), + }; entrypoint(infos, data); }, DynamicProgram::Bpf { .. } => { @@ -100,9 +109,10 @@ mod tests { use bincode::serialize; use signature::Pubkey; use std::path::Path; + use std::thread; #[test] - fn test_create_library_path_1() { + fn test_create_library_path() { let path = create_library_path("noop"); assert_eq!(true, Path::new(&path).exists()); let path = create_library_path("print"); @@ -198,6 +208,44 @@ mod tests { assert_eq!(1, accounts[1].tokens); } + #[test] + fn test_program_move_funds_succes_many_threads() { + let num_threads = 42; // number of threads to spawn + let num_iters = 100; // number of iterations of test in each thread + let mut threads = Vec::new(); + for _t in 0..num_threads { + threads.push(thread::spawn(move || { + for _i in 0..num_iters { + { + let tokens: i64 = 100; + let data: Vec = serialize(&tokens).unwrap(); + let keys = vec![Pubkey::default(); 2]; + let mut accounts = vec![Account::default(), Account::default()]; + accounts[0].tokens = 100; + accounts[1].tokens = 1; + + { + let mut infos: Vec<_> = (&keys) + .into_iter() + .zip(&mut accounts) + .map(|(key, account)| KeyedAccount { key, account }) + .collect(); + + let dp = DynamicProgram::new("move_funds".to_string()); + dp.call(&mut infos, &data); + } + assert_eq!(0, accounts[0].tokens); + assert_eq!(101, accounts[1].tokens); + } + } + })); + } + + for thread in threads { + thread.join().unwrap(); + } + } + // TODO add more tests to validate the Userdata and Account data is // moving across the boundary correctly diff --git a/src/system_program.rs b/src/system_program.rs index 45c76ed72ab076..e29500eb0e52be 100644 --- a/src/system_program.rs +++ b/src/system_program.rs @@ -86,7 +86,6 @@ impl SystemProgram { accounts[1].tokens += tokens; } SystemProgram::Load { program_id, name } => { - println!("Load program: {}", name); let mut hashmap = loaded_programs.write().unwrap(); hashmap.insert(program_id, DynamicProgram::new(name)); } @@ -105,6 +104,7 @@ mod test { use signature::{Keypair, KeypairUtil, Pubkey}; use std::collections::HashMap; use std::sync::RwLock; + use std::thread; use system_program::SystemProgram; use transaction::Transaction; @@ -240,27 +240,88 @@ mod test { accounts[0].tokens = 100; accounts[1].tokens = 1; let tokens: i64 = 100; - let _data: Vec = serialize(&tokens).unwrap(); + let data: Vec = serialize(&tokens).unwrap(); { let hash = loaded_programs.write().unwrap(); match hash.get(&program_id) { - Some(_dp) => { - let mut _infos: Vec<_> = (&keys) + Some(dp) => { + let mut infos: Vec<_> = (&keys) .into_iter() .zip(&mut accounts) .map(|(key, account)| KeyedAccount { key, account }) .collect(); - //dp.call(&mut infos, &data); + dp.call(&mut infos, &data); } None => panic!("failed to find program in hash"), } } - //assert_eq!(0, accounts[0].tokens); - //assert_eq!(101, accounts[1].tokens); + assert_eq!(0, accounts[0].tokens); + assert_eq!(101, accounts[1].tokens); } } #[test] + fn test_load_call_many_threads() { + let num_threads = 42; + let num_iters = 100; + let mut threads = Vec::new(); + for _t in 0..num_threads { + threads.push(thread::spawn(move || { + let _tid = thread::current().id(); + for _i in 0..num_iters { + // first load the program + let loaded_programs = RwLock::new(HashMap::new()); + { + let from = Keypair::new(); + let mut accounts = vec![Account::default(), Account::default()]; + let program_id = Pubkey::default(); // same program id for both + let tx = Transaction::system_load( + &from, + Hash::default(), + 0, + program_id, + "move_funds".to_string(), + ); + + SystemProgram::process_transaction(&tx, &mut accounts, &loaded_programs); + } + // then call the program + { + let program_id = Pubkey::default(); // same program id for both + let keys = vec![Pubkey::default(), Pubkey::default()]; + let mut accounts = vec![Account::default(), Account::default()]; + accounts[0].tokens = 100; + accounts[1].tokens = 1; + let tokens: i64 = 100; + let data: Vec = serialize(&tokens).unwrap(); + { + let hash = loaded_programs.write().unwrap(); + match hash.get(&program_id) { + Some(dp) => { + let mut infos: Vec<_> = (&keys) + .into_iter() + .zip(&mut accounts) + .map(|(key, account)| KeyedAccount { key, account }) + .collect(); + + dp.call(&mut infos, &data); + } + None => panic!("failed to find program in hash"), + } + } + assert_eq!(0, accounts[0].tokens); + assert_eq!(101, accounts[1].tokens); + } + } + })); + } + + for thread in threads { + thread.join().unwrap(); + } + + } + #[test] fn test_create_assign() { let from = Keypair::new(); let program = Keypair::new(); @@ -282,7 +343,6 @@ mod test { assert_eq!(accounts[0].tokens, 0); assert_eq!(accounts[1].tokens, 1); } - /// Detect binary changes in the serialized program userdata, which could have a downstream /// affect on SDKs and DApps #[test] From dffb77bf55c2ff8634bc82c9d791257c6072de21 Mon Sep 17 00:00:00 2001 From: Jack May Date: Sun, 23 Sep 2018 12:10:11 -0700 Subject: [PATCH 6/7] fix fmt --- src/system_program.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/system_program.rs b/src/system_program.rs index e29500eb0e52be..9d928969aa1686 100644 --- a/src/system_program.rs +++ b/src/system_program.rs @@ -319,7 +319,6 @@ mod test { for thread in threads { thread.join().unwrap(); } - } #[test] fn test_create_assign() { From 6170c9c4512f9ab1db3a0810e9880d7c35a32282 Mon Sep 17 00:00:00 2001 From: Jack May Date: Sun, 23 Sep 2018 21:55:36 -0700 Subject: [PATCH 7/7] spelling --- src/system_program.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/system_program.rs b/src/system_program.rs index 9d928969aa1686..1f0ed0a0b97df6 100644 --- a/src/system_program.rs +++ b/src/system_program.rs @@ -29,7 +29,7 @@ pub enum SystemProgram { /// * Transaction::keys[1] - destination Move { tokens: i64 }, /// Load a program - /// programn_id - id to associate this program + /// program_id - id to associate this program /// nanme - file path of the program to load Load { program_id: Pubkey, name: String }, }