Skip to content

Commit

Permalink
Get distributed trees working again
Browse files Browse the repository at this point in the history
  • Loading branch information
skailasa committed Sep 13, 2023
1 parent 244a7db commit 4e93e67
Show file tree
Hide file tree
Showing 11 changed files with 838 additions and 809 deletions.
9 changes: 9 additions & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"rust-analyzer.linkedProjects": [
"./tree/Cargo.toml",
"./tree/Cargo.toml",
"./tree/Cargo.toml",
"./tree/Cargo.toml",
"./tree/Cargo.toml"
]
}
8 changes: 0 additions & 8 deletions traits/src/tree.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,6 @@ pub trait Tree {
where
Self: 'a;

fn new(
points: Self::PointDataSlice<'_>,
adaptive: bool,
n_crit: Option<u64>,
depth: Option<u64>,
global_idxs: Self::GlobalIndexSlice<'_>
) -> Self;

// Get depth of tree.
fn get_depth(&self) -> u64;

Expand Down
129 changes: 77 additions & 52 deletions tree/examples/test_uniform.rs
Original file line number Diff line number Diff line change
@@ -1,31 +1,49 @@
// //? mpirun -n {{NPROCESSES}} --features "mpi"
// ? mpirun -n {{NPROCESSES}} --features "mpi"

// use rand::prelude::*;
// use rand::SeedableRng;
use rand::prelude::*;
use rand::SeedableRng;

// use mpi::{environment::Universe, topology::UserCommunicator, traits::*};
use mpi::{environment::Universe, topology::UserCommunicator, traits::*};

// use bempp_traits::tree::Tree;
use bempp_traits::tree::Tree;

// use bempp_tree::types::{
// domain::Domain, morton::MortonKey, multi_node::MultiNodeTree, point::PointType,
// };
use rlst::{
dense::{
base_matrix::BaseMatrix, data_container::VectorContainer, matrix::Matrix, rlst_col_vec,
rlst_mat, rlst_pointer_mat, traits::*, Dot, global,
},
};

// pub fn points_fixture(npoints: i32) -> Vec<[f64; 3]> {
// let mut range = StdRng::seed_from_u64(0);
// let between = rand::distributions::Uniform::from(0.0..1.0);
// let mut points: Vec<[PointType; 3]> = Vec::new();
use bempp_tree::types::{
domain::Domain, morton::MortonKey, multi_node::MultiNodeTree, point::PointType,
};

// for _ in 0..npoints {
// points.push([
// between.sample(&mut range),
// between.sample(&mut range),
// between.sample(&mut range),
// ])
// }
fn points_fixture(
npoints: usize,
min: Option<f64>,
max: Option<f64>,
) -> Matrix<f64, BaseMatrix<f64, VectorContainer<f64>, Dynamic, Dynamic>, Dynamic, Dynamic>
{
// Generate a set of randomly distributed points
let mut range = StdRng::seed_from_u64(0);

// points
// }
let between;
if let (Some(min), Some(max)) = (min, max) {
between = rand::distributions::Uniform::from(min..max);
} else {
between = rand::distributions::Uniform::from(0.0_f64..1.0_f64);
}

let mut points = rlst_mat![f64, (npoints, 3)];

for i in 0..npoints {
points[[i, 0]] = between.sample(&mut range);
points[[i, 1]] = between.sample(&mut range);
points[[i, 2]] = between.sample(&mut range);
}

points
}

// /// Test that the leaves on separate nodes do not overlap.
// fn test_no_overlaps(world: &UserCommunicator, tree: &MultiNodeTree) {
Expand Down Expand Up @@ -82,34 +100,41 @@
// }
// }

// fn main() {
// // Setup an MPI environment
// let universe: Universe = mpi::initialize().unwrap();
// let world = universe.world();
// let comm = world.duplicate();

// // Setup tree parameters
// let adaptive = false;
// let n_crit: Option<_> = None;
// let k: Option<_> = None;
// let depth = Some(4);
// let n_points = 10000;

// let points = points_fixture(n_points);

// let tree = MultiNodeTree::new(&comm, k, &points, adaptive, n_crit, depth);

// test_global_bounds(&comm);
// if world.rank() == 0 {
// println!("\t ... test_global_bounds passed on uniform tree");
// }
// test_uniform(&tree);
// if world.rank() == 0 {
// println!("\t ... test_uniform passed on uniform tree");
// }
// test_no_overlaps(&comm, &tree);
// if world.rank() == 0 {
// println!("\t ... test_no_overlaps passed on uniform tree");
// }
// }
fn main() {}
fn main() {
// Setup an MPI environment
let universe: Universe = mpi::initialize().unwrap();
let world = universe.world();
let comm = world.duplicate();

// Setup tree parameters
let adaptive = false;
let n_crit: Option<_> = None;
let k = 2;
let depth = Some(3);
let n_points = 10000;

// Generate some random test data local to each process
let points = points_fixture(n_points, None, None);
let global_idxs: Vec<_> = (0..n_points).collect();

// Calculate the global domain
let domain = Domain::from_global_points(points.data(), &comm);

// Create a uniform tree
let tree = MultiNodeTree::new(&comm, &points.data(), adaptive, n_crit, depth, k, &global_idxs);

println!("Rank {:?}, leaves {:?}", world.rank(), tree.leaves.len())
// test_global_bounds(&comm);
// if world.rank() == 0 {
// println!("\t ... test_global_bounds passed on uniform tree");
// }
// test_uniform(&tree);
// if world.rank() == 0 {
// println!("\t ... test_uniform passed on uniform tree");
// }
// test_no_overlaps(&comm, &tree);
// if world.rank() == 0 {
// println!("\t ... test_no_overlaps passed on uniform tree");
// }
}
// fn main() {}
144 changes: 72 additions & 72 deletions tree/src/implementations/impl_domain_mpi.rs
Original file line number Diff line number Diff line change
@@ -1,82 +1,82 @@
// use memoffset::offset_of;
// use mpi::{
// datatype::{Equivalence, UncommittedUserDatatype, UserDatatype},
// topology::UserCommunicator,
// traits::*,
// Address,
// };
use memoffset::offset_of;
use mpi::{
datatype::{Equivalence, UncommittedUserDatatype, UserDatatype},
topology::UserCommunicator,
traits::*,
Address,
};

// use crate::types::{domain::Domain, point::PointType};
use crate::types::{domain::Domain, point::PointType};

// unsafe impl Equivalence for Domain {
// type Out = UserDatatype;
// fn equivalent_datatype() -> Self::Out {
// UserDatatype::structured(
// &[1, 1],
// &[
// offset_of!(Domain, origin) as Address,
// offset_of!(Domain, diameter) as Address,
// ],
// &[
// UncommittedUserDatatype::contiguous(3, &PointType::equivalent_datatype()).as_ref(),
// UncommittedUserDatatype::contiguous(3, &PointType::equivalent_datatype()).as_ref(),
// ],
// )
// }
// }
unsafe impl Equivalence for Domain {
type Out = UserDatatype;
fn equivalent_datatype() -> Self::Out {
UserDatatype::structured(
&[1, 1],
&[
offset_of!(Domain, origin) as Address,
offset_of!(Domain, diameter) as Address,
],
&[
UncommittedUserDatatype::contiguous(3, &PointType::equivalent_datatype()).as_ref(),
UncommittedUserDatatype::contiguous(3, &PointType::equivalent_datatype()).as_ref(),
],
)
}
}

// impl Domain {
// /// Compute the points domain over all nodes.
// pub fn from_global_points(local_points: &[[PointType; 3]], comm: &UserCommunicator) -> Domain {
// let size = comm.size();
impl Domain {
/// Compute the points domain over all nodes.
pub fn from_global_points(local_points: &[PointType], comm: &UserCommunicator) -> Domain {
let size = comm.size();

// let local_domain = Domain::from_local_points(local_points);
// let local_bounds: Vec<Domain> = vec![local_domain; size as usize];
// let mut buffer = vec![Domain::default(); size as usize];
let local_domain = Domain::from_local_points(local_points);
let local_bounds: Vec<Domain> = vec![local_domain; size as usize];
let mut buffer = vec![Domain::default(); size as usize];

// comm.all_to_all_into(&local_bounds, &mut buffer[..]);
comm.all_to_all_into(&local_bounds, &mut buffer[..]);

// // Find minimum origin
// let min_x = buffer
// .iter()
// .min_by(|a, b| a.origin[0].partial_cmp(&b.origin[0]).unwrap())
// .unwrap()
// .origin[0];
// let min_y = buffer
// .iter()
// .min_by(|a, b| a.origin[1].partial_cmp(&b.origin[1]).unwrap())
// .unwrap()
// .origin[1];
// let min_z = buffer
// .iter()
// .min_by(|a, b| a.origin[2].partial_cmp(&b.origin[2]).unwrap())
// .unwrap()
// .origin[2];
// Find minimum origin
let min_x = buffer
.iter()
.min_by(|a, b| a.origin[0].partial_cmp(&b.origin[0]).unwrap())
.unwrap()
.origin[0];
let min_y = buffer
.iter()
.min_by(|a, b| a.origin[1].partial_cmp(&b.origin[1]).unwrap())
.unwrap()
.origin[1];
let min_z = buffer
.iter()
.min_by(|a, b| a.origin[2].partial_cmp(&b.origin[2]).unwrap())
.unwrap()
.origin[2];

// let min_origin = [min_x, min_y, min_z];
let min_origin = [min_x, min_y, min_z];

// // Find maximum diameter (+max origin)
// let max_x = buffer
// .iter()
// .max_by(|a, b| a.diameter[0].partial_cmp(&b.diameter[0]).unwrap())
// .unwrap()
// .diameter[0];
// let max_y = buffer
// .iter()
// .max_by(|a, b| a.diameter[1].partial_cmp(&b.diameter[1]).unwrap())
// .unwrap()
// .diameter[1];
// let max_z = buffer
// .iter()
// .max_by(|a, b| a.diameter[2].partial_cmp(&b.diameter[2]).unwrap())
// .unwrap()
// .diameter[2];
// Find maximum diameter (+max origin)
let max_x = buffer
.iter()
.max_by(|a, b| a.diameter[0].partial_cmp(&b.diameter[0]).unwrap())
.unwrap()
.diameter[0];
let max_y = buffer
.iter()
.max_by(|a, b| a.diameter[1].partial_cmp(&b.diameter[1]).unwrap())
.unwrap()
.diameter[1];
let max_z = buffer
.iter()
.max_by(|a, b| a.diameter[2].partial_cmp(&b.diameter[2]).unwrap())
.unwrap()
.diameter[2];

// let max_diameter = [max_x, max_y, max_z];
let max_diameter = [max_x, max_y, max_z];

// Domain {
// origin: min_origin,
// diameter: max_diameter,
// }
// }
// }
Domain {
origin: min_origin,
diameter: max_diameter,
}
}
}
44 changes: 22 additions & 22 deletions tree/src/implementations/impl_morton_mpi.rs
Original file line number Diff line number Diff line change
@@ -1,23 +1,23 @@
// use crate::types::morton::{KeyType, MortonKey};
// use memoffset::offset_of;
// use mpi::{
// datatype::{Equivalence, UncommittedUserDatatype, UserDatatype},
// Address,
// };
use crate::types::morton::{KeyType, MortonKey};
use memoffset::offset_of;
use mpi::{
datatype::{Equivalence, UncommittedUserDatatype, UserDatatype},
Address,
};

// unsafe impl Equivalence for MortonKey {
// type Out = UserDatatype;
// fn equivalent_datatype() -> Self::Out {
// UserDatatype::structured(
// &[1, 1],
// &[
// offset_of!(MortonKey, anchor) as Address,
// offset_of!(MortonKey, morton) as Address,
// ],
// &[
// UncommittedUserDatatype::contiguous(3, &KeyType::equivalent_datatype()).as_ref(),
// UncommittedUserDatatype::contiguous(1, &KeyType::equivalent_datatype()).as_ref(),
// ],
// )
// }
// }
unsafe impl Equivalence for MortonKey {
type Out = UserDatatype;
fn equivalent_datatype() -> Self::Out {
UserDatatype::structured(
&[1, 1],
&[
offset_of!(MortonKey, anchor) as Address,
offset_of!(MortonKey, morton) as Address,
],
&[
UncommittedUserDatatype::contiguous(3, &KeyType::equivalent_datatype()).as_ref(),
UncommittedUserDatatype::contiguous(1, &KeyType::equivalent_datatype()).as_ref(),
],
)
}
}
Loading

0 comments on commit 4e93e67

Please sign in to comment.