Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

make the upper limit of allocated atoms and pairs hard coded #358

Merged
merged 3 commits into from
Jan 8, 2024
Merged

make the upper limit of allocated atoms and pairs hard coded #358

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a467650
make the upper limit of allocated atoms and pairs hard coded instead …
arvidn Jan 6, 2024
61883a4
update test
arvidn Jan 6, 2024
c0e8395
update benchmark
arvidn Jan 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 5 additions & 5 deletions benches/run-program.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,11 +209,11 @@ fn run_program_benchmark(c: &mut Criterion) {
("hash-string", long_strings),
("hash-tree", large_tree::<16>),
("large-block", large_block),
("loop_add", single_value::<4000000>),
("loop_ior", single_value::<4000000>),
("loop_not", single_value::<4000000>),
("loop_sub", single_value::<4000000>),
("loop_xor", single_value::<4000000>),
("loop_add", single_value::<3675000>),
("loop_ior", single_value::<3675000>),
("loop_not", single_value::<3675000>),
("loop_sub", single_value::<3675000>),
("loop_xor", single_value::<3675000>),
("matrix-multiply", matrix::<50, 50>),
("point-pow", point_pow),
("pubkey-tree", large_tree::<10>),
Expand Down
55 changes: 26 additions & 29 deletions src/allocator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,14 +56,11 @@ pub struct Allocator {

// the atom_vec may not grow past this
heap_limit: usize,

// the pair_vec may not grow past this
pair_limit: usize,

// the atom_vec may not grow past this
atom_limit: usize,
}

const MAX_NUM_ATOMS: usize = 62500000;
const MAX_NUM_PAIRS: usize = 62500000;

impl Default for Allocator {
fn default() -> Self {
Self::new()
Expand All @@ -72,29 +69,18 @@ impl Default for Allocator {

impl Allocator {
pub fn new() -> Self {
Self::new_limited(
u32::MAX as usize,
i32::MAX as usize,
(i32::MAX - 1) as usize,
)
Self::new_limited(u32::MAX as usize)
}

pub fn new_limited(heap_limit: usize, pair_limit: usize, atom_limit: usize) -> Self {
pub fn new_limited(heap_limit: usize) -> Self {
// we have a maximum of 4 GiB heap, because pointers are 32 bit unsigned
assert!(heap_limit <= u32::MAX as usize);
// the atoms and pairs share a single 32 bit address space, where
// negative numbers are atoms and positive numbers are pairs. That's why
// we have one more slot for pairs than atoms
assert!(pair_limit <= i32::MAX as usize);
assert!(atom_limit < i32::MAX as usize);

let mut r = Self {
u8_vec: Vec::new(),
pair_vec: Vec::new(),
atom_vec: Vec::new(),
heap_limit,
pair_limit,
atom_limit,
};
r.u8_vec.reserve(1024 * 1024);
r.atom_vec.reserve(256);
Expand Down Expand Up @@ -136,7 +122,7 @@ impl Allocator {
if (self.heap_limit - start as usize) < v.len() {
return err(self.null(), "out of memory");
}
if self.atom_vec.len() == self.atom_limit {
if self.atom_vec.len() == MAX_NUM_ATOMS {
return err(self.null(), "too many atoms");
}
self.u8_vec.extend_from_slice(v);
Expand All @@ -159,7 +145,7 @@ impl Allocator {

pub fn new_pair(&mut self, first: NodePtr, rest: NodePtr) -> Result<NodePtr, EvalErr> {
let r = self.pair_vec.len() as i32;
if self.pair_vec.len() == self.pair_limit {
if self.pair_vec.len() == MAX_NUM_PAIRS {
return err(self.null(), "too many pairs");
}
self.pair_vec.push(IntPair { first, rest });
Expand All @@ -170,7 +156,7 @@ impl Allocator {
if node.0 >= 0 {
return err(node, "(internal error) substr expected atom, got pair");
}
if self.atom_vec.len() == self.atom_limit {
if self.atom_vec.len() == MAX_NUM_ATOMS {
return err(self.null(), "too many atoms");
}
let atom = self.atom_vec[(-node.0 - 1) as usize];
Expand All @@ -192,7 +178,7 @@ impl Allocator {
}

pub fn new_concat(&mut self, new_size: usize, nodes: &[NodePtr]) -> Result<NodePtr, EvalErr> {
if self.atom_vec.len() == self.atom_limit {
if self.atom_vec.len() == MAX_NUM_ATOMS {
return err(self.null(), "too many atoms");
}
let start = self.u8_vec.len();
Expand Down Expand Up @@ -471,7 +457,7 @@ fn test_allocate_pair() {

#[test]
fn test_allocate_heap_limit() {
let mut a = Allocator::new_limited(6, i32::MAX as usize, (i32::MAX - 1) as usize);
let mut a = Allocator::new_limited(6);
// we can't allocate 6 bytes
assert_eq!(a.new_atom(b"foobar").unwrap_err().1, "out of memory");
// but 5 is OK
Expand All @@ -480,7 +466,7 @@ fn test_allocate_heap_limit() {

#[test]
fn test_allocate_atom_limit() {
let mut a = Allocator::new_limited(u32::MAX as usize, i32::MAX as usize, 5);
let mut a = Allocator::new();
// we can allocate 5 atoms total
// keep in mind that we always have 2 pre-allocated atoms for null and one,
// so with a limit of 5, we only have 3 slots left at this point.
Expand All @@ -490,17 +476,28 @@ fn test_allocate_atom_limit() {

// the 4th fails and ensure not to append atom to the stack
assert_eq!(a.u8_vec.len(), 10);
for _ in 3..MAX_NUM_ATOMS - 2 {
// exhaust the numebr of atoms allowed to be allocated
let _ = a.new_atom(b"foo").unwrap();
}
assert_eq!(a.new_atom(b"foobar").unwrap_err().1, "too many atoms");
assert_eq!(a.u8_vec.len(), 10);

// the pre-allocated null() and one() also count against the limit, and they
// use 0 and 1 bytes respectively
assert_eq!(a.u8_vec.len(), MAX_NUM_ATOMS * 3 - 3 - 2);
}

#[test]
fn test_allocate_pair_limit() {
let mut a = Allocator::new_limited(u32::MAX as usize, 1, (i32::MAX - 1) as usize);
let mut a = Allocator::new();
let atom = a.new_atom(b"foo").unwrap();
// one pair is OK
let _pair1 = a.new_pair(atom, atom).unwrap();
// but not 2
for _ in 1..MAX_NUM_PAIRS {
// exhaust the numebr of pairs allowed to be allocated
let _ = a.new_pair(atom, atom).unwrap();
}

assert_eq!(a.new_pair(atom, atom).unwrap_err().1, "too many pairs");
}

Expand Down Expand Up @@ -601,7 +598,7 @@ fn test_sexp() {

#[test]
fn test_concat_limit() {
let mut a = Allocator::new_limited(9, i32::MAX as usize, (i32::MAX - 1) as usize);
let mut a = Allocator::new_limited(9);
let atom1 = a.new_atom(b"f").unwrap();
let atom2 = a.new_atom(b"o").unwrap();
let atom3 = a.new_atom(b"o").unwrap();
Expand Down
124 changes: 85 additions & 39 deletions tests/run-programs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,47 @@
import platform
from colorama import init, Fore, Style
from run import run_clvm
from os.path import isfile
from os.path import isfile, splitext, basename

init()
ret = 0

for fn in glob.glob('programs/large-atom-*.hex.invalid'):

expect = {
"args-add": "cost exceeded",
"args-all": "cost exceeded",
"args-and": "cost exceeded",
"args-any": "cost exceeded",
"args-cat": "cost exceeded",
"args-mul": "cost exceeded",
"args-or": "cost exceeded",
"args-point_add": "cost exceeded",
"args-sha": "cost exceeded",
"args-sub": "cost exceeded",
"args-unknown-1": "cost exceeded",
"args-unknown-2": "cost exceeded",
"args-unknown-3": "cost exceeded",
"args-unknown-4": "cost exceeded",
"args-unknown-5": "too many pairs",
"args-unknown-6": "too many pairs",
"args-unknown-7": "too many pairs",
"args-unknown-8": "too many pairs",
"args-unknown-9": "too many pairs",
"args-xor": "cost exceeded",
"recursive-add": "cost exceeded",
"recursive-ash": "cost exceeded",
"recursive-cat": "cost exceeded",
"recursive-cons": "too many pairs",
"recursive-div": "cost exceeded",
"recursive-lsh": "cost exceeded",
"recursive-mul": "cost exceeded",
"recursive-not": "cost exceeded",
"recursive-pubkey": "cost exceeded",
"recursive-sub": "too many pairs",
"softfork-1": "cost exceeded",
"softfork-2": "cost exceeded",
}

for fn in glob.glob("programs/large-atom-*.hex.invalid"):
try:
print(fn)
run_clvm(fn)
Expand All @@ -23,80 +57,92 @@
print(Fore.GREEN + f"OK: expected: {e}" + Style.RESET_ALL)


for fn in glob.glob('programs/*.clvm'):

hexname = fn[:-4] + 'hex'
for fn in glob.glob("programs/*.clvm"):
hexname = fn[:-4] + "hex"
if isfile(hexname):
continue
with open(hexname, 'w+') as out:
with open(hexname, "w+") as out:
print(f"compiling {fn}")
proc = subprocess.Popen(['opc', fn], stdout=out)
proc = subprocess.Popen(["opc", fn], stdout=out)
proc.wait()

for fn in glob.glob('programs/*.env'):
hexenv = fn + 'hex'
for fn in glob.glob("programs/*.env"):
hexenv = fn + "hex"
if isfile(hexenv):
continue
with open(hexenv, 'w+') as out:
with open(hexenv, "w+") as out:
print(f"compiling {fn}")
proc = subprocess.Popen(['opc', fn], stdout=out)
proc = subprocess.Popen(["opc", fn], stdout=out)
proc.wait()

for hexname in sorted(glob.glob('programs/*.hex')):

hexenv = hexname[:-3] + 'envhex'
for hexname in sorted(glob.glob("programs/*.hex")):
hexenv = hexname[:-3] + "envhex"

command = ['./run.py', hexname, hexenv]
command = ["./run.py", hexname, hexenv]

# prepend the size command, to measure RSS
if platform.system() == 'Darwin':
command = ['/usr/bin/time', '-l'] + command;
if platform.system() == 'Linux':
command = ['/usr/bin/time'] + command;
if platform.system() == "Darwin":
command = ["/usr/bin/time", "-l"] + command
if platform.system() == "Linux":
command = ["/usr/bin/time"] + command

print(' '.join(command))
print(" ".join(command))
start = time.perf_counter()
proc = subprocess.run(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
output = proc.stderr.decode('UTF-8')
output += proc.stdout.decode('UTF-8')
output = proc.stderr.decode("UTF-8")
output += proc.stdout.decode("UTF-8")
end = time.perf_counter()

if 'FAIL: cost exceeded' not in output:
test = splitext(basename(hexname))[0]
expected_error = expect[test]
if f"FAIL: {expected_error}" not in output:
ret += 1
print(Fore.RED + '\nTEST FAILURE: expected cost to be exceeded\n' + Style.RESET_ALL)
print(
Fore.RED
+ f'\nTEST FAILURE: expected "{expected_error}"\n'
+ Style.RESET_ALL
)
print(output)

print(Fore.YELLOW + (' Runtime: %0.2f s' % (end - start)) + Style.RESET_ALL)
print(Fore.YELLOW + (" Runtime: %0.2f s" % (end - start)) + Style.RESET_ALL)

# parse RSS (MacOS and Linux only)
size = None
if platform.system() == 'Darwin':
for l in output.split('\n'):
if 'maximum resident set size' not in l:
if platform.system() == "Darwin":
for l in output.split("\n"):
if "maximum resident set size" not in l:
continue
val, key = l.strip().split(' ', 1)
if key == 'maximum resident set size':
val, key = l.strip().split(" ", 1)
if key == "maximum resident set size":
size = int(val) / 1024 / 1024
if platform.system() == 'Linux':
if platform.system() == "Linux":
# example output:
# 10.49user 0.32system 0:10.84elapsed 99%CPU (0avgtext+0avgdata 189920maxresident)k
for l in output.split('\n'):
if 'maxresident)k' not in l:
for l in output.split("\n"):
if "maxresident)k" not in l:
continue
size = int(l.split('maxresident)k')[0].split(' ')[-1]) / 1024
size = int(l.split("maxresident)k")[0].split(" ")[-1]) / 1024
if size != None:
print(Fore.YELLOW + (' Resident Size: %d MiB' % size) + Style.RESET_ALL)
print(Fore.YELLOW + (" Resident Size: %d MiB" % size) + Style.RESET_ALL)

if size > 2300:
ret += 1
print(Fore.RED + '\nTEST FAILURE: Max memory use exceeded (limit: 2300 MB)\n' + Style.RESET_ALL)
print(
Fore.RED
+ "\nTEST FAILURE: Max memory use exceeded (limit: 2300 MB)\n"
+ Style.RESET_ALL
)

# cost 10923314721 roughly corresponds to 11 seconds
if end - start > 11:
ret += 1
print(Fore.RED + '\nTEST FAILURE: Time exceeded: %f (limit: 11)\n' % (end - start) + Style.RESET_ALL)
print(
Fore.RED
+ "\nTEST FAILURE: Time exceeded: %f (limit: 11)\n" % (end - start)
+ Style.RESET_ALL
)

if ret:
print(Fore.RED + f'\n There were {ret} failures!\n' + Style.RESET_ALL)
print(Fore.RED + f"\n There were {ret} failures!\n" + Style.RESET_ALL)

sys.exit(ret)
2 changes: 1 addition & 1 deletion wheel/src/api.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ pub fn run_serialized_chia_program(
flags: u32,
) -> PyResult<(u64, LazyNode)> {
let mut allocator = if flags & LIMIT_HEAP != 0 {
Allocator::new_limited(500000000, 62500000, 62500000)
Allocator::new_limited(500000000)
} else {
Allocator::new()
};
Expand Down
Loading