From bbe2161a187e4282a9aeece5a945a667a054a604 Mon Sep 17 00:00:00 2001 From: Philippe Marguinaud Date: Fri, 31 May 2024 11:43:02 +0000 Subject: [PATCH] Update buddy_alloc.h --- buddy_alloc.h | 691 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 413 insertions(+), 278 deletions(-) diff --git a/buddy_alloc.h b/buddy_alloc.h index 569959b..1e827ce 100644 --- a/buddy_alloc.h +++ b/buddy_alloc.h @@ -1,8 +1,6 @@ /* * Copyright 2021 Stanislav Paskalev - */ - -/* + * * A binary buddy memory allocator * * To include and use it in your project do the following @@ -10,21 +8,30 @@ * 2. Include the header in places where you need to use the allocator * 3. In one of your source files #define BUDDY_ALLOC_IMPLEMENTATION * and then import the header. This will insert the implementation. + * + * Latest version is available at https://github.com/spaskalev/buddy_alloc */ #ifndef BUDDY_ALLOC_H #define BUDDY_ALLOC_H +#ifndef BUDDY_HEADER #include +#include #include #include -#include #include #include +#ifndef BUDDY_PRINTF +#include +#endif +#endif #ifdef __cplusplus +#ifndef BUDDY_CPP_MANGLED extern "C" { #endif +#endif struct buddy; @@ -60,13 +67,13 @@ struct buddy *buddy_embed_alignment(unsigned char *main, size_t memory_size, siz struct buddy *buddy_resize(struct buddy *buddy, size_t new_memory_size); /* Tests if the allocator can be shrunk in half */ -unsigned int buddy_can_shrink(struct buddy *buddy); +bool buddy_can_shrink(struct buddy *buddy); /* Tests if the allocator is completely empty */ -unsigned int buddy_is_empty(struct buddy *buddy); +bool buddy_is_empty(struct buddy *buddy); /* Tests if the allocator is completely full */ -unsigned int buddy_is_full(struct buddy *buddy); +bool buddy_is_full(struct buddy *buddy); /* Reports the arena size */ size_t buddy_arena_size(struct buddy *buddy); @@ -86,11 +93,11 @@ void *buddy_malloc(struct buddy *buddy, size_t requested_size); void *buddy_calloc(struct buddy *buddy, size_t members_count, size_t member_size); /* Realloc semantics are a joke. See realloc. */ -void *buddy_realloc(struct buddy *buddy, void *ptr, size_t requested_size); +void *buddy_realloc(struct buddy *buddy, void *ptr, size_t requested_size, bool ignore_data); /* Realloc-like behavior that checks for overflow. See reallocarray*/ void *buddy_reallocarray(struct buddy *buddy, void *ptr, - size_t members_count, size_t member_size); + size_t members_count, size_t member_size, bool ignore_data); /* Use the specified buddy to free memory. See free. */ void buddy_free(struct buddy *buddy, void *ptr); @@ -113,28 +120,30 @@ void buddy_unsafe_release_range(struct buddy *buddy, void *ptr, size_t requested */ /* - * Iterate through the allocated slots and call the provided function for each of them. + * Iterate through the free and allocated slots and call the provided function for each of them. * * If the provided function returns a non-NULL result the iteration stops and the result * is returned to called. NULL is returned upon completing iteration without stopping. * * The iteration order is implementation-defined and may change between versions. */ -void *buddy_walk(struct buddy *buddy, void *(fp)(void *ctx, void *addr, size_t slot_size), void *ctx); +void *buddy_walk(struct buddy *buddy, void *(fp)(void *ctx, void *addr, size_t slot_size, size_t allocated), void *ctx); /* * Miscellaneous functions */ /* - * Calculates the fragmentation in the allocator in a 0.0 - 1.0 range. + * Calculates the fragmentation in the allocator in a 0 - 255 range. * NOTE: if you are using a non-power-of-two sized arena the maximum upper bound can be lower. */ -float buddy_fragmentation(struct buddy *buddy); +unsigned char buddy_fragmentation(struct buddy *buddy); #ifdef __cplusplus +#ifndef BUDDY_CPP_MANGLED } #endif +#endif #endif /* BUDDY_ALLOC_H */ @@ -142,15 +151,16 @@ float buddy_fragmentation(struct buddy *buddy); #undef BUDDY_ALLOC_IMPLEMENTATION #ifdef __cplusplus +#ifndef BUDDY_CPP_MANGLED extern "C" { #endif +#endif #ifndef BUDDY_ALLOC_ALIGN #define BUDDY_ALLOC_ALIGN (sizeof(size_t) * CHAR_BIT) #endif #ifdef __cplusplus - #ifndef BUDDY_ALIGNOF #define BUDDY_ALIGNOF(x) alignof(x) #endif @@ -177,12 +187,16 @@ typedef signed long ssize_t; #define _SSIZE_T_DEFINED #endif +#ifndef BUDDY_PRINTF +#define BUDDY_PRINTF printf +#endif + /* * Debug functions */ /* Implementation defined */ -static void buddy_debug(FILE *stream, struct buddy *buddy); +void buddy_debug(struct buddy *buddy); struct buddy_tree; @@ -220,7 +234,7 @@ static size_t buddy_tree_sizeof(uint8_t order); static struct buddy_tree *buddy_tree_init(unsigned char *at, uint8_t order); /* Indicates whether this is a valid position for the tree */ -static unsigned int buddy_tree_valid(struct buddy_tree *t, struct buddy_tree_pos pos); +static bool buddy_tree_valid(struct buddy_tree *t, struct buddy_tree_pos pos); /* Returns the order of the specified buddy allocation tree */ static uint8_t buddy_tree_order(struct buddy_tree *t); @@ -264,7 +278,7 @@ static size_t buddy_tree_index(struct buddy_tree_pos pos); static struct buddy_tree_interval buddy_tree_interval(struct buddy_tree *t, struct buddy_tree_pos pos); /* Checks if one interval contains another */ -static unsigned int buddy_tree_interval_contains(struct buddy_tree_interval outer, +static bool buddy_tree_interval_contains(struct buddy_tree_interval outer, struct buddy_tree_interval inner); /* Return a walk state structure starting from the root of a tree */ @@ -291,23 +305,23 @@ static void buddy_tree_release(struct buddy_tree *t, struct buddy_tree_pos pos); static struct buddy_tree_pos buddy_tree_find_free(struct buddy_tree *t, uint8_t depth); /* Tests if the indicated position is available for allocation */ -static unsigned int buddy_tree_is_free(struct buddy_tree *t, struct buddy_tree_pos pos); +static bool buddy_tree_is_free(struct buddy_tree *t, struct buddy_tree_pos pos); /* Tests if the tree can be shrank in half */ -static unsigned int buddy_tree_can_shrink(struct buddy_tree *t); +static bool buddy_tree_can_shrink(struct buddy_tree *t); /* * Debug functions */ /* Implementation defined */ -static void buddy_tree_debug(FILE *stream, struct buddy_tree *t, struct buddy_tree_pos pos, size_t start_size); +static void buddy_tree_debug(struct buddy_tree *t, struct buddy_tree_pos pos, size_t start_size); /* Implementation defined */ -static unsigned int buddy_tree_check_invariant(struct buddy_tree *t, struct buddy_tree_pos pos); +unsigned int buddy_tree_check_invariant(struct buddy_tree *t, struct buddy_tree_pos pos); -/* Report fragmentation in a 0.0 - 1.0 range */ -static float buddy_tree_fragmentation(struct buddy_tree *t); +/* Report fragmentation in a 0 - 255 range */ +static unsigned char buddy_tree_fragmentation(struct buddy_tree *t); /* * A char-backed bitset implementation @@ -325,7 +339,7 @@ static inline void bitset_set(unsigned char *bitset, size_t pos); static inline void bitset_clear(unsigned char *bitset, size_t pos); -static inline unsigned int bitset_test(const unsigned char *bitset, size_t pos); +static inline bool bitset_test(const unsigned char *bitset, size_t pos); static void bitset_shift_left(unsigned char *bitset, size_t from_pos, size_t to_pos, size_t by); @@ -336,7 +350,7 @@ static void bitset_shift_right(unsigned char *bitset, size_t from_pos, size_t to */ /* Implementation defined */ -static void bitset_debug(FILE *stream, unsigned char *bitset, size_t length); +void bitset_debug(unsigned char *bitset, size_t length); /* * Bits @@ -351,12 +365,15 @@ static size_t highest_bit_position(size_t value); /* Returns the nearest larger or equal power of two */ static inline size_t ceiling_power_of_two(size_t value); +/* Return two to the power of order */ +static inline size_t two_to_the_power_of(size_t order); + /* * Math */ -/* Approximates the square root of a float */ -static inline float approximate_square_root(float f); +/* Calculates the integer square root of an integer */ +static inline size_t integer_square_root(size_t f); /* Implementation @@ -376,7 +393,6 @@ struct buddy { ptrdiff_t main_offset; } arena; size_t buddy_flags; - unsigned char buddy_tree[]; }; struct buddy_embed_check { @@ -400,7 +416,7 @@ static void buddy_toggle_virtual_slots(struct buddy *buddy, unsigned int state); static void buddy_toggle_range_reservation(struct buddy *buddy, void *ptr, size_t requested_size, unsigned int state); static struct buddy *buddy_resize_standard(struct buddy *buddy, size_t new_memory_size); static struct buddy *buddy_resize_embedded(struct buddy *buddy, size_t new_memory_size); -static unsigned int buddy_is_free(struct buddy *buddy, size_t from); +static bool buddy_is_free(struct buddy *buddy, size_t from); static struct buddy_embed_check buddy_embed_offset(size_t memory_size, size_t alignment); static struct buddy_tree_pos deepest_position_for_offset(struct buddy *buddy, size_t offset); @@ -409,13 +425,15 @@ size_t buddy_sizeof(size_t memory_size) { } size_t buddy_sizeof_alignment(size_t memory_size, size_t alignment) { + size_t buddy_tree_order; + if (!is_valid_alignment(alignment)) { return 0; /* invalid */ } if (memory_size < alignment) { return 0; /* invalid */ } - size_t buddy_tree_order = buddy_tree_order_for_memory(memory_size, alignment); + buddy_tree_order = buddy_tree_order_for_memory(memory_size, alignment); return sizeof(struct buddy) + buddy_tree_sizeof((uint8_t)buddy_tree_order); } @@ -425,6 +443,9 @@ struct buddy *buddy_init(unsigned char *at, unsigned char *main, size_t memory_s struct buddy *buddy_init_alignment(unsigned char *at, unsigned char *main, size_t memory_size, size_t alignment) { + size_t at_alignment, main_alignment, buddy_size, buddy_tree_order; + struct buddy *buddy; + if (at == NULL) { return NULL; } @@ -437,11 +458,11 @@ struct buddy *buddy_init_alignment(unsigned char *at, unsigned char *main, size_ if (!is_valid_alignment(alignment)) { return NULL; /* invalid */ } - size_t at_alignment = ((uintptr_t) at) % BUDDY_ALIGNOF(struct buddy); + at_alignment = ((uintptr_t) at) % BUDDY_ALIGNOF(struct buddy); if (at_alignment != 0) { return NULL; } - size_t main_alignment = ((uintptr_t) main) % BUDDY_ALIGNOF(size_t); + main_alignment = ((uintptr_t) main) % BUDDY_ALIGNOF(size_t); if (main_alignment != 0) { return NULL; } @@ -449,19 +470,19 @@ struct buddy *buddy_init_alignment(unsigned char *at, unsigned char *main, size_ if (memory_size % alignment) { memory_size -= (memory_size % alignment); } - size_t size = buddy_sizeof_alignment(memory_size, alignment); - if (size == 0) { + buddy_size = buddy_sizeof_alignment(memory_size, alignment); + if (buddy_size == 0) { return NULL; } - size_t buddy_tree_order = buddy_tree_order_for_memory(memory_size, alignment); + buddy_tree_order = buddy_tree_order_for_memory(memory_size, alignment); /* TODO check for overlap between buddy metadata and main block */ - struct buddy *buddy = (struct buddy *) at; + buddy = (struct buddy *) at; buddy->arena.main = main; buddy->memory_size = memory_size; buddy->buddy_flags = 0; buddy->alignment = alignment; - buddy_tree_init(buddy->buddy_tree, (uint8_t) buddy_tree_order); + buddy_tree_init((unsigned char *)buddy + sizeof(*buddy), (uint8_t) buddy_tree_order); buddy_toggle_virtual_slots(buddy, 1); return buddy; } @@ -471,18 +492,21 @@ struct buddy *buddy_embed(unsigned char *main, size_t memory_size) { } struct buddy *buddy_embed_alignment(unsigned char *main, size_t memory_size, size_t alignment) { + struct buddy_embed_check check_result; + struct buddy *buddy; + if (! main) { return NULL; } if (!is_valid_alignment(alignment)) { return NULL; /* invalid */ } - struct buddy_embed_check result = buddy_embed_offset(memory_size, alignment); - if (! result.can_fit) { + check_result = buddy_embed_offset(memory_size, alignment); + if (! check_result.can_fit) { return NULL; } - struct buddy *buddy = buddy_init_alignment(main+result.offset, main, result.offset, alignment); + buddy = buddy_init_alignment(main+check_result.offset, main, check_result.offset, alignment); if (! buddy) { /* regular initialization failed */ return NULL; } @@ -505,6 +529,8 @@ struct buddy *buddy_resize(struct buddy *buddy, size_t new_memory_size) { } static struct buddy *buddy_resize_standard(struct buddy *buddy, size_t new_memory_size) { + size_t new_buddy_tree_order; + /* Trim down memory to alignment */ if (new_memory_size % buddy->alignment) { new_memory_size -= (new_memory_size % buddy->alignment); @@ -519,7 +545,7 @@ static struct buddy *buddy_resize_standard(struct buddy *buddy, size_t new_memor buddy_toggle_virtual_slots(buddy, 0); /* Calculate new tree order and resize it */ - size_t new_buddy_tree_order = buddy_tree_order_for_memory(new_memory_size, buddy->alignment); + new_buddy_tree_order = buddy_tree_order_for_memory(new_memory_size, buddy->alignment); buddy_tree_resize(buddy_tree(buddy), (uint8_t) new_buddy_tree_order); /* Store the new memory size and reconstruct any virtual slots */ @@ -531,54 +557,60 @@ static struct buddy *buddy_resize_standard(struct buddy *buddy, size_t new_memor } static struct buddy *buddy_resize_embedded(struct buddy *buddy, size_t new_memory_size) { + struct buddy_embed_check check_result; + unsigned char *main, *buddy_destination; + struct buddy *resized, *relocated; + /* Ensure that the embedded allocator can fit */ - struct buddy_embed_check result = buddy_embed_offset(new_memory_size, buddy->alignment); - if (! result.can_fit) { + check_result = buddy_embed_offset(new_memory_size, buddy->alignment); + if (! check_result.can_fit) { return NULL; } /* Resize the allocator in the normal way */ - struct buddy *resized = buddy_resize_standard(buddy, result.offset); + resized = buddy_resize_standard(buddy, check_result.offset); if (! resized) { return NULL; } /* Get the absolute main address. The relative will be invalid after relocation. */ - unsigned char *main = buddy_main(buddy); + main = buddy_main(buddy); /* Relocate the allocator */ - unsigned char *buddy_destination = buddy_main(buddy) + result.offset; - memmove(buddy_destination, resized, result.buddy_size); + buddy_destination = buddy_main(buddy) + check_result.offset; + memmove(buddy_destination, resized, check_result.buddy_size); /* Update the main offset in the allocator */ - struct buddy *relocated = (struct buddy *) buddy_destination; + relocated = (struct buddy *) buddy_destination; relocated->arena.main_offset = buddy_destination - main; return relocated; } -unsigned int buddy_can_shrink(struct buddy *buddy) { +bool buddy_can_shrink(struct buddy *buddy) { if (buddy == NULL) { - return 0; + return false; } return buddy_is_free(buddy, buddy->memory_size / 2); } -unsigned int buddy_is_empty(struct buddy *buddy) { +bool buddy_is_empty(struct buddy *buddy) { if (buddy == NULL) { - return 1; + return false; } return buddy_is_free(buddy, 0); } -unsigned int buddy_is_full(struct buddy *buddy) { +bool buddy_is_full(struct buddy *buddy) { + struct buddy_tree *tree; + struct buddy_tree_pos pos; + if (buddy == NULL) { - return 0; + return false; } - struct buddy_tree *tree = buddy_tree(buddy); - struct buddy_tree_pos pos = buddy_tree_root(); - size_t status = buddy_tree_status(tree, pos); - return status == buddy_tree_order(tree); + tree = buddy_tree(buddy); + pos = buddy_tree_root(); + return buddy_tree_status(tree, pos) == buddy_tree_order(tree); } size_t buddy_arena_size(struct buddy *buddy) { @@ -618,6 +650,10 @@ static size_t buddy_tree_order_for_memory(size_t memory_size, size_t alignment) } void *buddy_malloc(struct buddy *buddy, size_t requested_size) { + size_t target_depth; + struct buddy_tree *tree; + struct buddy_tree_pos pos; + if (buddy == NULL) { return NULL; } @@ -636,9 +672,9 @@ void *buddy_malloc(struct buddy *buddy, size_t requested_size) { return NULL; } - size_t target_depth = depth_for_size(buddy, requested_size); - struct buddy_tree *tree = buddy_tree(buddy); - struct buddy_tree_pos pos = buddy_tree_find_free(tree, (uint8_t) target_depth); + target_depth = depth_for_size(buddy, requested_size); + tree = buddy_tree(buddy); + pos = buddy_tree_find_free(tree, (uint8_t) target_depth); if (! buddy_tree_valid(tree, pos)) { return NULL; /* no slot found */ @@ -652,6 +688,9 @@ void *buddy_malloc(struct buddy *buddy, size_t requested_size) { } void *buddy_calloc(struct buddy *buddy, size_t members_count, size_t member_size) { + size_t total_size; + void *result; + if (members_count == 0 || member_size == 0) { /* See the gleeful remark in malloc */ members_count = 1; @@ -661,15 +700,20 @@ void *buddy_calloc(struct buddy *buddy, size_t members_count, size_t member_size if (((members_count * member_size)/members_count) != member_size) { return NULL; } - size_t total_size = members_count * member_size; - void *result = buddy_malloc(buddy, total_size); + total_size = members_count * member_size; + result = buddy_malloc(buddy, total_size); if (result) { memset(result, 0, total_size); } return result; } -void *buddy_realloc(struct buddy *buddy, void *ptr, size_t requested_size) { +void *buddy_realloc(struct buddy *buddy, void *ptr, size_t requested_size, bool ignore_data) { + struct buddy_tree *tree; + struct buddy_tree_pos origin, new_pos; + size_t current_depth, target_depth; + void *source, *destination; + /* * realloc is a joke: * - NULL ptr degrades into malloc @@ -691,17 +735,17 @@ void *buddy_realloc(struct buddy *buddy, void *ptr, size_t requested_size) { } /* Find the position tracking this address */ - struct buddy_tree *tree = buddy_tree(buddy); - struct buddy_tree_pos origin = position_for_address(buddy, (unsigned char *) ptr); + tree = buddy_tree(buddy); + origin = position_for_address(buddy, (unsigned char *) ptr); if (! buddy_tree_valid(tree, origin)) { return NULL; } - size_t current_depth = buddy_tree_depth(origin); - size_t target_depth = depth_for_size(buddy, requested_size); + current_depth = buddy_tree_depth(origin); + target_depth = depth_for_size(buddy, requested_size); /* Release the position and perform a search */ buddy_tree_release(tree, origin); - struct buddy_tree_pos new_pos = buddy_tree_find_free(tree, (uint8_t) target_depth); + new_pos = buddy_tree_find_free(tree, (uint8_t) target_depth); if (! buddy_tree_valid(tree, new_pos)) { /* allocation failure, restore mark and return null */ @@ -715,44 +759,52 @@ void *buddy_realloc(struct buddy *buddy, void *ptr, size_t requested_size) { return ptr; } - /* Copy the content */ - void *source = address_for_position(buddy, origin); - void *destination = address_for_position(buddy, new_pos); - memmove(destination, source, size_for_depth(buddy, - current_depth > target_depth ? current_depth : target_depth)); + destination = address_for_position(buddy, new_pos); + + if (! ignore_data) { + /* Copy the content */ + source = address_for_position(buddy, origin); + memmove(destination, source, size_for_depth(buddy, + current_depth > target_depth ? current_depth : target_depth)); + } + /* Allocate and return */ buddy_tree_mark(tree, new_pos); return destination; } void *buddy_reallocarray(struct buddy *buddy, void *ptr, - size_t members_count, size_t member_size) { + size_t members_count, size_t member_size, bool ignore_data) { if (members_count == 0 || member_size == 0) { - return buddy_realloc(buddy, ptr, 0); + return buddy_realloc(buddy, ptr, 0, ignore_data); } /* Check for overflow */ if ((members_count * member_size)/members_count != member_size) { return NULL; } - return buddy_realloc(buddy, ptr, members_count * member_size); + return buddy_realloc(buddy, ptr, members_count * member_size, ignore_data); } void buddy_free(struct buddy *buddy, void *ptr) { + unsigned char *dst, *main; + struct buddy_tree *tree; + struct buddy_tree_pos pos; + if (buddy == NULL) { return; } if (ptr == NULL) { return; } - unsigned char *dst = (unsigned char *)ptr; - unsigned char *main = buddy_main(buddy); + dst = (unsigned char *)ptr; + main = buddy_main(buddy); if ((dst < main) || (dst >= (main + buddy->memory_size))) { return; } /* Find the position tracking this address */ - struct buddy_tree *tree = buddy_tree(buddy); - struct buddy_tree_pos pos = position_for_address(buddy, dst); + tree = buddy_tree(buddy); + pos = position_for_address(buddy, dst); if (! buddy_tree_valid(tree, pos)) { return; @@ -763,27 +815,32 @@ void buddy_free(struct buddy *buddy, void *ptr) { } void buddy_safe_free(struct buddy *buddy, void *ptr, size_t requested_size) { + unsigned char *dst, *main; + struct buddy_tree *tree; + struct buddy_tree_pos pos; + size_t allocated_size_for_depth; + if (buddy == NULL) { return; } if (ptr == NULL) { return; } - unsigned char *dst = (unsigned char *)ptr; - unsigned char *main = buddy_main(buddy); + dst = (unsigned char *)ptr; + main = buddy_main(buddy); if ((dst < main) || (dst >= (main + buddy->memory_size))) { return; } /* Find the position tracking this address */ - struct buddy_tree *tree = buddy_tree(buddy); - struct buddy_tree_pos pos = position_for_address(buddy, dst); + tree = buddy_tree(buddy); + pos = position_for_address(buddy, dst); if (! buddy_tree_valid(tree, pos)) { return; } - size_t allocated_size_for_depth = size_for_depth(buddy, pos.depth); + allocated_size_for_depth = size_for_depth(buddy, pos.depth); if (requested_size < buddy->alignment) { requested_size = buddy->alignment; } @@ -807,62 +864,70 @@ void buddy_unsafe_release_range(struct buddy *buddy, void *ptr, size_t requested } void *buddy_walk(struct buddy *buddy, - void *(fp)(void *ctx, void *addr, size_t slot_size), void *ctx) { + void *(fp)(void *ctx, void *addr, size_t slot_size, size_t allocated), + void *ctx) { + unsigned char *main; + size_t effective_memory_size, tree_order, pos_status, pos_size; + struct buddy_tree *tree; + unsigned char *addr; + struct buddy_tree_walk_state state; + struct buddy_tree_pos test_pos; + void *callback_result; + if (buddy == NULL) { return NULL; } if (fp == NULL) { return NULL; } - unsigned char *main = buddy_main(buddy); - size_t effective_memory_size = buddy_effective_memory_size(buddy); - struct buddy_tree *tree = buddy_tree(buddy); - size_t tree_order = buddy_tree_order(tree); + main = buddy_main(buddy); + effective_memory_size = buddy_effective_memory_size(buddy); + tree = buddy_tree(buddy); + tree_order = buddy_tree_order(tree); - struct buddy_tree_walk_state state = buddy_tree_walk_state_root(); + state = buddy_tree_walk_state_root(); do { - size_t pos_status = buddy_tree_status(tree, state.current_pos); - if (pos_status == 0) { /* Empty position */ - state.going_up = 1; - } else if (pos_status != (tree_order - state.current_pos.depth + 1)) { /* Partially-allocated */ + pos_status = buddy_tree_status(tree, state.current_pos); + if (pos_status != (tree_order - state.current_pos.depth + 1)) { /* Partially-allocated */ + continue; + } + + /* + * The tree doesn't make a distinction of a fully-allocated node + * due to a single allocation and a fully-allocated due to maxed out + * child allocations - we need to check the children. + * A child-allocated node will have both children set to their maximum + * but it is sufficient to check just one for non-zero. + */ + test_pos = buddy_tree_left_child(state.current_pos); + if (buddy_tree_valid(tree, test_pos) && buddy_tree_status(tree, test_pos)) { continue; - } else { /* Fully-allocated */ + } + + /* Current node is free or allocated, process */ + pos_size = effective_memory_size >> (state.current_pos.depth - 1u); + addr = address_for_position(buddy, state.current_pos); + if (((size_t)(addr - main) + pos_size) > buddy->memory_size) { /* - * The tree doesn't make a distinction of a fully-allocated node - * due to a single allocation and a fully-allocated due to maxed out - * child allocations - we need to check the children. - * A child-allocated node will have both children set to their maximum - * but it is sufficient to check just one for non-zero. + * Do not process virtual slots + * As virtual slots are on the right side of the tree + * if we see a one with the current iteration order this + * means that all subsequent slots will be virtual, + * hence we can return early. */ - struct buddy_tree_pos left = buddy_tree_left_child(state.current_pos); - if (buddy_tree_valid(tree, left) && buddy_tree_status(tree, left)) { - continue; - } - - /* Current node is allocated, process */ - size_t pos_size = effective_memory_size >> (state.current_pos.depth - 1u); - unsigned char *addr = address_for_position(buddy, state.current_pos); - if (((size_t)(addr - main) + pos_size) > buddy->memory_size) { - /* - * Do not process virtual slots - * As virtual slots are on the right side of the tree - * if we see a one with the current iteration order this - * means that all subsequent slots will be virtual, - * hence we can return early. - */ - return NULL; - } else { - void *result = (fp)(ctx, addr, pos_size); - if (result != NULL) { - return result; - } - } + return NULL; + } + callback_result = (fp)(ctx, addr, pos_size, pos_status > 0); + if (callback_result != NULL) { + return callback_result; } + state.going_up = 1; + } while (buddy_tree_walk(tree, &state)); return NULL; } -float buddy_fragmentation(struct buddy *buddy) { +unsigned char buddy_fragmentation(struct buddy *buddy) { if (buddy == NULL) { return 0; } @@ -870,11 +935,12 @@ float buddy_fragmentation(struct buddy *buddy) { } static size_t depth_for_size(struct buddy *buddy, size_t requested_size) { + size_t depth, effective_memory_size; if (requested_size < buddy->alignment) { requested_size = buddy->alignment; } - size_t depth = 1; - size_t effective_memory_size = buddy_effective_memory_size(buddy); + depth = 1; + effective_memory_size = buddy_effective_memory_size(buddy); while ((effective_memory_size / requested_size) >> 1u) { depth++; effective_memory_size >>= 1u; @@ -887,7 +953,7 @@ static inline size_t size_for_depth(struct buddy *buddy, size_t depth) { } static struct buddy_tree *buddy_tree(struct buddy *buddy) { - return (struct buddy_tree*) buddy->buddy_tree; + return (struct buddy_tree*) ((unsigned char *)buddy + sizeof(*buddy)); } static size_t buddy_effective_memory_size(struct buddy *buddy) { @@ -900,8 +966,7 @@ static size_t buddy_virtual_slots(struct buddy *buddy) { if (effective_memory_size == memory_size) { return 0; } - size_t delta = effective_memory_size - memory_size; - return delta / buddy->alignment; + return (effective_memory_size - memory_size) / buddy->alignment; } static unsigned char *address_for_position(struct buddy *buddy, struct buddy_tree_pos pos) { @@ -918,16 +983,20 @@ static struct buddy_tree_pos deepest_position_for_offset(struct buddy *buddy, si } static struct buddy_tree_pos position_for_address(struct buddy *buddy, const unsigned char *addr) { - /* Find the deepest position tracking this address */ - unsigned char *main = buddy_main(buddy); - size_t offset = (size_t) (addr - main); + unsigned char *main; + struct buddy_tree *tree; + struct buddy_tree_pos pos; + size_t offset; + + main = buddy_main(buddy); + offset = (size_t) (addr - main); if (offset % buddy->alignment) { return INVALID_POS; /* invalid alignment */ } - struct buddy_tree *tree = buddy_tree(buddy); - struct buddy_tree_pos pos = deepest_position_for_offset(buddy, offset); + tree = buddy_tree(buddy); + pos = deepest_position_for_offset(buddy, offset); /* Find the actual allocated position tracking this address */ while (!buddy_tree_status(tree, pos)) { @@ -953,27 +1022,31 @@ static unsigned char *buddy_main(struct buddy *buddy) { } static unsigned int buddy_relative_mode(struct buddy *buddy) { - return buddy->buddy_flags & BUDDY_RELATIVE_MODE; + return (unsigned int)buddy->buddy_flags & BUDDY_RELATIVE_MODE; } static void buddy_toggle_virtual_slots(struct buddy *buddy, unsigned int state) { - size_t memory_size = buddy->memory_size; + size_t delta, memory_size, effective_memory_size; + void (*toggle)(struct buddy_tree *, struct buddy_tree_pos); + struct buddy_tree *tree; + struct buddy_tree_pos pos; + + memory_size = buddy->memory_size; /* Mask/unmask the virtual space if memory is not a power of two */ - size_t effective_memory_size = buddy_effective_memory_size(buddy); + effective_memory_size = buddy_effective_memory_size(buddy); if (effective_memory_size == memory_size) { return; } /* Get the area that we need to mask and pad it to alignment */ /* Node memory size is already aligned to buddy->alignment */ - size_t delta = effective_memory_size - memory_size; + delta = effective_memory_size - memory_size; /* Determine whether to mark or release */ - void (*toggle)(struct buddy_tree *, struct buddy_tree_pos) = - state ? &buddy_tree_mark : &buddy_tree_release; + toggle = state ? &buddy_tree_mark : &buddy_tree_release; - struct buddy_tree *tree = buddy_tree(buddy); - struct buddy_tree_pos pos = buddy_tree_right_child(buddy_tree_root()); + tree = buddy_tree(buddy); + pos = buddy_tree_right_child(buddy_tree_root()); while (delta) { size_t current_pos_size = size_for_depth(buddy, buddy_tree_depth(pos)); if (delta == current_pos_size) { @@ -998,6 +1071,12 @@ static void buddy_toggle_virtual_slots(struct buddy *buddy, unsigned int state) } static void buddy_toggle_range_reservation(struct buddy *buddy, void *ptr, size_t requested_size, unsigned int state) { + unsigned char *dst, *main; + void (*toggle)(struct buddy_tree *, struct buddy_tree_pos); + struct buddy_tree *tree; + size_t offset; + struct buddy_tree_pos pos; + if (buddy == NULL) { return; } @@ -1007,20 +1086,19 @@ static void buddy_toggle_range_reservation(struct buddy *buddy, void *ptr, size_ if (requested_size == 0) { return; } - unsigned char *dst = (unsigned char *)ptr; - unsigned char *main = buddy_main(buddy); + dst = (unsigned char *)ptr; + main = buddy_main(buddy); if ((dst < main) || ((dst + requested_size) > (main + buddy->memory_size))) { return; } /* Determine whether to mark or release */ - void (*toggle)(struct buddy_tree *, struct buddy_tree_pos) = - state ? &buddy_tree_mark : &buddy_tree_release; + toggle = state ? &buddy_tree_mark : &buddy_tree_release; /* Find the deepest position tracking this address */ - struct buddy_tree *tree = buddy_tree(buddy); - size_t offset = (size_t) (dst - main); - struct buddy_tree_pos pos = deepest_position_for_offset(buddy, offset); + tree = buddy_tree(buddy); + offset = (size_t) (dst - main); + pos = deepest_position_for_offset(buddy, offset); /* Advance one position at a time and process */ while (requested_size) { @@ -1034,75 +1112,84 @@ static void buddy_toggle_range_reservation(struct buddy *buddy, void *ptr, size_ /* Internal function that checks if there are any allocations after the indicated relative memory index. Used to check if - the arena can be downsized. */ -static unsigned int buddy_is_free(struct buddy *buddy, size_t from) { - /* from is already adjusted for alignment */ - - size_t effective_memory_size = buddy_effective_memory_size(buddy); - size_t virtual_slots = buddy_virtual_slots(buddy); - size_t to = effective_memory_size - + the arena can be downsized. + The from argument is already adjusted for alignment by caller */ +static bool buddy_is_free(struct buddy *buddy, size_t from) { + struct buddy_tree *tree; + struct buddy_tree_interval query_range; + struct buddy_tree_pos pos; + size_t effective_memory_size, virtual_slots, to; + + effective_memory_size = buddy_effective_memory_size(buddy); + virtual_slots = buddy_virtual_slots(buddy); + to = effective_memory_size - ((virtual_slots ? virtual_slots : 1) * buddy->alignment); - struct buddy_tree *t = buddy_tree(buddy); + tree = buddy_tree(buddy); - struct buddy_tree_interval query_range = {0}; query_range.from = deepest_position_for_offset(buddy, from); query_range.to = deepest_position_for_offset(buddy, to); - struct buddy_tree_pos pos = deepest_position_for_offset(buddy, from); - while(buddy_tree_valid(t, pos) && (pos.index < query_range.to.index)) { - struct buddy_tree_interval current_test_range = buddy_tree_interval(t, pos); + pos = deepest_position_for_offset(buddy, from); + while(buddy_tree_valid(tree, pos) && (pos.index < query_range.to.index)) { + struct buddy_tree_interval current_test_range = buddy_tree_interval(tree, pos); struct buddy_tree_interval parent_test_range = - buddy_tree_interval(t, buddy_tree_parent(pos)); + buddy_tree_interval(tree, buddy_tree_parent(pos)); while(buddy_tree_interval_contains(query_range, parent_test_range)) { pos = buddy_tree_parent(pos); current_test_range = parent_test_range; - parent_test_range = buddy_tree_interval(t, buddy_tree_parent(pos)); + parent_test_range = buddy_tree_interval(tree, buddy_tree_parent(pos)); } /* pos is now tracking an overlapping segment */ - if (! buddy_tree_is_free(t, pos)) { - return 0; + if (! buddy_tree_is_free(tree, pos)) { + return false; } /* Advance check */ pos = buddy_tree_right_adjacent(current_test_range.to); } - return 1; + return true; } static struct buddy_embed_check buddy_embed_offset(size_t memory_size, size_t alignment) { - struct buddy_embed_check result = {0}; - result.can_fit = 1; + size_t buddy_size, offset; + struct buddy_embed_check check_result; - size_t buddy_size = buddy_sizeof_alignment(memory_size, alignment); + memset(&check_result, 0, sizeof(check_result)); + check_result.can_fit = 1; + buddy_size = buddy_sizeof_alignment(memory_size, alignment); if (buddy_size >= memory_size) { - result.can_fit = 0; + check_result.can_fit = 0; } - size_t offset = memory_size - buddy_size; + offset = memory_size - buddy_size; if (offset % BUDDY_ALIGNOF(struct buddy) != 0) { buddy_size += offset % BUDDY_ALIGNOF(struct buddy); if (buddy_size >= memory_size) { - result.can_fit = 0; + check_result.can_fit = 0; } offset = memory_size - buddy_size; } - if (result.can_fit) { - result.offset = offset; - result.buddy_size = buddy_size; + if (check_result.can_fit) { + check_result.offset = offset; + check_result.buddy_size = buddy_size; } - return result; + return check_result; } -static void buddy_debug(FILE *stream, struct buddy *buddy) { - fprintf(stream, "buddy allocator at: %p arena at: %p\n", (void *)buddy, (void *)buddy_main(buddy)); - fprintf(stream, "memory size: %zu\n", buddy->memory_size); - fprintf(stream, "mode: "); - fprintf(stream, buddy_relative_mode(buddy) ? "embedded" : "standard"); - fprintf(stream, "\n"); - fprintf(stream, "virtual slots: %zu\n", buddy_virtual_slots(buddy)); - fprintf(stream, "allocator tree follows:\n"); - buddy_tree_debug(stream, buddy_tree(buddy), buddy_tree_root(), buddy_effective_memory_size(buddy)); +void buddy_debug(struct buddy *buddy) { + BUDDY_PRINTF("buddy allocator at: %p arena at: %p\n", (void *)buddy, (void *)buddy_main(buddy)); + BUDDY_PRINTF("memory size: %zu\n", buddy->memory_size); + BUDDY_PRINTF("mode: "); + if (buddy_relative_mode(buddy)) { + BUDDY_PRINTF("embedded"); + } else { + BUDDY_PRINTF("standard"); + } + BUDDY_PRINTF("\n"); + BUDDY_PRINTF("virtual slots: %zu\n", buddy_virtual_slots(buddy)); + BUDDY_PRINTF("allocator tree follows:\n"); + buddy_tree_debug(buddy_tree(buddy), buddy_tree_root(), buddy_effective_memory_size(buddy)); } /* @@ -1113,7 +1200,6 @@ struct buddy_tree { size_t upper_pos_bound; size_t size_for_order_offset; uint8_t order; - size_t data[]; }; struct internal_position { @@ -1137,7 +1223,7 @@ static void buddy_tree_populate_size_for_order(struct buddy_tree *t); static inline size_t buddy_tree_size_for_order(struct buddy_tree *t, uint8_t to); static void write_to_internal_position(unsigned char *bitset, struct internal_position pos, size_t value); static size_t read_from_internal_position(unsigned char *bitset, struct internal_position pos); -static size_t compare_with_internal_position(unsigned char *bitset, struct internal_position pos, size_t value); +static inline unsigned char compare_with_internal_position(unsigned char *bitset, struct internal_position pos, size_t value); static inline size_t size_for_order(uint8_t order, uint8_t to) { size_t result = 0; @@ -1152,33 +1238,39 @@ static inline size_t size_for_order(uint8_t order, uint8_t to) { static inline struct internal_position buddy_tree_internal_position_order( size_t tree_order, struct buddy_tree_pos pos) { - struct internal_position p = {0}; + struct internal_position p; + size_t total_offset, local_index; + p.local_offset = tree_order - buddy_tree_depth(pos) + 1; - size_t total_offset = size_for_order((uint8_t) tree_order, (uint8_t) p.local_offset); - size_t local_index = buddy_tree_index_internal(pos); + total_offset = size_for_order((uint8_t) tree_order, (uint8_t) p.local_offset); + local_index = buddy_tree_index_internal(pos); p.bitset_location = total_offset + (p.local_offset * local_index); return p; } static inline struct internal_position buddy_tree_internal_position_tree( struct buddy_tree *t, struct buddy_tree_pos pos) { - struct internal_position p = {0}; + struct internal_position p; + size_t total_offset, local_index; + p.local_offset = t->order - buddy_tree_depth(pos) + 1; - size_t total_offset = buddy_tree_size_for_order(t, (uint8_t) p.local_offset); - size_t local_index = buddy_tree_index_internal(pos); + total_offset = buddy_tree_size_for_order(t, (uint8_t) p.local_offset); + local_index = buddy_tree_index_internal(pos); p.bitset_location = total_offset + (p.local_offset * local_index); return p; } static size_t buddy_tree_sizeof(uint8_t order) { - size_t tree_size = sizeof(struct buddy_tree); + size_t tree_size, bitset_size, size_for_order_size; + + tree_size = sizeof(struct buddy_tree); /* Account for the bitset */ - size_t bitset_size = bitset_sizeof(size_for_order(order, 0)); + bitset_size = bitset_sizeof(size_for_order(order, 0)); if (bitset_size % sizeof(size_t)) { bitset_size += (bitset_size % sizeof(size_t)); } /* Account for the size_for_order memoization */ - size_t size_for_order_size = ((order+2) * sizeof(size_t)); + size_for_order_size = ((order+2) * sizeof(size_t)); return tree_size + bitset_size + size_for_order_size; } @@ -1187,7 +1279,7 @@ static struct buddy_tree *buddy_tree_init(unsigned char *at, uint8_t order) { struct buddy_tree *t = (struct buddy_tree*) at; memset(at, 0, size); t->order = order; - t->upper_pos_bound = 1u << t->order; + t->upper_pos_bound = two_to_the_power_of(t->order); buddy_tree_populate_size_for_order(t); return t; } @@ -1204,6 +1296,8 @@ static void buddy_tree_resize(struct buddy_tree *t, uint8_t desired_order) { } static void buddy_tree_grow(struct buddy_tree *t, uint8_t desired_order) { + struct buddy_tree_pos pos; + while (desired_order > t->order) { /* Grow the tree a single order at a time */ size_t current_order = t->order; @@ -1217,7 +1311,7 @@ static void buddy_tree_grow(struct buddy_tree *t, uint8_t desired_order) { t->order + 1u, next_pos); /* There are this many nodes at the current level */ - size_t node_count = 1u << (current_order - 1u); + size_t node_count = two_to_the_power_of(current_order - 1u); /* Transfer the bits*/ bitset_shift_right(buddy_tree_bits(t), @@ -1237,35 +1331,37 @@ static void buddy_tree_grow(struct buddy_tree *t, uint8_t desired_order) { } /* Advance the order and refresh the root */ t->order += 1u; - t->upper_pos_bound = 1u << t->order; + t->upper_pos_bound = two_to_the_power_of(t->order); buddy_tree_populate_size_for_order(t); /* Update the root */ - struct buddy_tree_pos right = buddy_tree_right_child(buddy_tree_root()); - update_parent_chain(t, right, buddy_tree_internal_position_tree(t, right), 0); + pos = buddy_tree_right_child(buddy_tree_root()); + update_parent_chain(t, pos, buddy_tree_internal_position_tree(t, pos), 0); } } static void buddy_tree_shrink(struct buddy_tree *t, uint8_t desired_order) { + size_t current_order, next_order, node_count; + struct buddy_tree_pos left_start; + struct internal_position current_internal, next_internal; + while (desired_order < t->order) { if (!buddy_tree_can_shrink(t)) { return; } /* Shrink the tree a single order at a time */ - size_t current_order = t->order; - size_t next_order = current_order - 1; + current_order = t->order; + next_order = current_order - 1; - struct buddy_tree_pos left_start = buddy_tree_left_child(buddy_tree_root()); + left_start = buddy_tree_left_child(buddy_tree_root()); while(buddy_tree_valid(t, left_start)) { /* Get handles into the rows at the tracked depth */ - struct internal_position current_internal = buddy_tree_internal_position_order( - current_order, left_start); - struct internal_position next_internal = buddy_tree_internal_position_order( - next_order, buddy_tree_parent(left_start)); + current_internal = buddy_tree_internal_position_order(current_order, left_start); + next_internal = buddy_tree_internal_position_order(next_order, buddy_tree_parent(left_start)); /* There are this many nodes at the current level */ - size_t node_count = 1u << (left_start.depth - 1u); + node_count = two_to_the_power_of(left_start.depth - 1u); /* Transfer the bits*/ bitset_shift_left(buddy_tree_bits(t), @@ -1279,12 +1375,12 @@ static void buddy_tree_shrink(struct buddy_tree *t, uint8_t desired_order) { /* Advance the order */ t->order = (uint8_t) next_order; - t->upper_pos_bound = 1u << t->order; + t->upper_pos_bound = two_to_the_power_of(t->order); buddy_tree_populate_size_for_order(t); } } -static unsigned int buddy_tree_valid(struct buddy_tree *t, struct buddy_tree_pos pos) { +static bool buddy_tree_valid(struct buddy_tree *t, struct buddy_tree_pos pos) { return pos.index && (pos.index < t->upper_pos_bound); } @@ -1303,7 +1399,7 @@ static struct buddy_tree_pos buddy_tree_leftmost_child(struct buddy_tree *t) { static struct buddy_tree_pos buddy_tree_leftmost_child_internal(size_t tree_order) { struct buddy_tree_pos result; - result.index = 1u << (tree_order - 1u); + result.index = two_to_the_power_of(tree_order - 1u); result.depth = tree_order; return result; } @@ -1351,13 +1447,13 @@ static size_t buddy_tree_index(struct buddy_tree_pos pos) { static inline size_t buddy_tree_index_internal(struct buddy_tree_pos pos) { /* Clear out the highest bit, this gives us the index * in a row of sibling nodes */ - size_t mask = 1u << (pos.depth - 1u); + size_t mask = two_to_the_power_of(pos.depth - 1u); size_t result = pos.index & ~mask; return result; } static inline unsigned char *buddy_tree_bits(struct buddy_tree *t) { - return (unsigned char *) t->data; + return ((unsigned char *) t) + sizeof(*t); } static void buddy_tree_populate_size_for_order(struct buddy_tree *t) { @@ -1368,13 +1464,13 @@ static void buddy_tree_populate_size_for_order(struct buddy_tree *t) { t->size_for_order_offset = bitset_offset / sizeof(size_t); t->size_for_order_offset++; for (size_t i = 0; i <= t->order; i++) { - *(t->data+t->size_for_order_offset+i) = size_for_order(t->order, (uint8_t) i); + *((size_t *)(((unsigned char *) t) + sizeof(*t)) + t->size_for_order_offset + i) = size_for_order(t->order, (uint8_t) i); } } static inline size_t buddy_tree_size_for_order(struct buddy_tree *t, uint8_t to) { - return *(t->data+t->size_for_order_offset+to); + return *((size_t *)(((unsigned char *) t) + sizeof(*t)) + t->size_for_order_offset + to); } static void write_to_internal_position(unsigned char *bitset, struct internal_position pos, size_t value) { @@ -1393,15 +1489,17 @@ static size_t read_from_internal_position(unsigned char *bitset, struct internal return bitset_count_range(bitset, pos.bitset_location, pos.bitset_location+pos.local_offset-1); } -static size_t compare_with_internal_position(unsigned char *bitset, struct internal_position pos, size_t value) { +static inline unsigned char compare_with_internal_position(unsigned char *bitset, struct internal_position pos, size_t value) { return bitset_test(bitset, pos.bitset_location+value-1); } static struct buddy_tree_interval buddy_tree_interval(struct buddy_tree *t, struct buddy_tree_pos pos) { - struct buddy_tree_interval result = {0}; + struct buddy_tree_interval result; + size_t depth; + result.from = pos; result.to = pos; - size_t depth = pos.depth; + depth = pos.depth; while (depth != t->order) { result.from = buddy_tree_left_child(result.from); result.to = buddy_tree_right_child(result.to); @@ -1410,7 +1508,7 @@ static struct buddy_tree_interval buddy_tree_interval(struct buddy_tree *t, stru return result; } -static unsigned int buddy_tree_interval_contains(struct buddy_tree_interval outer, +static bool buddy_tree_interval_contains(struct buddy_tree_interval outer, struct buddy_tree_interval inner) { return (inner.from.index >= outer.from.index) && (inner.from.index <= outer.to.index) @@ -1419,7 +1517,8 @@ static unsigned int buddy_tree_interval_contains(struct buddy_tree_interval oute } static struct buddy_tree_walk_state buddy_tree_walk_state_root(void) { - struct buddy_tree_walk_state state = {0}; + struct buddy_tree_walk_state state; + memset(&state, 0, sizeof(state)); state.starting_pos = buddy_tree_root(); state.current_pos = buddy_tree_root(); return state; @@ -1480,17 +1579,19 @@ static void buddy_tree_release(struct buddy_tree *t, struct buddy_tree_pos pos) static void update_parent_chain(struct buddy_tree *t, struct buddy_tree_pos pos, struct internal_position pos_internal, size_t size_current) { + size_t size_sibling, size_parent, target_parent; unsigned char *bits = buddy_tree_bits(t); + while (pos.index != 1) { pos_internal.bitset_location += pos_internal.local_offset - (2 * pos_internal.local_offset * (pos.index & 1u)); - size_t size_sibling = read_from_internal_position(bits, pos_internal); + size_sibling = read_from_internal_position(bits, pos_internal); pos = buddy_tree_parent(pos); pos_internal = buddy_tree_internal_position_tree(t, pos); - size_t size_parent = read_from_internal_position(bits, pos_internal); + size_parent = read_from_internal_position(bits, pos_internal); - size_t target_parent = (size_current || size_sibling) + target_parent = (size_current || size_sibling) * ((size_current <= size_sibling ? size_current : size_sibling) + 1); if (target_parent == size_parent) { return; @@ -1502,35 +1603,41 @@ static void update_parent_chain(struct buddy_tree *t, struct buddy_tree_pos pos, } static struct buddy_tree_pos buddy_tree_find_free(struct buddy_tree *t, uint8_t target_depth) { - struct buddy_tree_pos current_pos = buddy_tree_root(); - uint8_t target_status = target_depth - 1; - size_t current_depth = buddy_tree_depth(current_pos); + struct buddy_tree_pos current_pos, left_pos, right_pos; + uint8_t target_status; + size_t current_depth, right_status; + struct internal_position left_internal, right_internal; + unsigned char *tree_bits; + + current_pos = buddy_tree_root(); + target_status = target_depth - 1; + current_depth = buddy_tree_depth(current_pos); if (buddy_tree_status(t, current_pos) > target_status) { return INVALID_POS; /* No position available down the tree */ } + tree_bits = buddy_tree_bits(t); while (current_depth != target_depth) { /* Advance criteria */ target_status -= 1; current_depth += 1; - struct buddy_tree_pos left_pos = buddy_tree_left_child(current_pos); - struct buddy_tree_pos right_pos = buddy_tree_sibling(left_pos); + left_pos = buddy_tree_left_child(current_pos); + right_pos = buddy_tree_sibling(left_pos); - struct internal_position left_internal = buddy_tree_internal_position_tree(t, left_pos); + left_internal = buddy_tree_internal_position_tree(t, left_pos); - struct internal_position right_internal = left_internal; + right_internal = left_internal; right_internal.bitset_location += right_internal.local_offset; /* advance to the right */ - if (compare_with_internal_position(buddy_tree_bits(t), left_internal, target_status+1)) { /* left branch is busy, pick right */ + if (compare_with_internal_position(tree_bits, left_internal, target_status+1)) { /* left branch is busy, pick right */ current_pos = right_pos; - } else if (compare_with_internal_position(buddy_tree_bits(t), right_internal, target_status+1)) { /* right branch is busy, pick left */ + } else if (compare_with_internal_position(tree_bits, right_internal, target_status+1)) { /* right branch is busy, pick left */ current_pos = left_pos; } else { /* One of the child nodes must be read in order to compare it to its sibling. */ - right_internal.local_offset = target_status; /* reduce the read span since we know the right_status is equal or less than target_status */ - size_t right_status = read_from_internal_position(buddy_tree_bits(t), right_internal); + right_status = read_from_internal_position(tree_bits, right_internal); if (right_status) { - if (compare_with_internal_position(buddy_tree_bits(t), left_internal, right_status)) { + if (compare_with_internal_position(tree_bits, left_internal, right_status)) { current_pos = left_pos; /* Left is equal or more busy than right, prefer left */ } else { current_pos = right_pos; @@ -1543,9 +1650,9 @@ static struct buddy_tree_pos buddy_tree_find_free(struct buddy_tree *t, uint8_t return current_pos; } -static unsigned int buddy_tree_is_free(struct buddy_tree *t, struct buddy_tree_pos pos) { +static bool buddy_tree_is_free(struct buddy_tree *t, struct buddy_tree_pos pos) { if (buddy_tree_status(t, pos)) { - return 0; + return false; } pos = buddy_tree_parent(pos); while(buddy_tree_valid(t, pos)) { @@ -1556,23 +1663,25 @@ static unsigned int buddy_tree_is_free(struct buddy_tree *t, struct buddy_tree_p } pos = buddy_tree_parent(pos); } - return 1; + return true; } -static unsigned int buddy_tree_can_shrink(struct buddy_tree *t) { +static bool buddy_tree_can_shrink(struct buddy_tree *t) { + struct internal_position root_internal; + size_t root_value; + if (buddy_tree_status(t, buddy_tree_right_child(buddy_tree_root())) != 0) { - return 0; /* Refusing to shrink with right subtree still used! */ + return false; /* Refusing to shrink with right subtree still used! */ } - struct internal_position root_internal = - buddy_tree_internal_position_tree(t, buddy_tree_root()); - size_t root_value = read_from_internal_position(buddy_tree_bits(t), root_internal); + root_internal = buddy_tree_internal_position_tree(t, buddy_tree_root()); + root_value = read_from_internal_position(buddy_tree_bits(t), root_internal); if (root_value == root_internal.local_offset) { - return 0; /* Refusing to shrink with the root fully-allocated! */ + return false; /* Refusing to shrink with the root fully-allocated! */ } - return 1; + return true; } -static void buddy_tree_debug(FILE *stream, struct buddy_tree *t, struct buddy_tree_pos pos, +static void buddy_tree_debug(struct buddy_tree *t, struct buddy_tree_pos pos, size_t start_size) { struct buddy_tree_walk_state state = buddy_tree_walk_state_root(); state.current_pos = pos; @@ -1580,20 +1689,20 @@ static void buddy_tree_debug(FILE *stream, struct buddy_tree *t, struct buddy_tr struct internal_position pos_internal = buddy_tree_internal_position_tree(t, state.current_pos); size_t pos_status = read_from_internal_position(buddy_tree_bits(t), pos_internal); size_t pos_size = start_size >> ((buddy_tree_depth(state.current_pos) - 1u) % ((sizeof(size_t) * CHAR_BIT)-1)); - fprintf(stream, "%.*s", + BUDDY_PRINTF("%.*s", (int) buddy_tree_depth(state.current_pos), " "); - fprintf(stream, "pos index: %zu pos depth: %zu status: %zu bitset-len: %zu bitset-at: %zu", + BUDDY_PRINTF("pos index: %zu pos depth: %zu status: %zu bitset-len: %zu bitset-at: %zu", state.current_pos.index, state.current_pos.depth, pos_status, pos_internal.local_offset, pos_internal.bitset_location); if (pos_status == pos_internal.local_offset) { - fprintf(stream, " size: %zu", pos_size); + BUDDY_PRINTF(" size: %zu", pos_size); } - fprintf(stream, "\n"); + BUDDY_PRINTF("\n"); } while (buddy_tree_walk(t, &state)); } -static unsigned int buddy_tree_check_invariant(struct buddy_tree *t, struct buddy_tree_pos pos) { +unsigned int buddy_tree_check_invariant(struct buddy_tree *t, struct buddy_tree_pos pos) { unsigned int fail = 0; struct buddy_tree_walk_state state = buddy_tree_walk_state_root(); state.current_pos = pos; @@ -1618,8 +1727,8 @@ static unsigned int buddy_tree_check_invariant(struct buddy_tree *t, struct budd if (violated) { fail = 1; - fprintf(stdout, "invariant violation at position [ index: %zu depth: %zu ]!\n", pos.index, pos.depth); - fprintf(stdout, "current: %zu left %zu right %zu max %zu\n", + BUDDY_PRINTF("invariant violation at position [ index: %zu depth: %zu ]!\n", pos.index, pos.depth); + BUDDY_PRINTF("current: %zu left %zu right %zu max %zu\n", current_status, left_child_status, right_child_status, current_internal.local_offset); } @@ -1631,22 +1740,29 @@ static unsigned int buddy_tree_check_invariant(struct buddy_tree *t, struct budd * Calculate tree fragmentation based on free slots. * Based on https://asawicki.info/news_1757_a_metric_for_memory_fragmentation */ -static float buddy_tree_fragmentation(struct buddy_tree *t) { - uint8_t tree_order = buddy_tree_order(t); - size_t root_status = buddy_tree_status(t, buddy_tree_root()); +static unsigned char buddy_tree_fragmentation(struct buddy_tree *t) { + const size_t fractional_bits = 8; + const size_t fractional_mask = 255; + + uint8_t tree_order; + size_t root_status, quality, total_free_size, virtual_size, quality_percent; + struct buddy_tree_walk_state state; + + tree_order = buddy_tree_order(t); + root_status = buddy_tree_status(t, buddy_tree_root()); if (root_status == 0) { /* Emptry tree */ return 0; } - size_t quality = 0; - size_t total_free_size = 0; + quality = 0; + total_free_size = 0; - struct buddy_tree_walk_state state = buddy_tree_walk_state_root(); + state = buddy_tree_walk_state_root(); do { size_t pos_status = buddy_tree_status(t, state.current_pos); if (pos_status == 0) { /* Empty node, process */ - size_t virtual_size = 1ul << ((tree_order - state.current_pos.depth) % ((sizeof(size_t) * CHAR_BIT)-1)); + virtual_size = two_to_the_power_of((tree_order - state.current_pos.depth) % ((sizeof(size_t) * CHAR_BIT)-1)); quality += (virtual_size * virtual_size); total_free_size += virtual_size; /* Ascend */ @@ -1661,9 +1777,10 @@ static float buddy_tree_fragmentation(struct buddy_tree *t) { return 0; } - float quality_percent = approximate_square_root((float) quality) / (float) total_free_size; - float fragmentation = 1 - (quality_percent * quality_percent); - return fragmentation; + quality_percent = (integer_square_root(quality) << fractional_bits) / total_free_size; + quality_percent *= quality_percent; + quality_percent >>= fractional_bits; + return fractional_mask - (quality_percent & fractional_mask); } /* @@ -1688,7 +1805,7 @@ static inline void bitset_clear(unsigned char *bitset, size_t pos) { bitset[bucket] &= ~bitset_index_mask[index]; } -static inline unsigned int bitset_test(const unsigned char *bitset, size_t pos) { +static inline bool bitset_test(const unsigned char *bitset, size_t pos) { size_t bucket = pos / CHAR_BIT; size_t index = pos % CHAR_BIT; return bitset[bucket] & bitset_index_mask[index]; @@ -1742,6 +1859,8 @@ static void bitset_set_range(unsigned char *bitset, size_t from_pos, size_t to_p } static size_t bitset_count_range(unsigned char *bitset, size_t from_pos, size_t to_pos) { + size_t result; + size_t from_bucket = from_pos / CHAR_BIT; size_t to_bucket = to_pos / CHAR_BIT; @@ -1752,7 +1871,7 @@ static size_t bitset_count_range(unsigned char *bitset, size_t from_pos, size_t return popcount_byte(bitset[from_bucket] & bitset_char_mask[from_index][to_index]); } - size_t result = popcount_byte(bitset[from_bucket] & bitset_char_mask[from_index][7]) + result = popcount_byte(bitset[from_bucket] & bitset_char_mask[from_index][7]) + popcount_byte(bitset[to_bucket] & bitset_char_mask[0][to_index]); while(++from_bucket != to_bucket) { result += popcount_byte(bitset[from_bucket]); @@ -1788,9 +1907,9 @@ static void bitset_shift_right(unsigned char *bitset, size_t from_pos, size_t to bitset_clear_range(bitset, from_pos, from_pos+by-1); } -static void bitset_debug(FILE *stream, unsigned char *bitset, size_t length) { +void bitset_debug(unsigned char *bitset, size_t length) { for (size_t i = 0; i < length; i++) { - fprintf(stream, "%zu: %d\n", i, bitset_test(bitset, i) && 1); + BUDDY_PRINTF("%zu: %d\n", i, bitset_test(bitset, i) > 0); } } @@ -1838,20 +1957,36 @@ static size_t highest_bit_position(size_t value) { static inline size_t ceiling_power_of_two(size_t value) { value += !value; /* branchless x -> { 1 for 0, x for x } */ - return ((size_t)1u) << (highest_bit_position(value + value - 1)-1); + return two_to_the_power_of(highest_bit_position(value + value - 1)-1); +} + +static inline size_t two_to_the_power_of(size_t order) { + return ((size_t)1) << order; } -static inline float approximate_square_root(float f) { - /* As listed in https://en.wikipedia.org/wiki/Methods_of_computing_square_roots */ - union { float f; uint32_t i; } val = {f}; - val.i -= 1 << 23; /* Subtract 2^m. */ - val.i >>= 1; /* Divide by 2. */ - val.i += 1 << 29; /* Add ((b + 1) / 2) * 2^m. */ - return val.f; /* Interpret again as float */ +static inline size_t integer_square_root(size_t op) { + /* by Martin Guy, 1985 - http://medialab.freaknet.org/martin/src/sqrt/ */ + size_t result = 0; + size_t cursor = (SIZE_MAX - (SIZE_MAX >> 1)) >> 1; /* second-to-top bit set */ + while (cursor > op) { + cursor >>= 2; + } + /* "cursor" starts at the highest power of four <= than the argument. */ + while (cursor != 0) { + if (op >= result + cursor) { + op -= result + cursor; + result += 2 * cursor; + } + result >>= 1; + cursor >>= 2; + } + return result; } #ifdef __cplusplus +#ifndef BUDDY_CPP_MANGLED } #endif +#endif #endif /* BUDDY_ALLOC_IMPLEMENTATION */