make struct BottomUp smaller

This makes struct BottomUp smaller by replacing two fixed-size buffers
of 4 KiB for pathnames with dynamically allocated buffers that are only
as large as they need to be to store the given path.

This reduces the size of struct BottomUp from 8504 bytes to 328 bytes.

I found that parallel_rmr used 63916 KiB of virtual memory to delete a
tree of 55987 dirs and 111972 files before this change, and 1131492 KiB
after this change.

include/BottomUp.h

@@ -98,9 +98,9 @@ extern "C" {
   the user, so the imeplementation is not opaque.
 */
 struct BottomUp {
-    char name[MAXPATH];
+    char *name;
     size_t name_len;
-    char alt_name[MAXPATH];
+    char *alt_name;
     size_t alt_name_len;
     struct stat st;
     struct {
@@ -181,6 +181,9 @@ int parallel_bottomup(char **root_names, const size_t root_count,
                       #endif
 );
 
+/* free a struct BottomUp */
+void bottomup_destroy(void *p);
+
 #ifdef __cplusplus
 }
 #endif

src/BottomUp.c

@@ -108,6 +108,86 @@ struct UserArgs {
     #endif
 };
 
+/*
+ * new_pathname() -
+ *   Allocates and initializes a new pathname from the given dirname and basename.
+ *
+ *     - dirname_len and basename_len should be WITHOUT the null terminator.
+ *
+ *     - If basename is NULL, then this only uses the dirname and doesn't append a path component
+ *       for the basename.
+ *
+ *   Fills in the path and path length in the given struct BottomUp.
+ */
+static void new_pathname(struct BottomUp *work, const char *dirname, size_t dirname_len,
+                         const char *basename, size_t basename_len) {
+    size_t new_len = 0; // does NOT count null terminator
+
+    if (basename)
+        new_len = dirname_len + basename_len + 1; // +1 for path separator
+    else
+        new_len = dirname_len;
+
+    char *path = calloc(new_len + 1, sizeof(char));  // Additional +1 for null terminator
+    if (!path) {
+        fprintf(stderr, "%s(): out of memory", __func__);
+        /* No point in continuing if we run out of memory: */
+        exit(1);
+    }
+
+    if (basename)
+        SNFORMAT_S(path, new_len + 1, 3, dirname, dirname_len, "/", (size_t) 1, basename, basename_len); 
+    else
+        strncpy(path, dirname, new_len + 1);
+
+    work->name = path;
+    work->name_len = new_len;
+}
+
+/*
+ * new_alt_pathname() -
+ *   Allocates and initializes a new alternate pathname, that is, a name that escapes
+ *   characters that cannot appear in a sqlite uri.
+ *
+ *   Fills in alt_name and alt_name_len in the given struct BottomUp.
+ */
+static void new_alt_pathname(struct BottomUp *work, const char *dirname, size_t dirname_len,
+                             const char *basename, size_t basename_len) {
+    // Alternate name may be longer than the provided dirname and basename together
+    size_t len = 4096;
+    size_t new_len = 0;
+
+    char *path = calloc(len, sizeof(char));
+    if (!path)
+        goto oom;
+
+    if (basename) {
+        // Assume dirname is already sanitized, only need to sanitize basename:
+        new_len = SNFORMAT_S(path, len, 2, dirname, dirname_len, "/", (size_t) 1);
+        new_len += sqlite_uri_path(path + new_len, len - new_len, basename, &basename_len);
+    } else {
+        // Need to sanitize dirname:
+        new_len = sqlite_uri_path(path, len, dirname, &dirname_len);
+    }
+
+    char *new = calloc(new_len + 1, sizeof(char));
+    if (!new)
+        goto oom;
+
+    strncpy(new, path, new_len);
+    free(path);
+
+    work->alt_name = new;
+    work->alt_name_len = new_len;
+
+    return;
+
+oom:
+    fprintf(stderr, "%s(): out of memory", __func__);
+    /* No point in continuing if we run out of memory: */
+    exit(1);
+}
+
 static int ascend_to_top(QPTPool_t *ctx, const size_t id, void *data, void *args) {
     timestamp_create_start(ascend);
 
@@ -151,8 +231,8 @@ static int ascend_to_top(QPTPool_t *ctx, const size_t id, void *data, void *args
     /* clean up 'struct BottomUp's here, when they are */
     /* children instead of when they are the parent */
     timestamp_create_start(cleanup);
-    sll_destroy(&bu->subdirs, free);
-    sll_destroy(&bu->subnondirs, free);
+    sll_destroy(&bu->subdirs, bottomup_destroy);
+    sll_destroy(&bu->subnondirs, bottomup_destroy);
 
     /* mutex is not needed any more */
     pthread_mutex_destroy(&bu->refs.mutex);
@@ -167,24 +247,29 @@ static int ascend_to_top(QPTPool_t *ctx, const size_t id, void *data, void *args
     else {
         /* reached root */
         timestamp_create_start(free_root);
-        free(bu);
+        bottomup_destroy(bu);
         timestamp_end_print(ua->timestamp_buffers, id, "free_root", free_root);
     }
 
     timestamp_end_print(ua->timestamp_buffers, id, "ascend_to_top", ascend);
     return asc_rc;
 }
 
+/*
+ * track() - add the given struct BottomUp src to the given sll_t for later processing.
+ *
+ * Takes ownership of the name and alt_name from src.
+ */
 static struct BottomUp *track(struct BottomUp *src,
                               const size_t user_struct_size, sll_t *sll,
                               const size_t level, const int generate_alt_name) {
-    struct BottomUp *copy = malloc(user_struct_size);
+    struct BottomUp *copy = calloc(user_struct_size, 1);
 
-    memcpy(copy->name, src->name, src->name_len + 1); /* NULL terminate */
+    copy->name = src->name;
     copy->name_len = src->name_len;
 
     if (generate_alt_name) {
-        memcpy(copy->alt_name, src->alt_name, src->alt_name_len + 1); /* NULL terminate */
+        copy->alt_name = src->alt_name;
         copy->alt_name_len = src->alt_name_len;
     }
 
@@ -211,7 +296,7 @@ static int descend_to_bottom(QPTPool_t *ctx, const size_t id, void *data, void *
 
     if (!dir) {
         fprintf(stderr, "Error: Could not open directory \"%s\": %s\n", bu->name, strerror(errno));
-        free(data);
+        bottomup_destroy(bu);
         timestamp_end_print(ua->timestamp_buffers, id, "descend_to_bottom", descend);
         return 1;
     }
@@ -240,21 +325,13 @@ static int descend_to_bottom(QPTPool_t *ctx, const size_t id, void *data, void *
             continue;
         }
 
-        struct BottomUp new_work;
-        new_work.name_len = SNFORMAT_S(new_work.name, sizeof(new_work.name), 3,
-                                       bu->name, bu->name_len,
-                                       "/", (size_t) 1,
-                                       entry->d_name, name_len);
+        struct BottomUp new_work = { 0 };
+
+        new_pathname(&new_work, bu->name, bu->name_len, entry->d_name, name_len);
 
         if (ua->generate_alt_name) {
             /* append converted entry name to converted directory */
-            new_work.alt_name_len = SNFORMAT_S(new_work.alt_name, sizeof(new_work.alt_name), 2,
-                                               bu->alt_name, bu->alt_name_len,
-                                               "/", (size_t) 1);
-
-            new_work.alt_name_len += sqlite_uri_path(new_work.alt_name + new_work.alt_name_len,
-                                                     sizeof(new_work.alt_name) - new_work.alt_name_len,
-                                                     entry->d_name, &name_len);
+            new_alt_pathname(&new_work, bu->alt_name, bu->alt_name_len, entry->d_name, name_len);
         }
 
         timestamp_create_start(lstat_entry);
@@ -334,11 +411,11 @@ static int descend_to_bottom(QPTPool_t *ctx, const size_t id, void *data, void *
     }
     else {
         timestamp_create_start(cleanup_after_error);
-        sll_destroy(&bu->subdirs, free);
-        sll_destroy(&bu->subnondirs, free);
+        sll_destroy(&bu->subdirs, bottomup_destroy);
+        sll_destroy(&bu->subnondirs, bottomup_destroy);
         bu->subdir_count = 0;
         bu->subnondir_count = 0;
-        free(bu);
+        bottomup_destroy(bu);
         timestamp_end_print(ua->timestamp_buffers, id, "cleanup_after_error", cleanup_after_error);
     }
 
@@ -415,17 +492,16 @@ int parallel_bottomup_enqueue(QPTPool_t *pool,
     struct UserArgs *ua = NULL;
     QPTPool_get_args(pool, (void **) &ua);
 
-    struct BottomUp *root = (struct BottomUp *) malloc(ua->user_struct_size);
-    root->name_len = SNFORMAT_S(root->name, MAXPATH, 1, path, len);
+    struct BottomUp *root = (struct BottomUp *) calloc(ua->user_struct_size, 1);
+    new_pathname(root, path, len, NULL, 0);
 
     if (ua->generate_alt_name) {
         size_t name_len = root->name_len;
-        root->alt_name_len = sqlite_uri_path(root->alt_name, sizeof(root->alt_name),
-                                             root->name, &name_len);
+        new_alt_pathname(root, root->name, name_len, NULL, 0);
 
         if (name_len != root->name_len) {
             fprintf(stderr, "%s could not fit into ALT_NAME buffer\n", root->name);
-            free(root);
+            bottomup_destroy(root);
             return -1;
         }
     }
@@ -505,3 +581,15 @@ int parallel_bottomup(char **root_names, const size_t root_count,
 
     return -(parallel_bottomup_fini(pool) || (root_count != good_roots));
 }
+
+/* free a struct BottomUp */
+void bottomup_destroy(void *p) {
+    struct BottomUp *b = (struct BottomUp *) p;
+    if (b->name)
+        free(b->name);
+
+    if (b->alt_name)
+        free(b->alt_name);
+
+    free(b);
+}

src/parallel_rmr.c

@@ -141,7 +141,10 @@ int main(int argc, char * argv[]) {
 
     timestamp_print_destroy(timestamp_buffers);
 
+
     input_fini(&in);
 
+    dump_memory_usage();
+
     return rc?EXIT_FAILURE:EXIT_SUCCESS;
 }