memory usage testing

CMakeLists.txt

@@ -122,6 +122,9 @@ set(CMAKE_C_STANDARD_REQUIRED ON)
 # Need _XOPEN_SRC
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_XOPEN_SRC")
 
+add_compile_definitions(-DDEBUG)
+set(CMAKE_C_FLAGS "-O0 -ggdb")
+
 # Define the DEBUG macro when building in debug mode
 set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DDEBUG -Wall")
 

contrib/make_testindex.cpp

@@ -470,10 +470,9 @@ void generatecurr(ThreadArgs *arg, const std::size_t files, std::list <off_t> &s
 
         const std::size_t bucket = bucket_rng(gen);
 
-        struct work work;
+        struct work *work = new_work_with_name("", s.str().c_str());
         struct entry_data ed;
 
-        SNPRINTF(work.name, MAXPATH, "%s", s.str().c_str());
         ed.type = 'f';
         ed.linkname[0] = '\0';
 
@@ -501,7 +500,7 @@ void generatecurr(ThreadArgs *arg, const std::size_t files, std::list <off_t> &s
         ed.statuso.st_atime = std::max(ed.statuso.st_ctime, (time_t) time_rng(gen));
         ed.statuso.st_mtime = std::max(ed.statuso.st_ctime, (time_t) time_rng(gen));
 
-        work.pinode   = rng(gen);
+        work->pinode   = rng(gen);
         ed.crtime     = rng(gen);
         ed.ossint1    = rng(gen);
         ed.ossint2    = rng(gen);
@@ -512,14 +511,16 @@ void generatecurr(ThreadArgs *arg, const std::size_t files, std::list <off_t> &s
         sumit(&summary, &ed);
 
         // insert the row
-        insertdbgo(&work, &ed, res);
+        insertdbgo(work, &ed, res);
 
         // store stats in local variables first
         // to prevent lock contention and
         // because nothing is inserted until
         // the transaction ends
         curr_size += ed.statuso.st_size;
         curr_sizes.push_back(ed.statuso.st_size);
+
+        free(work);
     }
 
     // complete the transaction
@@ -547,10 +548,9 @@ void generatecurr(ThreadArgs *arg, const std::size_t files, std::list <off_t> &s
     }
 
     // summarize this directory
-    struct work work;
+    struct work *work = new_work_with_name("", arg->path.c_str());
     struct entry_data ed;
 
-    SNPRINTF(work.name, MAXPATH, "%s", arg->path.c_str());
     ed.type = 'd';
     ed.linkname[0] = '\0';
 
@@ -584,7 +584,8 @@ void generatecurr(ThreadArgs *arg, const std::size_t files, std::list <off_t> &s
     ed.ossint3 = rng(gen);
     ed.ossint4 = rng(gen);
 
-    insertsumdb(on_disk, work.name, &work, &ed, &summary);
+    insertsumdb(on_disk, work->name, work, &ed, &summary);
+    free(work);
 
     sqlite3_close(on_disk);
 

contrib/verifytraceintree.cpp

@@ -187,7 +187,7 @@ static int check_stanza(QPTPool_t *, const size_t id, void *data, void *args) {
         // parse the line to check other values
         char buf[MAXPATH] = {};
         memcpy(buf, sa->line.c_str(), sa->line.size());
-        struct work work;
+        struct work *work;
         struct entry_data ed;
         linetowork(buf, sa->line.size(), csa->delim, &work, &ed);
         xattrs_cleanup(&ed.xattrs);
@@ -268,13 +268,13 @@ static int check_stanza(QPTPool_t *, const size_t id, void *data, void *args) {
         // parse the line
         char buf[MAXPATH] = {};
         memcpy(buf, line.c_str(), line.size());
-        struct work work;
+        struct work *work;
         struct entry_data ed;
         linetowork(buf, line.size(), csa->delim, &work, &ed);
         xattrs_cleanup(&ed.xattrs);
 
         // extract the basename from the entry name
-        char *bufbase = basename(work.name);
+        char *bufbase = basename(work->name);
 
         // query the database for the current entry
         char sql[MAXSQL];

include/bf.h

@@ -281,24 +281,37 @@ typedef enum {
    CLOSE_DB  = 0x04,
 } CleanUpTasks;
 
-/* minimum data needs to be passed around between threads */
+/*
+ * Minimum data needs to be passed around between threads.
+ *
+ * struct work generally should not be allocated directly - the helper
+ *    new_work_with_name() should be used to ensure correct size is allocated.
+ *
+ * Similarly, the size of struct work should not be calculated with
+ * sizeof, but rather using the helper struct_work_size() to account for the
+ * storage for name.
+ */
 struct work {
    compressed_t  compressed;
    refstr_t      orig_root;              /* argv[i] */
    refstr_t      root_parent;            /* dirname(realpath(argv[i])) */
    size_t        root_basename_len;      /* strlen(basename(argv[i])) */
    size_t        level;
-   char          name[MAXPATH];
-   size_t        name_len;
-   size_t        basename_len;           /* can usually get through readdir */
    long long int pinode;
    size_t        recursion_level;
 
    /* probably shouldn't be here */
    char *        fullpath;
    size_t        fullpath_len;
+
+   size_t        name_len;               /* == strlen(name) - meaning excludes NUL! */
+   size_t        basename_len;           /* can usually get through readdir */
+   char          name[];
 };
 
+size_t struct_work_size(struct work *w);
+struct work *new_work_with_name(const char *prefix, const char *name);
+
 /* extra data used by entries that does not depend on data from other directories */
 struct entry_data {
    char          type;

include/compress.h

@@ -78,7 +78,9 @@ extern "C" {
  */
 typedef struct compressed {
     int8_t          yes;  /* is this struct compressed? */
+    uint16_t        orig_len; /* not a size_t to fit in hole in struct */
     size_t          len;  /* includes self; only meaningful if yes == 1 */
+    char            data[0];
 } compressed_t;
 
 #if HAVE_ZLIB /* or any other algorithm */
@@ -89,8 +91,7 @@ typedef struct compressed {
 
 void *compress_struct(const int comp, void *src, const size_t struct_len);
 
-/* initialize dst with pointer to valid buffer */
-void decompress_struct(void **dst, void *src, const size_t struct_len);
+void decompress_struct(void **dst, void *src);
 
 /*
  * used is the struct that was used for operations (decompressed or original data)

include/gufi_query/gqw.h

@@ -69,7 +69,9 @@ OF SUCH DAMAGE.
 
 /* additional data gufi_query needs */
 typedef struct gufi_query_work {
-    struct work work;
+    compressed_t comp;
+    /* TODO: make this inline? */
+    struct work *work;
 
     /*
      * some characters need to be converted for sqlite3,

include/trace.h

@@ -83,7 +83,7 @@ int worktofile(FILE *file, const char delim, const size_t prefix_len, struct wor
 
 /* convert a formatted string to a work struct or attach name */
 int linetowork(char *line, const size_t len, const char delim,
-               struct work *work, struct entry_data *ed);
+               struct work **work, struct entry_data *ed);
 
 int *open_traces(char **trace_names, size_t trace_count);
 void close_traces(int *traces, size_t trace_count);

src/bfwreaddirplus2db.c

@@ -219,19 +219,17 @@ static int reprocessdir(struct input *in, void *passv, DIR *dir) {
                 continue;
         }
 
-        struct work qwork;
+        struct work *qwork = new_work_with_name(passmywork->name,  entry->d_name);
+        qwork->basename_len = len;
+
         struct entry_data qwork_ed;
-        memset(&qwork, 0, sizeof(qwork));
         memset(&qwork_ed, 0, sizeof(qwork_ed));
-        qwork.pinode = ed.statuso.st_ino;
-
-        qwork.name_len = SNPRINTF(qwork.name, MAXPATH, "%s/%s", passmywork->name, entry->d_name);
-        qwork.basename_len = len;
+        qwork->pinode = ed.statuso.st_ino;
 
-        lstat(qwork.name, &qwork_ed.statuso);
+        lstat(qwork->name, &qwork_ed.statuso);
         xattrs_setup(&qwork_ed.xattrs);
         if (in->process_xattrs) {
-            xattrs_get(qwork.name, &qwork_ed.xattrs);
+            xattrs_get(qwork->name, &qwork_ed.xattrs);
         }
 
         /*
@@ -241,16 +239,17 @@ static int reprocessdir(struct input *in, void *passv, DIR *dir) {
         if (!S_ISDIR(qwork_ed.statuso.st_mode)) {
             if (S_ISLNK(qwork_ed.statuso.st_mode)) {
                 qwork_ed.type = 'l';
-                readlink(qwork.name, qwork_ed.linkname, MAXPATH);
+                readlink(qwork->name, qwork_ed.linkname, MAXPATH);
             } else if (S_ISREG(qwork_ed.statuso.st_mode)) {
                 qwork_ed.type = 'f';
             }
 
             sumit(&summary, &qwork_ed);
-            insertdbgo(&qwork, &qwork_ed, res);
+            insertdbgo(qwork, &qwork_ed, res);
         }
 
         xattrs_cleanup(&qwork_ed.xattrs);
+        free(qwork);
     }
 
     stopdb(db);
@@ -390,19 +389,15 @@ static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {
                 continue;
         }
 
-        struct work qwork;
+        struct work *qwork = new_work_with_name(passmywork->name, entry->d_name);
         struct entry_data qwork_ed;
-        memset(&qwork, 0, sizeof(qwork));
         memset(&qwork_ed, 0, sizeof(qwork_ed));
 
-        SNPRINTF(qwork.name, MAXPATH, "%s/%s", passmywork->name, entry->d_name);
-        qwork.pinode = ed.statuso.st_ino;
+        qwork->pinode = ed.statuso.st_ino;
         qwork_ed.statuso.st_ino = entry->d_ino;
 
         if (entry->d_type == DT_DIR) {
-            struct work *work = calloc(1, sizeof(struct work));
-            memcpy(work, &qwork, sizeof(qwork));
-            QPTPool_enqueue(ctx, id, processdir, work);
+            QPTPool_enqueue(ctx, id, processdir, qwork);
             continue;
         }
         else if (entry->d_type == DT_LNK) {
@@ -427,7 +422,7 @@ static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {
                         startdb(ta->db);
                     }
 
-                    insertdbgor(&qwork, &qwork_ed, ta->res);
+                    insertdbgor(qwork, &qwork_ed, ta->res);
 
                     if (in->stride > 0) {
                         stopdb(ta->db);
@@ -444,7 +439,7 @@ static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {
                 if (in->suspectmethod > 2) {
                     /* stat the file/link and if ctime or mtime is >= provided last run time mark dir suspect */
                     struct stat st;
-                    lstat(qwork.name, &st);
+                    lstat(qwork->name, &st);
                     if (st.st_ctime >= locsuspecttime) ed.suspect = 1;
                     if (st.st_mtime >= locsuspecttime) ed.suspect = 1;
                 }
@@ -463,9 +458,9 @@ static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {
                  * the file or link without the name as the sortf
                  */
                 fprintf(ta->file, "%s%c%" STAT_ino "%c%lld%c%c%c%s%c\n",
-                        qwork.name,              in->delim,
+                        qwork->name,             in->delim,
                         qwork_ed.statuso.st_ino, in->delim,
-                        qwork.pinode,            in->delim,
+                        qwork->pinode,           in->delim,
                         qwork_ed.type,           in->delim,
                         passmywork->name,        in->delim);
 
@@ -474,6 +469,8 @@ static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {
                 }
             }
         }
+
+        free(qwork);
     }
 
     /*
@@ -613,9 +610,7 @@ static int processinit(struct PoolArgs *pa, QPTPool_t *ctx) {
         }
     }
 
-    struct work *mywork = calloc(1, sizeof(struct work));
-
-    mywork->name_len = SNFORMAT_S(mywork->name, sizeof(mywork->name), 1, pa->in.name.data, pa->in.name.len);
+    struct work *mywork = new_work_with_name("", pa->in.name.data);
 
     QPTPool_enqueue(ctx, 0, processdir, mywork);
 

src/compress.c

@@ -105,57 +105,71 @@ static void decompress_zlib(void *dst, size_t *dst_len, void *src, const size_t
 }
 #endif
 
-/* always return new address */
+/*
+ * compress_struct() should act like realloc() - it either returns a new pointer,
+ *   and frees the original, or returns the original.
+ *
+ *   comp       - if true, do compression
+ *   src        - must be a valid free()able pointer.
+ *   struct_len - size of *src
+ */
 void *compress_struct(const int comp, void *src, const size_t struct_len) {
-    /* not checking src == NULL */
-    compressed_t *dst = malloc(struct_len);
-    dst->yes = 0;
-    dst->len = 0;
-
+    #if HAVE_ZLIB
     if (comp) {
+        compressed_t *dst = malloc(struct_len);
+        dst->yes = 0;
         dst->len = struct_len - COMP_OFFSET;
+        dst->orig_len = struct_len;
 
-        #if HAVE_ZLIB
-        if (compress_zlib(((unsigned char *) dst) + COMP_OFFSET, &dst->len,
-                          ((unsigned char *) src) + COMP_OFFSET,  dst->len) == Z_OK) {
+        if (compress_zlib(dst->data, &dst->len,
+                          ((compressed_t *) src)->data,  dst->len) == Z_OK) {
             dst->yes = 1;
             dst->len += COMP_OFFSET;
         }
-        #endif
 
         /* if compressed enough, reallocate to reduce memory usage */
         if (dst->yes && (dst->len < struct_len)) {
             void *r = realloc(dst, COMP_OFFSET + dst->len);
             if (r) {
-                dst = r;
+                free(src);
+                return r;
             }
 
-            /* if realloc failed, dst can still be used */
-            return dst;
+            /* realloc() failed - free() the new buffer and return src */
+            free(dst);
         }
     }
+    #endif
 
-    /* if no compression or compression failed, copy src */
-    memcpy(dst, src, struct_len);
-
-    return dst;
+    return src;
 }
 
-void decompress_struct(void **dst, void *src, const size_t struct_len) {
-    /* not checking src == NULL */
+/*
+ * decompress_struct() -
+ *
+ *     src - pointer to optionally compressed data item
+ *     dst - will be updated to hold pointer to uncompressed buffer.
+ *
+ *     If src is compressed, then free() src and make dst point to a new buffer.
+ *     If src is not compressed, then make dst point to src.
+ *
+ *     Returns: 0 if success, and *dst will be a valid pointer
+ *         else 1 and *dst will not be a valid pointer
+ */
+void decompress_struct(void **dst, void *src) {
+    #if HAVE_ZLIB
     compressed_t *comp = (compressed_t *) src;
     if (comp->yes) {
-        compressed_t *decomp = (compressed_t *) *dst;
-        decomp->len = struct_len - COMP_OFFSET;
+        compressed_t *new = calloc(1, comp->orig_len);
+        size_t new_len = comp->orig_len - COMP_OFFSET;
 
-        #if HAVE_ZLIB
-        decompress_zlib(((unsigned char *) *dst) + COMP_OFFSET, &decomp->len,
-                        ((unsigned char *)  src) + COMP_OFFSET,    comp->len);
-        #endif
+        decompress_zlib(new->data, &new_len, comp->data, comp->len);
 
         free(src);
+        *dst = new;
         return;
     }
+    #endif
 
     /*
      * compression not enabled or compression enabled but