db: Add --json and --simple-metadata (#1583)

* db: Add --json and --simple-metadata

* cleanup

* rebase and update

src/json/json5.h

@@ -111,6 +111,9 @@ struct JAST {
     else
       return add(std::move(key), JSON_FALSE, "false");
   }
+  JAST &add(std::string key, uint64_t value) {
+    return add(std::move(key), JSON_INTEGER, std::to_string(value));
+  }
   JAST &add(std::string key, int value) {
     return add(std::move(key), JSON_INTEGER, std::to_string(value));
   }

src/runtime/database.cpp

@@ -1126,6 +1126,78 @@ JAST JobReflection::to_simple_json() const {
   return json;
 }
 
+JAST JobReflection::to_structured_json() const {
+  JAST json(JSON_OBJECT);
+  json.add("job", job);
+  json.add("label", label);
+  json.add("stale", stale);
+  json.add("directory", directory);
+
+  JAST &commandline_json = json.add("commandline", JSON_ARRAY);
+  for (const std::string &line : commandline) {
+    commandline_json.add("", line);
+  }
+
+  JAST &environment_json = json.add("environment", JSON_ARRAY);
+  for (const std::string &line : environment) {
+    environment_json.add("", line);
+  }
+
+  json.add("stack", stack);
+  json.add("stdin_file", stdin_file);
+  json.add("starttime", starttime.as_int64());
+  json.add("endtime", endtime.as_int64());
+  json.add("wake_start", wake_start.as_int64());
+  json.add("wake_cmdline", wake_cmdline);
+
+  std::string out_stream;
+  std::string err_stream;
+  for (auto &write : std_writes) {
+    if (write.second == 1) {
+      out_stream += write.first;
+    }
+    if (write.second == 2) {
+      err_stream += write.first;
+    }
+  }
+
+  json.add("stdout", out_stream);
+  json.add("stderr", err_stream);
+
+  JAST &usage_json = json.add("usage", JSON_OBJECT);
+  usage_json.add("status", usage.status);
+  usage_json.add("runtime", usage.runtime);
+  usage_json.add("cputime", usage.cputime);
+  usage_json.add("membytes", usage.membytes);
+  usage_json.add("ibytes", usage.ibytes);
+  usage_json.add("obytes", usage.obytes);
+
+  JAST &visible_json = json.add("visible_files", JSON_ARRAY);
+  for (const auto &visible_file : visible) {
+    visible_json.add("", visible_file.path);
+  }
+
+  JAST &input_json = json.add("input_files", JSON_ARRAY);
+  for (const auto &input : inputs) {
+    input_json.add("", input.path);
+  }
+
+  JAST &output_json = json.add("output_files", JSON_ARRAY);
+  for (const auto &output : outputs) {
+    output_json.add("", output.path);
+  }
+
+  JAST &tags_json = json.add("tags", JSON_ARRAY);
+  for (const auto &tag : tags) {
+    JAST &tag_json = tags_json.add("", JSON_OBJECT);
+    tag_json.add("uri", tag.uri);
+    tag_json.add("content", tag.content);
+  }
+
+  return json;
+}
+
+// TODO: Delete this and update --timeline to use to_structured_json()
 JAST JobReflection::to_json() const {
   JAST json(JSON_OBJECT);
   json.add("job", job);

src/runtime/database.h

@@ -82,6 +82,7 @@ struct JobReflection {
   std::vector<JobTag> tags;
 
   JAST to_json() const;
+  JAST to_structured_json() const;
   JAST to_simple_json() const;
 };
 

tools/wake/cli_options.h

@@ -43,6 +43,8 @@ struct CommandLineOptions {
   bool failed;
   bool script;
   bool metadata;
+  bool simple_metadata;
+  bool json;
   bool version;
   bool html;
   bool global;
@@ -135,6 +137,8 @@ struct CommandLineOptions {
       {'f', "failed", GOPT_ARGUMENT_FORBIDDEN},
       {'s', "script", GOPT_ARGUMENT_FORBIDDEN},
       {0, "metadata", GOPT_ARGUMENT_FORBIDDEN},
+      {0, "simple-metadata", GOPT_ARGUMENT_FORBIDDEN},
+      {0, "json", GOPT_ARGUMENT_FORBIDDEN},
       {0, "init", GOPT_ARGUMENT_REQUIRED},
       {0, "version", GOPT_ARGUMENT_FORBIDDEN},
       {'g', "globals", GOPT_ARGUMENT_FORBIDDEN},
@@ -192,6 +196,8 @@ struct CommandLineOptions {
     failed = arg(options, "failed")->count;
     script = arg(options, "script")->count;
     metadata = arg(options, "metadata")->count;
+    simple_metadata = arg(options, "simple-metadata")->count;
+    json = arg(options, "json")->count;
     version = arg(options, "version")->count;
     html = arg(options, "html")->count;
     global = arg(options, "globals")->count;

tools/wake/describe.cpp

@@ -55,7 +55,22 @@ static std::string describe_hash(const std::string &hash, bool verbose, bool sta
   return hash.substr(0, SHORT_HASH);
 }
 
-static void describe_metadata(const std::vector<JobReflection> &jobs, bool debug, bool verbose) {
+static void describe_json(const std::vector<JobReflection> &jobs) {
+  TermInfoBuf tbuf(std::cout.rdbuf());
+  std::ostream out(&tbuf);
+
+  JAST json(JSON_OBJECT);
+  JAST &job_array = json.add("jobs", JSON_ARRAY);
+
+  for (auto &job : jobs) {
+    job_array.add("", job.to_structured_json());
+  }
+
+  out << json;
+}
+
+static void describe_metadata(const std::vector<JobReflection> &jobs, bool debug, bool verbose,
+                              bool files) {
   TermInfoBuf tbuf(std::cout.rdbuf());
   std::ostream out(&tbuf);
 
@@ -82,12 +97,17 @@ static void describe_metadata(const std::vector<JobReflection> &jobs, bool debug
       for (auto &in : job.visible)
         out << "  " << describe_hash(in.hash, verbose, job.stale) << " " << in.path << std::endl;
     }
-    out << "Inputs:" << std::endl;
-    for (auto &in : job.inputs)
-      out << "  " << describe_hash(in.hash, verbose, job.stale) << " " << in.path << std::endl;
-    out << "Outputs:" << std::endl;
-    for (auto &output : job.outputs)
-      out << "  " << describe_hash(output.hash, verbose, false) << " " << output.path << std::endl;
+    if (files) {
+      out << "Inputs:" << std::endl;
+      for (auto &in : job.inputs) {
+        out << "  " << describe_hash(in.hash, verbose, job.stale) << " " << in.path << std::endl;
+      }
+      out << "Outputs:" << std::endl;
+      for (auto &output : job.outputs) {
+        out << "  " << describe_hash(output.hash, verbose, false) << " " << output.path
+            << std::endl;
+      }
+    }
     if (debug) {
       out << "Stack:";
       indent(out, "  ", job.stack);
@@ -359,15 +379,23 @@ void describe(const std::vector<JobReflection> &jobs, DescribePolicy policy, con
       break;
     }
     case DescribePolicy::METADATA: {
-      describe_metadata(jobs, false, false);
+      describe_metadata(jobs, false, false, true);
+      break;
+    }
+    case DescribePolicy::SIMPLE_METADATA: {
+      describe_metadata(jobs, false, false, false);
+      break;
+    }
+    case DescribePolicy::JSON: {
+      describe_json(jobs);
       break;
     }
     case DescribePolicy::DEBUG: {
-      describe_metadata(jobs, true, true);
+      describe_metadata(jobs, true, true, true);
       break;
     }
     case DescribePolicy::VERBOSE: {
-      describe_metadata(jobs, false, true);
+      describe_metadata(jobs, false, true, true);
       break;
     }
     case DescribePolicy::TAG_URI: {

tools/wake/describe.h

@@ -29,6 +29,8 @@ struct DescribePolicy {
     SCRIPT,
     HUMAN,
     METADATA,
+    SIMPLE_METADATA,
+    JSON,
     DEBUG,
     VERBOSE,
     TIMELINE,
@@ -57,6 +59,18 @@ struct DescribePolicy {
     return policy;
   }
 
+  static DescribePolicy simple_metadata() {
+    DescribePolicy policy;
+    policy.type = SIMPLE_METADATA;
+    return policy;
+  }
+
+  static DescribePolicy json() {
+    DescribePolicy policy;
+    policy.type = JSON;
+    return policy;
+  }
+
   static DescribePolicy debug() {
     DescribePolicy policy;
     policy.type = DEBUG;

tools/wake/main.cpp

@@ -111,6 +111,14 @@ DescribePolicy get_describe_policy(const CommandLineOptions &clo) {
     return DescribePolicy::metadata();
   }
 
+  if (clo.simple_metadata) {
+    return DescribePolicy::simple_metadata();
+  }
+
+  if (clo.json) {
+    return DescribePolicy::json();
+  }
+
   if (clo.script) {
     return DescribePolicy::script();
   }
@@ -267,6 +275,8 @@ void print_help(const char *argv0) {
     << "    --simple-timeline  Report simplified timeline of captured jobs as HTML"        << std::endl
     << "    --verbose  -v      Report metadata, stdout and stderr of captured jobs"        << std::endl
     << "    --metadata         Report metadata of captured jobs"                           << std::endl
+    << "    --simple-metadata  Report metadata of captured jobs without file list"         << std::endl
+    << "    --json             Report metadata, stdout and stderr of jobs as json"         << std::endl
     << "    --debug    -d      Report stack frame of captured jobs"                        << std::endl
     << "    --simple           Report only label, cmdline, and tags of captured jobs"      << std::endl
     << "    --script   -s      Format captured jobs as an executable shell script"         << std::endl
@@ -415,8 +425,8 @@ int main(int argc, char **argv) {
   // DescribePolicy::human() is the default and doesn't have a flag.
   // DescribePolicy::debug() is overloaded and can't be marked as a db flag
   // DescribePolicy::verbose() is overloaded and can't be marked as a db flag
-  bool is_db_inspect_render =
-      clo.taguri || clo.script || clo.metadata || clo.timeline || clo.simple || clo.simple_timeline;
+  bool is_db_inspect_render = clo.taguri || clo.script || clo.metadata || clo.timeline ||
+                              clo.simple || clo.simple_timeline || clo.json || clo.simple_metadata;
 
   bool is_db_inspection = is_db_inspect_capture || is_db_inspect_render;