Introduce new tool: -t compdb-targets

todo:

 - review coding-style guidelines & fix any violations

Co-authored-by: Linkun Chen <[email protected]>
Co-authored-by: csmoe <[email protected]>
Co-authored-by: James Widman <[email protected]>

doc/manual.asciidoc

@@ -289,6 +289,11 @@ http://clang.llvm.org/docs/JSONCompilationDatabase.html[JSON format] expected
 by the Clang tooling interface.
 _Available since Ninja 1.2._
 
+`compdb-targets`:: like `compdb`, but takes a list of targets instead of rules,
+and expects at least one target. The resulting compilation database contains
+all commands required to build the indicated targets, and _only_ those
+commands.
+
 `deps`:: show all dependencies stored in the `.ninja_deps` file. When given a
 target, show just the target's dependencies. _Available since Ninja 1.4._
 

doc/style.css

@@ -53,3 +53,7 @@ div.chapter {
 p {
     margin-top: 0;
 }
+
+code.literal {
+    white-space: nowrap;
+}

misc/output_test.py

@@ -13,13 +13,49 @@
 import unittest
 from textwrap import dedent
 from typing import Dict
+from collections.abc import Callable
 
 default_env = dict(os.environ)
 default_env.pop('NINJA_STATUS', None)
 default_env.pop('CLICOLOR_FORCE', None)
 default_env['TERM'] = ''
 NINJA_PATH = os.path.abspath('./ninja')
 
+def cook(raw_output: bytes) -> str:
+  # When running in a smart terminal, Ninja uses CR (\r) to
+  # return the cursor to the start of the current line, prints
+  # something, then uses `\x1b[K` to clear everything until
+  # the end of the line.
+  #
+  # Thus printing 'FOO', 'BAR', 'ZOO' on the same line, then
+  # jumping to the next one results in the following output
+  # on Posix:
+  #
+  # '\rFOO\x1b[K\rBAR\x1b[K\rZOO\x1b[K\r\n'
+  #
+  # The following splits the output at both \r, \n and \r\n
+  # boundaries, which gives:
+  #
+  #  [ '\r', 'FOO\x1b[K\r', 'BAR\x1b[K\r', 'ZOO\x1b[K\r\n' ]
+  #
+  decoded_lines = raw_output.decode('utf-8').splitlines(True)
+
+  # Remove any item that ends with a '\r' as this means its
+  # content will be overwritten by the next item in the list.
+  # For the previous example, this gives:
+  #
+  #  [ 'ZOO\x1b[K\r\n' ]
+  #
+  final_lines = [ l for l in decoded_lines if not l.endswith('\r') ]
+
+  # Return a single string that concatenates all filtered lines
+  # while removing any remaining \r in it. Needed to transform
+  # \r\n into \n.
+  #
+  #  "ZOO\x1b[K\n'
+  #
+  return ''.join(final_lines).replace('\r', '')
+
 class BuildDir:
     def __init__(self, build_ninja: str):
         self.build_ninja = dedent(build_ninja)
@@ -41,6 +77,7 @@ def run(
         pipe: bool = False,
         raw_output: bool = False,
         env: Dict[str, str] = default_env,
+        print_err_output = True,
     ) -> str:
         """Run Ninja command, and get filtered output.
 
@@ -56,6 +93,10 @@ def run(
 
           env: Optional environment dictionary to run the command in.
 
+          print_err_output: set to False if the test expects ninja to print
+            something to stderr. (Otherwise, an error message from Ninja
+            probably represents a failed test.)
+
         Returns:
           A UTF-8 string corresponding to the output (stdout only) of the
           Ninja command. By default, partial lines that were overwritten
@@ -74,58 +115,41 @@ def run(
                 output = subprocess.check_output(['script', '-qfec', ninja_cmd, '/dev/null'],
                                                  cwd=self.d.name, env=env)
         except subprocess.CalledProcessError as err:
-            sys.stdout.buffer.write(err.output)
+            if print_err_output:
+              sys.stdout.buffer.write(err.output)
+            err.cooked_output = cook(err.output)
             raise err
 
         if raw_output:
             return output.decode('utf-8')
-
-        # When running in a smart terminal, Ninja uses CR (\r) to
-        # return the cursor to the start of the current line, prints
-        # something, then uses `\x1b[K` to clear everything until
-        # the end of the line.
-        #
-        # Thus printing 'FOO', 'BAR', 'ZOO' on the same line, then
-        # jumping to the next one results in the following output
-        # on Posix:
-        #
-        # '\rFOO\x1b[K\rBAR\x1b[K\rZOO\x1b[K\r\n'
-        #
-        # The following splits the output at both \r, \n and \r\n
-        # boundaries, which gives:
-        #
-        #  [ '\r', 'FOO\x1b[K\r', 'BAR\x1b[K\r', 'ZOO\x1b[K\r\n' ]
-        #
-        decoded_lines = output.decode('utf-8').splitlines(True)
-
-        # Remove any item that ends with a '\r' as this means its
-        # content will be overwritten by the next item in the list.
-        # For the previous example, this gives:
-        #
-        #  [ 'ZOO\x1b[K\r\n' ]
-        #
-        final_lines = [ l for l in decoded_lines if not l.endswith('\r') ]
-
-        # Return a single string that concatenates all filtered lines
-        # while removing any remaining \r in it. Needed to transform
-        # \r\n into \n.
-        #
-        #  "ZOO\x1b[K\n'
-        #
-        return ''.join(final_lines).replace('\r', '')
+        return cook(output)
 
 def run(
     build_ninja: str,
     flags: str = '',
     pipe: bool = False,
     raw_output: bool = False,
     env: Dict[str, str] = default_env,
+    print_err_output = True,
 ) -> str:
     """Run Ninja with a given build plan in a temporary directory.
     """
     with BuildDir(build_ninja) as b:
         return b.run(flags, pipe, raw_output, env)
 
+def run_and_generate_expected_output(
+    build_ninja: str,
+    flags: str = '',
+    expected: Callable[[str], str] = None,
+    pipe: bool = False,
+    raw_output: bool = False,
+    env: Dict[str, str] = default_env,
+) -> (str, str):
+    with BuildDir(build_ninja) as b:
+        actual_output = b.run(flags, pipe, raw_output, env)
+        expected_output = expected(os.path.realpath(b.d.name))
+        return (expected_output, actual_output)
+
 @unittest.skipIf(platform.system() == 'Windows', 'These test methods do not work on Windows')
 class Output(unittest.TestCase):
     BUILD_SIMPLE_ECHO = '\n'.join((
@@ -371,6 +395,62 @@ def test_tool_inputs(self) -> None:
         )
 
 
+    def test_tool_compdb_targets(self) -> None:
+        self.maxDiff = 90000
+        plan = '''
+rule cat
+  command = cat $in $out
+build out1 : cat in1
+build out2 : cat in2 out1
+build out3 : cat out2 out1
+build out4 : cat in4
+'''
+
+        def test_expected_error(flags, expected):
+          actual = ''
+          try:
+            actual = run(plan, flags, print_err_output=False)
+          except subprocess.CalledProcessError as err:
+            actual = err.cooked_output
+          self.assertEqual(expected, actual)
+
+
+        test_expected_error('-t compdb-targets',
+            "ninja: fatal: compdb-targets expects the name of at least one target\n")
+
+        test_expected_error('-t compdb-targets in1',
+            "ninja: fatal: 'in1' is not a target (i.e. it is not an output of any `build` statement)\n")
+
+        test_expected_error('-t compdb-targets nonexistent_target',
+            "ninja: fatal: unknown target 'nonexistent_target'\n")
+
+
+        (expected, actual) =  run_and_generate_expected_output(plan, flags='-t compdb-targets out3',
+                                               expected=lambda directory:
+f'''[
+  {{
+    "directory": "{directory}",
+    "command": "cat in1 out1",
+    "file": "in1",
+    "output": "out1"
+  }},
+  {{
+    "directory": "{directory}",
+    "command": "cat in2 out1 out2",
+    "file": "in2",
+    "output": "out2"
+  }},
+  {{
+    "directory": "{directory}",
+    "command": "cat out2 out1 out3",
+    "file": "out2",
+    "output": "out3"
+  }}
+]
+''')
+        self.assertEqual(expected, actual)
+
+
     def test_explain_output(self):
         b = BuildDir('''\
             build .FORCE: phony

src/graph.h

@@ -16,8 +16,10 @@
 #define NINJA_GRAPH_H_
 
 #include <algorithm>
+#include <cassert>
 #include <queue>
 #include <set>
+#include <unordered_set>
 #include <string>
 #include <vector>
 
@@ -459,4 +461,48 @@ struct InputsCollector {
   std::set<const Node*> visited_nodes_;
 };
 
+/// Collects the transitive set of edges that lead into a given set
+/// of starting nodes. Used to implement the `compdb` tool.
+///
+/// When collecting inputs, the outputs of phony edges are always ignored
+/// from the result, but are followed by the dependency walk.
+///
+/// Usage is:
+/// - Create instance.
+/// - Call CollectFrom() for each root node to collect edges from.
+/// - Call TakeResult() to retrieve the list of edges.
+///
+struct InEdge_Collector {
+
+  void CollectFrom(const Node* node) {
+    assert(node);
+
+    if (!visited_nodes_.insert(node).second)
+      return;
+
+    Edge* edge = node->in_edge();
+    if (!edge || !visited_edges_.insert(edge).second)
+      return;
+
+    for (Node* input_node : edge->inputs_)
+      CollectFrom(input_node);
+
+    if (!edge->is_phony())
+      in_edges.push_back(edge);
+
+  }
+
+  std::vector<Edge*> TakeResult() { return std::move(in_edges); }
+  bool Empty() { return in_edges.empty(); }
+
+  std::unordered_set<const Node*> visited_nodes_;
+  std::unordered_set<Edge*> visited_edges_;
+
+  /// we use a vector to preserve order from requisites to their dependents.
+  /// This may help LSP server performance in languages that support modules,
+  /// but it also ensures that the output of `-t compdb --targets foo` is
+  /// consistent, which is useful in regression tests.
+  std::vector<Edge*> in_edges;
+};
+
 #endif  // NINJA_GRAPH_H_

src/graph_test.cc

@@ -282,6 +282,54 @@ TEST_F(GraphTest, InputsCollector) {
   EXPECT_EQ("out3", inputs[4]);
 }
 
+TEST_F(GraphTest, InEdgeCollector) {
+  ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+                                      "build out1: cat in1\n"
+                                      "build mid1: cat in1\n"
+                                      "build out2: cat mid1\n"
+                                      "build out3 out4: cat mid1\n"
+                                      "build all: phony out1 out2 out3\n"));
+  {
+    InEdge_Collector collector;
+    auto& edges = collector.in_edges;
+
+    // Start visit from out2; this should add `build mid1` and `build out2` to
+    // the edge list.
+    collector.CollectFrom(GetNode("out2"));
+    ASSERT_EQ(2u, edges.size());
+    EXPECT_EQ("cat in1 > mid1", edges[0]->EvaluateCommand());
+    EXPECT_EQ("cat mid1 > out2", edges[1]->EvaluateCommand());
+
+    // Add a visit from out1, this should append `build out1`
+    collector.CollectFrom(GetNode("out1"));
+    ASSERT_EQ(3u, edges.size());
+    EXPECT_EQ("cat in1 > out1", edges[2]->EvaluateCommand());
+
+    // Another visit from all; this should add edges for out1, out2 and out3,
+    // but not all (because it's phony).
+    collector.CollectFrom(GetNode("all"));
+    ASSERT_EQ(4u, edges.size());
+    EXPECT_EQ("cat in1 > mid1", edges[0]->EvaluateCommand());
+    EXPECT_EQ("cat mid1 > out2", edges[1]->EvaluateCommand());
+    EXPECT_EQ("cat in1 > out1", edges[2]->EvaluateCommand());
+    EXPECT_EQ("cat mid1 > out3 out4", edges[3]->EvaluateCommand());
+  }
+
+  {
+    InEdge_Collector collector;
+    auto& edges = collector.in_edges;
+
+    // Starting directly from all, will add `build out1` before `build mid1`
+    // compared to the previous example above.
+    collector.CollectFrom(GetNode("all"));
+    ASSERT_EQ(4u, edges.size());
+    EXPECT_EQ("cat in1 > out1", edges[0]->EvaluateCommand());
+    EXPECT_EQ("cat in1 > mid1", edges[1]->EvaluateCommand());
+    EXPECT_EQ("cat mid1 > out2", edges[2]->EvaluateCommand());
+    EXPECT_EQ("cat mid1 > out3 out4", edges[3]->EvaluateCommand());
+  }
+}
+
 TEST_F(GraphTest, InputsCollectorWithEscapes) {
   ASSERT_NO_FATAL_FAILURE(AssertParse(
       &state_,