Parallel processing support

oclint-json-compilation-database

@@ -7,12 +7,31 @@ import argparse
 import re
 import subprocess
 import sys
+import queue
+import threading
+import multiprocessing
+import shutil
 
 OCLINT_BIN_FOLDER = os.path.dirname(os.path.abspath(__file__))
 OCLINT_BIN = OCLINT_BIN_FOLDER + os.sep + "oclint"
 if platform.system() == "Windows":
     OCLINT_BIN += ".exe"
 
+
+def run_lint(oclint_arguments, queue, lock):
+    """Takes filenames out of queue and runs oclint on them."""
+    while True:
+        name = queue.get()
+        invocation = oclint_arguments + [name]
+        proc = subprocess.Popen(invocation, stdout=subprocess.PIPE, stderr=sys.stdout.fileno())
+        out, err = proc.communicate()
+        lock.acquire()
+        print("OC lint parsing file: " + name)
+        sys.stdout.write(out.decode("utf-8", errors="backslashreplace"))
+        lock.release()
+        queue.task_done()
+
+
 arg_parser = argparse.ArgumentParser(description='OCLint for JSON Compilation Database (compile_commands.json)')
 arg_parser.add_argument("-v", action="store_true", dest="invocation", help="show invocation command with arguments")
 arg_parser.add_argument('-debug', '--debug', action="store_true", dest="debug", help="invoke OCLint in debug mode")
@@ -21,6 +40,8 @@ arg_parser.add_argument('-e', '-exclude', '--exclude', action='append', dest='ex
 arg_parser.add_argument('-p', action='store', metavar='build-path', dest='build_path', default=os.getcwd(),
                         help="specify the directory containing compile_commands.json")
 arg_parser.add_argument('oclint_args', nargs='*', help="arguments that are passed to OCLint invocation")
+arg_parser.add_argument('-j', type=int, default=0,
+                        help='number of oclint instances to be run in parallel.')
 args = arg_parser.parse_args()
 
 def get_source_path(file_attr, dir_attr):
@@ -48,8 +69,18 @@ def source_list_exclusion_filter(source_list, exclusion_filter):
             filtered_list.append(path)
     return filtered_list
 
-if not source_exist_at(OCLINT_BIN):
-    print("Error: OCLint executable file not found.")
+
+def check_working_oclint(oclint_bin) -> bool:
+    try:
+        with open(os.devnull, 'w') as dev_null:
+            subprocess.check_call([oclint_bin, "--version"], stdout=dev_null)
+    except subprocess.CalledProcessError:
+        print("Error: OCLint executable was not found or it does not work.")
+        return False
+    return True
+
+
+if not check_working_oclint(OCLINT_BIN):
     sys.exit(99)
 
 json_compilation_database = os.path.join(args.build_path, 'compile_commands.json')
@@ -63,12 +94,12 @@ for file_item in compilation_database:
     file_path = file_item["file"]
     if not platform.system() == "Windows":
         file_path = get_source_path(file_item["file"], file_item["directory"])
-    if source_exist_at(file_path) and not file_path in source_list:
+    if source_exist_at(file_path) and file_path not in source_list:
         source_list.append(file_path)
 if args.includes:
     matched_list = []
     for inclusion_filter in args.includes:
-        matched_list.extend( source_list_inclusion_filter(source_list, inclusion_filter) )
+        matched_list.extend(source_list_inclusion_filter(source_list, inclusion_filter))
     source_list = matched_list
 if args.excludes:
     for exclusion_filter in args.excludes:
@@ -78,10 +109,39 @@ if args.oclint_args:
     oclint_arguments += args.oclint_args
 if args.debug:
     oclint_arguments.append('-debug')
-oclint_arguments += source_list
+
+max_task = args.j
+if max_task == 0:
+    max_task = multiprocessing.cpu_count()
+
 if args.invocation:
     print('------------------------------ OCLint ------------------------------')
     print(subprocess.list2cmdline(oclint_arguments))
     print('--------------------------------------------------------------------')
-exit_code = subprocess.call(oclint_arguments)
-sys.exit(exit_code)
+
+try:
+    # Spin up a bunch of oclint-launching threads.
+    queue = queue.Queue(max_task)
+    lock = threading.RLock()
+    for _ in range(max_task):
+        t = threading.Thread(target=run_lint,
+                             args=(oclint_arguments, queue, lock))
+        t.daemon = True
+        t.start()
+
+    # Fill the queue with files.
+    for name in source_list:
+        queue.put(name)
+
+    # Wait for all threads to be done.
+    queue.join()
+
+except KeyboardInterrupt:
+    # This is a sad hack. Unfortunately subprocess goes
+    # bonkers with ctrl-c and we start forking merrily.
+    print('\nCtrl-C detected, goodbye.')
+    if args.fix:
+        shutil.rmtree(tmpdir)
+    os.kill(0, 9)
+
+sys.exit(0)