From 0783b9e81328729353cfad7ec25391c03d7f772a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A1szl=C3=B3=20Csomor?= <laszlocsomor@google.com>
Date: Fri, 11 Aug 2017 10:28:36 +0200
Subject: [PATCH] Rewrite //:combine_distfiles.sh in Python

It can only pack to zip for now (packing to tar is
not trivial and I haven't figured it out yet).

This allows building //:bazel-distfile on Windows.
Previously it was either timing out or taking so
long that it was unbearable (over 10 minutes).
I never waited long enough to see it build.

The new Python version runs under just a few
seconds.

Change-Id: I3264eb7132dd58c581c4216e5bbab035a79d716d
PiperOrigin-RevId: 164954162
---
 BUILD                                         | 15 ++++--
 combine_distfiles.py                          | 50 +++++++++++++++++++
 ...istfiles.sh => combine_distfiles_to_tar.sh |  9 +---
 src/BUILD                                     |  1 +
 4 files changed, 64 insertions(+), 11 deletions(-)
 create mode 100755 combine_distfiles.py
 rename combine_distfiles.sh => combine_distfiles_to_tar.sh (86%)

diff --git a/BUILD b/BUILD
index e4064a44cb4da1..3c506215c87983 100644
--- a/BUILD
+++ b/BUILD
@@ -68,6 +68,13 @@ pkg_tar(
     visibility = ["//:__subpackages__"],
 )
 
+py_binary(
+    name = "combine_distfiles",
+    srcs = ["combine_distfiles.py"],
+    visibility = ["//visibility:private"],
+    deps = ["//src:create_embedded_tools_lib"],
+)
+
 genrule(
     name = "bazel-distfile",
     srcs = [
@@ -75,8 +82,8 @@ genrule(
         "//src:derived_java_srcs",
     ],
     outs = ["bazel-distfile.zip"],
-    cmd = "$(location :combine_distfiles.sh) $@ $(SRCS)",
-    tools = ["combine_distfiles.sh"],
+    cmd = "$(location :combine_distfiles) $@ $(SRCS)",
+    tools = [":combine_distfiles"],
     # Public but bazel-only visibility.
     visibility = ["//:__subpackages__"],
 )
@@ -88,8 +95,8 @@ genrule(
         "//src:derived_java_srcs",
     ],
     outs = ["bazel-distfile.tar"],
-    cmd = "env USE_TAR=YES $(location :combine_distfiles.sh) $@ $(SRCS)",
-    tools = ["combine_distfiles.sh"],
+    cmd = "$(location :combine_distfiles_to_tar.sh) $@ $(SRCS)",
+    tools = ["combine_distfiles_to_tar.sh"],
     # Public but bazel-only visibility.
     visibility = ["//:__subpackages__"],
 )
diff --git a/combine_distfiles.py b/combine_distfiles.py
new file mode 100755
index 00000000000000..e3fcd870f8b16c
--- /dev/null
+++ b/combine_distfiles.py
@@ -0,0 +1,50 @@
+# pylint: disable=g-bad-file-header
+# pylint: disable=g-direct-third-party-import
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Creates the Bazel source distribution archive."""
+
+import contextlib
+import os.path
+import sys
+import zipfile
+
+from src.create_embedded_tools_lib import copy_tar_to_zip
+from src.create_embedded_tools_lib import copy_zip_to_zip
+
+
+def main():
+  output_zip = os.path.join(os.getcwd(), sys.argv[1])
+  input_files = sorted(sys.argv[2:])
+
+  # Copy all the input_files into output_zip.
+  # Adding contextlib.closing to be python 2.6 (for centos 6.7) compatible
+  with contextlib.closing(
+      zipfile.ZipFile(output_zip, "w", zipfile.ZIP_DEFLATED)) as output_zip:
+
+    def _normalize(path):
+      return path[2:] if path.startswith("./") else path
+
+    for input_file in input_files:
+      if input_file.endswith(".tar"):
+        copy_tar_to_zip(output_zip, input_file, _normalize)
+      elif input_file.endswith(".zip"):
+        copy_zip_to_zip(output_zip, input_file, _normalize)
+      else:
+        raise Exception("unknown archive type \"%s\"" % input_file)
+
+
+if __name__ == "__main__":
+  main()
diff --git a/combine_distfiles.sh b/combine_distfiles_to_tar.sh
similarity index 86%
rename from combine_distfiles.sh
rename to combine_distfiles_to_tar.sh
index 77666d15306ce7..40a8f4a3367f8d 100755
--- a/combine_distfiles.sh
+++ b/combine_distfiles_to_tar.sh
@@ -40,10 +40,5 @@ do
     (cd "${PACKAGE_DIR}" && ${UNPACK} "${ARCHIVE}")
 done
 
-if [ -n "${USE_TAR:-}" ]
-then
-  (cd "${PACKAGE_DIR}" && tar -c -f "${OUTPUT}" --group=0 --owner=0 \
-     $(find . -type f | sort))
-else
-  (cd "${PACKAGE_DIR}" && find . -type f | sort | zip -qDX@ "${OUTPUT}")
-fi
+(cd "${PACKAGE_DIR}" && tar -c -f "${OUTPUT}" --group=0 --owner=0 \
+   $(find . -type f | sort))
diff --git a/src/BUILD b/src/BUILD
index d788f1f64484c2..ce62eae910bb3c 100644
--- a/src/BUILD
+++ b/src/BUILD
@@ -125,6 +125,7 @@ filegroup(
 py_library(
     name = "create_embedded_tools_lib",
     srcs = ["create_embedded_tools_lib.py"],
+    visibility = ["//:__pkg__"],
 )
 
 py_binary(