Add a system to produce Windows script .exes. (#2665)

Use the stubs generated by the uv project for this. The lineage extends
back to distutils through the Posy project. The API is setup and
manually tested to produce working Windows scripts and the packaging
system is updated to embed these stubs in the Pex PEX, wheel and scies.
Actually using the API to create scripts when creating venvs on Windows
is still left to do.

Work towards #2658.

.github/workflows/ci.yml

@@ -10,6 +10,8 @@ env:
   # We have integration tests that exercise `--scie` support and these can trigger downloads from
   # GitHub Releases that needed elevated rate limit quota, which this gives.
   SCIENCE_AUTH_API_GITHUB_COM_BEARER: ${{ secrets.GITHUB_TOKEN }}
+  # We fetch Windows script executable stubs when building Pex.
+  _PEX_FETCH_WINDOWS_STUBS_BEARER: ${{ secrets.GITHUB_TOKEN }}
 concurrency:
   group: CI-${{ github.ref }}
   # Queue on all branches and tags, but only cancel overlapping PR burns.

.github/workflows/release.yml

@@ -8,6 +8,11 @@ on:
       tag:
         description: The tag to manually run a deploy for.
         required: true
+env:
+  # We build Pex `--scie`s that fetch from science.
+  SCIENCE_AUTH_API_GITHUB_COM_BEARER: ${{ secrets.GITHUB_TOKEN }}
+  # We fetch Windows script executable stubs when building Pex.
+  _PEX_FETCH_WINDOWS_STUBS_BEARER: ${{ secrets.GITHUB_TOKEN }}
 jobs:
   org-check:
     name: Check GitHub Organization

.gitignore

@@ -9,4 +9,5 @@ __pycache__/
 /compatible-tox-hack/compatible_tox_hack.egg-info/
 /dist/
 /docs/_static_dynamic/
+/pex/windows/stubs/
 

build-backend/pex_build/setuptools/build.py

@@ -1,12 +1,13 @@
 # Copyright 2024 Pex project contributors.
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-from __future__ import absolute_import
+from __future__ import absolute_import, print_function
 
 import hashlib
 import os.path
 import subprocess
 import sys
+from collections import OrderedDict
 from zipfile import ZIP_DEFLATED
 
 import pex_build
@@ -15,16 +16,31 @@
 # We re-export all setuptools' PEP-517 build backend hooks here for the build frontend to call.
 from setuptools.build_meta import *  # NOQA
 
-from pex import hashing, requirements
-from pex.common import open_zip, temporary_dir
+from pex import hashing, requirements, windows
+from pex.common import open_zip, safe_copy, safe_mkdir, temporary_dir
 from pex.orderedset import OrderedSet
 from pex.pep_376 import Hash, InstalledFile, Record
+from pex.typing import cast
 from pex.version import __version__
 
 if pex_build.TYPE_CHECKING:
     from typing import Any, Dict, List, Optional
 
 
+def build_sdist(
+    sdist_directory,  # type: str
+    config_settings=None,  # type: Optional[Dict[str, Any]]
+):
+    # type: (...) -> str
+
+    for stub in windows.fetch_all_stubs():
+        print("Embedded Windows script stub", stub.path, file=sys.stderr)
+
+    return cast(
+        str, setuptools.build_meta.build_sdist(sdist_directory, config_settings=config_settings)
+    )
+
+
 def get_requires_for_build_wheel(config_settings=None):
     # type: (Optional[Dict[str, Any]]) -> List[str]
 
@@ -45,15 +61,42 @@ def build_wheel(
 ):
     # type: (...) -> str
 
-    wheel = setuptools.build_meta.build_wheel(
-        wheel_directory, config_settings=config_settings, metadata_directory=metadata_directory
-    )  # type: str
-    if pex_build.INCLUDE_DOCS:
-        wheel_path = os.path.join(wheel_directory, wheel)
-        with temporary_dir() as chroot:
-            with open_zip(wheel_path) as zip_fp:
-                zip_fp.extractall(chroot)
+    wheel = cast(
+        str,
+        setuptools.build_meta.build_wheel(
+            wheel_directory, config_settings=config_settings, metadata_directory=metadata_directory
+        ),
+    )
+    wheel_path = os.path.join(wheel_directory, wheel)
+    with temporary_dir() as chroot:
+        with open_zip(wheel_path) as zip_fp:
+            zip_fp.extractall(chroot)
+
+        dist_info_dir = "pex-{version}.dist-info".format(version=__version__)
+        record_path = os.path.join(chroot, dist_info_dir, "RECORD")
+        with open(record_path) as fp:
+            installed_files_by_path = OrderedDict(
+                (installed_file.path, installed_file) for installed_file in Record.read(fp)
+            )
+
+        for stub in windows.fetch_all_stubs():
+            stub_relpath = os.path.relpath(
+                stub.path, os.path.dirname(os.path.dirname(os.path.dirname(windows.__file__)))
+            )
+            if stub_relpath in installed_files_by_path:
+                continue
+            stub_dst = os.path.join(chroot, stub_relpath)
+            safe_mkdir(os.path.dirname(stub_dst))
+            safe_copy(stub.path, stub_dst)
+            data = stub.read_data()
+            installed_files_by_path[stub_relpath] = InstalledFile(
+                path=stub_relpath,
+                hash=Hash.create(hashlib.sha256(data)),
+                size=len(data),
+            )
+            print("Embedded Windows script stub", stub.path, file=sys.stderr)
 
+        if pex_build.INCLUDE_DOCS:
             out_dir = os.path.join(chroot, "pex", "docs")
             subprocess.check_call(
                 args=[
@@ -63,34 +106,31 @@ def build_wheel(
                     out_dir,
                 ]
             )
-            dist_info_dir = "pex-{version}.dist-info".format(version=__version__)
-            record_path = os.path.join(chroot, dist_info_dir, "RECORD")
-            with open(record_path) as fp:
-                installed_files = list(Record.read(fp))
             for root, _, files in os.walk(out_dir):
                 for f in files:
                     src = os.path.join(root, f)
                     dst = os.path.relpath(src, chroot)
                     hasher = hashlib.sha256()
                     hashing.file_hash(src, digest=hasher)
-                    installed_files.append(
-                        InstalledFile(path=dst, hash=Hash.create(hasher), size=os.path.getsize(src))
+                    installed_files_by_path[dst] = InstalledFile(
+                        path=dst, hash=Hash.create(hasher), size=os.path.getsize(src)
                     )
-            Record.write(record_path, installed_files)
-            with open_zip(wheel_path, "w", compression=ZIP_DEFLATED) as zip_fp:
-
-                def add_top_level_dir(name):
-                    # type: (str) -> None
-                    top = os.path.join(chroot, name)
-                    zip_fp.write(top, name + "/")
-                    for root, dirs, files in os.walk(top):
-                        dirs[:] = sorted(dirs)
-                        for path in sorted(files) + dirs:
-                            src = os.path.join(root, path)
-                            dst = os.path.relpath(src, chroot)
-                            zip_fp.write(src, dst)
-
-                add_top_level_dir("pex")
-                add_top_level_dir(dist_info_dir)
+
+        Record.write(record_path, installed_files_by_path.values())
+        with open_zip(wheel_path, "w", compression=ZIP_DEFLATED) as zip_fp:
+
+            def add_top_level_dir(name):
+                # type: (str) -> None
+                top = os.path.join(chroot, name)
+                zip_fp.write(top, name + "/")
+                for root, dirs, files in os.walk(top):
+                    dirs[:] = sorted(dirs)
+                    for path in sorted(files) + dirs:
+                        src = os.path.join(root, path)
+                        dst = os.path.relpath(src, chroot)
+                        zip_fp.write(src, dst)
+
+            add_top_level_dir("pex")
+            add_top_level_dir(dist_info_dir)
 
     return wheel

pex/os.py

@@ -6,15 +6,49 @@
 import os
 import sys
 
+from pex.enum import Enum
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from typing import Any, List, NoReturn, Text, Tuple, Union
 
+
+class _CurrentOs(object):
+    def __get__(self, obj, objtype=None):
+        # type: (...) -> Os.Value
+        if not hasattr(self, "_current"):
+            # N.B.: Python 2.7 uses "linux2".
+            if sys.platform.startswith("linux"):
+                self._current = Os.LINUX
+            elif sys.platform == "darwin":
+                self._current = Os.MACOS
+            elif sys.platform == "win32":
+                self._current = Os.WINDOWS
+            if not hasattr(self, "_current"):
+                raise ValueError(
+                    "The current operating system is not supported!: {system}".format(
+                        system=sys.platform
+                    )
+                )
+        return self._current
+
+
+class Os(Enum["Os.Value"]):
+    class Value(Enum.Value):
+        pass
+
+    LINUX = Value("linux")
+    MACOS = Value("macos")
+    WINDOWS = Value("windows")
+    CURRENT = _CurrentOs()
+
+
+Os.seal()
+
 # N.B.: Python 2.7 uses "linux2".
-LINUX = sys.platform.startswith("linux")
-MAC = sys.platform == "darwin"
-WINDOWS = sys.platform == "win32"
+LINUX = Os.CURRENT is Os.LINUX
+MAC = Os.CURRENT is Os.MACOS
+WINDOWS = Os.CURRENT is Os.WINDOWS
 
 
 HOME_ENV_VAR = "USERPROFILE" if WINDOWS else "HOME"

pex/pex_builder.py

@@ -552,7 +552,7 @@ def _prepare_bootstrap(self):
         )
 
         bootstrap_digest = hashlib.sha1()
-        bootstrap_packages = ["cache", "fs", "repl", "third_party", "venv"]
+        bootstrap_packages = ["cache", "fs", "repl", "third_party", "venv", "windows"]
         if self._pex_info.includes_tools:
             bootstrap_packages.extend(["commands", "tools"])
 

pex/scie/__init__.py

@@ -23,11 +23,11 @@
     ScieConfiguration,
     ScieInfo,
     ScieOptions,
-    SciePlatform,
     ScieStyle,
     Url,
 )
 from pex.scie.science import SCIENCE_RELEASES_URL, SCIENCE_REQUIREMENT
+from pex.sysconfig import SysPlatform
 from pex.typing import TYPE_CHECKING, cast
 from pex.variables import ENV, Variables
 
@@ -40,7 +40,6 @@
     "ScieConfiguration",
     "ScieInfo",
     "ScieOptions",
-    "SciePlatform",
     "ScieStyle",
     "build",
     "extract_options",
@@ -162,11 +161,11 @@ def register_options(parser):
         dest="scie_platforms",
         default=[],
         action="append",
-        type=SciePlatform.parse,
+        type=SysPlatform.parse,
         choices=[
             platform
-            for platform in SciePlatform.values()
-            if platform not in (SciePlatform.WINDOWS_AARCH64, SciePlatform.WINDOWS_X86_64)
+            for platform in SysPlatform.values()
+            if platform not in (SysPlatform.WINDOWS_AARCH64, SysPlatform.WINDOWS_X86_64)
         ],
         help=(
             "The platform to produce the native PEX scie executable for. Can be specified multiple "