cffi: Support using --system-zstd

make_cffi.py

@@ -15,19 +15,6 @@
 
 import cffi
 
-HERE = os.path.abspath(os.path.dirname(__file__))
-
-SOURCES = [
-    "zstd/zstd.c",
-]
-
-# Headers whose preprocessed output will be fed into cdef().
-HEADERS = [os.path.join(HERE, "zstd", p) for p in ("zstd.h", "zdict.h")]
-
-INCLUDE_DIRS = [
-    os.path.join(HERE, "zstd"),
-]
-
 # cffi can't parse some of the primitives in zstd.h. So we invoke the
 # preprocessor and feed its output into cffi.
 compiler = distutils.ccompiler.new_compiler()
@@ -158,59 +145,96 @@ def normalize_output(output):
     return b"\n".join(lines)
 
 
-ffi = cffi.FFI()
-# zstd.h uses a possible undefined MIN(). Define it until
-# https://github.com/facebook/zstd/issues/976 is fixed.
-# *_DISABLE_DEPRECATE_WARNINGS prevents the compiler from emitting a warning
-# when cffi uses the function. Since we statically link against zstd, even
-# if we use the deprecated functions it shouldn't be a huge problem.
-ffi.set_source(
-    "zstandard._cffi",
-    """
-#define MIN(a,b) ((a)<(b) ? (a) : (b))
-#define ZSTD_STATIC_LINKING_ONLY
-#define ZSTD_DISABLE_DEPRECATE_WARNINGS
-#include <zstd.h>
-#define ZDICT_STATIC_LINKING_ONLY
-#define ZDICT_DISABLE_DEPRECATE_WARNINGS
-#include <zdict.h>
-""",
-    sources=SOURCES,
-    include_dirs=INCLUDE_DIRS,
-)
-
-DEFINE = re.compile(b"^\\#define ([a-zA-Z0-9_]+) ")
-
-sources = []
-
-# Feed normalized preprocessor output for headers into the cdef parser.
-for header in HEADERS:
-    preprocessed = preprocess(header)
-    sources.append(normalize_output(preprocessed))
-
-    # #define's are effectively erased as part of going through preprocessor.
-    # So perform a manual pass to re-add those to the cdef source.
-    with open(header, "rb") as fh:
-        for line in fh:
-            line = line.strip()
-            m = DEFINE.match(line)
-            if not m:
-                continue
+def get_ffi(system_zstd = False):
+    zstd_sources = []
+    include_dirs = []
+    libraries = []
 
-            if m.group(1) == b"ZSTD_STATIC_LINKING_ONLY":
-                continue
+    if not system_zstd:
+        here = os.path.abspath(os.path.dirname(__file__))
 
-            # The parser doesn't like some constants with complex values.
-            if m.group(1) in (b"ZSTD_LIB_VERSION", b"ZSTD_VERSION_STRING"):
-                continue
+        zstd_sources += [
+            "zstd/zstd.c",
+        ]
+
+        # Headers whose preprocessed output will be fed into cdef().
+        headers = [os.path.join(here, "zstd", p) for p in ("zstd.h", "zdict.h")]
+
+        include_dirs += [
+            os.path.join(here, "zstd"),
+        ]
+    else:
+        libraries += ["zstd"]
+
+        # Locate headers using the preprocessor.
+        include_re = re.compile(r'^# \d+ "([^"]+/(?:zstd|zdict)\.h)"')
+        with tempfile.TemporaryDirectory() as temp_dir:
+            with open(os.path.join(temp_dir, "input.h"), "w") as f:
+                f.write("#include <zstd.h>\n#include <zdict.h>\n")
+            compiler.preprocess(os.path.join(temp_dir, "input.h"),
+                                os.path.join(temp_dir, "output.h"))
+            with open(os.path.join(temp_dir, "output.h"), "r") as f:
+                headers = list({
+                    m.group(1) for m in map(include_re.match, f)
+                    if m is not None
+                })
+
+    ffi = cffi.FFI()
+    # zstd.h uses a possible undefined MIN(). Define it until
+    # https://github.com/facebook/zstd/issues/976 is fixed.
+    # *_DISABLE_DEPRECATE_WARNINGS prevents the compiler from emitting a warning
+    # when cffi uses the function. Since we statically link against zstd, even
+    # if we use the deprecated functions it shouldn't be a huge problem.
+    ffi.set_source(
+        "zstandard._cffi",
+        """
+    #define MIN(a,b) ((a)<(b) ? (a) : (b))
+    #define ZSTD_STATIC_LINKING_ONLY
+    #define ZSTD_DISABLE_DEPRECATE_WARNINGS
+    #include <zstd.h>
+    #define ZDICT_STATIC_LINKING_ONLY
+    #define ZDICT_DISABLE_DEPRECATE_WARNINGS
+    #include <zdict.h>
+    """,
+        sources=zstd_sources,
+        include_dirs=include_dirs,
+        libraries=libraries,
+    )
+
+    DEFINE = re.compile(b"^\\#define ([a-zA-Z0-9_]+) ")
+
+    sources = []
+
+    # Feed normalized preprocessor output for headers into the cdef parser.
+    for header in headers:
+        preprocessed = preprocess(header)
+        sources.append(normalize_output(preprocessed))
+
+        # #define's are effectively erased as part of going through preprocessor.
+        # So perform a manual pass to re-add those to the cdef source.
+        with open(header, "rb") as fh:
+            for line in fh:
+                line = line.strip()
+                m = DEFINE.match(line)
+                if not m:
+                    continue
+
+                if m.group(1) == b"ZSTD_STATIC_LINKING_ONLY":
+                    continue
+
+                # The parser doesn't like some constants with complex values.
+                if m.group(1) in (b"ZSTD_LIB_VERSION", b"ZSTD_VERSION_STRING"):
+                    continue
+
+                # The ... is magic syntax by the cdef parser to resolve the
+                # value at compile time.
+                sources.append(m.group(0) + b" ...")
 
-            # The ... is magic syntax by the cdef parser to resolve the
-            # value at compile time.
-            sources.append(m.group(0) + b" ...")
+    cdeflines = b"\n".join(sources).splitlines()
+    cdeflines = [line for line in cdeflines if line.strip()]
+    ffi.cdef(b"\n".join(cdeflines).decode("latin1"))
+    return ffi
 
-cdeflines = b"\n".join(sources).splitlines()
-cdeflines = [line for line in cdeflines if line.strip()]
-ffi.cdef(b"\n".join(cdeflines).decode("latin1"))
 
 if __name__ == "__main__":
-    ffi.compile()
+    get_ffi().compile()

setup.py

@@ -112,7 +112,7 @@
 if CFFI_BACKEND and cffi:
     import make_cffi
 
-    extensions.append(make_cffi.ffi.distutils_extension())
+    extensions.append(make_cffi.get_ffi(system_zstd=SYSTEM_ZSTD).distutils_extension())
 
 version = None