hotfix: crash when macro lib files are gzipped (#629)

## Steps

* `Yosys.*`
  * Fixed blackbox Verilog and lib models causing a crash if they are
    gzipped and/or have the extension `.gz`.
    
## Tool Updates

* Relaxed requirement on `httpx` to include `0.28.X`, which has no removals
  compared to `0.27.0`.

Changelog.md

@@ -14,6 +14,23 @@
 ## Documentation
 -->
 
+# 2.3.2
+
+## Steps
+
+* `Yosys.*`
+  * Fixed blackbox Verilog and lib models causing a crash if they are
+    gzipped and/or have the extension `.gz`.
+
+## Tool Updates
+
+* Relaxed requirement on `httpx` to include `0.28.X`, which has no removals
+  compared to `0.27.0`.
+
+## Documentation
+
+* Clarified support for gzipped files in the Classic flow.
+
 # 2.3.1
 
 ## Tool Updates

docs/source/glossary.md

@@ -397,5 +397,23 @@ MPW
     of a wafer to be spread across multiple projects.
     
     {term}`OpenMPW` and {term}`chipIgnite` are examples of MPW projects.
+    
+dotlib
+
+    Also `.lib`.
+
+    A library format for macros including standard cells, modeling at an
+    abstract level the interface to and timing properties of a cell.
+    
+    Typically used for Synthesis and {term}`STA`.
+    
+Gzip
 
+    A free and open-source compression format. A great many number of tools
+    support Gzipped inputs transparently, i.e., any file beginning with the
+    bytes `1f 8b` is automatically decompressed without any special input
+    from the user.
+    
+    Gzipping is popular for text-heavy formats such as {term}`dotlib` or
+    {term}`SPEF` formats.
 ```

docs/source/usage/using_macros.md

@@ -56,7 +56,7 @@ views- the former of which is used in PnR and the latter is used for tape-out.
   * Used as a fallback during synthesis if neither Verilog headers nor regular
     netlists (`.gl.v`/`.nl.v`) exist. It is not recommended for this use as
     synthesis checks may fail.
-* Lib file (`.lib`): Optional
+* {term}`dotlib` file (`.lib`): Optional
   * May be used during STA (see [relevant section](#sta)).
   * Used as a last resort for synthesis if no Verilog header (`.vh`) or any
     netlists (`.nl.v`/`.gl.v`/`.pnl.v`) are available. It is not recommended for
@@ -83,13 +83,26 @@ thereof) and the values are a Python dataclass. You can find the API reference
 for the macros hashmap at {class}`openlane.common.Macro`, but a less mechanical
 explanation is as follows:
 
+```{tip}
+To save space in your repositories, {term}`Gzip`ped views may be supported
+depending on the flow. The Classic flow generally supports gzipping the
+following formats:
+
+* gds
+* lef
+* vh
+* lib
+* spef
+
+```
+
 * The keys contain the name of the Macro itself (not instances thereof.)
 * The values are:
   * A dictionary of instance names to instance objects
     * The instance objects in turn consist off:
       * `location`: A tuple of two numbers, in microns, indicating the location
         of the macro (optional)
-      * `orientation`: The orientation of the placed macro-- see page 250 of the
+      * `orientation`: The orientation of the placed macro-- see the
         {term}`LEFDEFREF` for a definition and visual.
   * `gds`: List of GDS files comprising the macro (usually only one)
   * `lef`: List of LEF files comprising the macro (usually only one)

openlane/common/misc.py

@@ -14,6 +14,7 @@
 import os
 import re
 import glob
+import gzip
 import typing
 import fnmatch
 import pathlib
@@ -29,6 +30,7 @@
     SupportsFloat,
     Union,
 )
+
 import httpx
 
 from .types import AnyPath, Path
@@ -374,3 +376,27 @@ def process_list_file(from_file: AnyPath) -> List[str]:
 
 def _get_process_limit() -> int:
     return int(os.getenv("_OPENLANE_MAX_CORES", os.cpu_count() or 1))
+
+
+def gzopen(filename, mode="rt"):
+    """
+    This method (tries to?) emulate the gzopen from the Linux Standard Base,
+    specifically this part:
+
+    If path refers to an uncompressed file, and mode refers to a read mode,
+    gzopen() shall attempt to open the file and return a gzFile object suitable
+    for reading directly from the file without any decompression.
+
+    gzip.open does not have this behavior.
+    """
+    try:
+        g = gzip.open(filename, mode=mode)
+        # Incredibly, it won't actually try to figure out if it's a gzipped
+        # file until you try to read from it.
+        if "r" in mode:
+            g.read(1)
+            g.seek(0)
+        return g
+    except gzip.BadGzipFile:
+        g.close()
+        return open(filename, mode=mode)

openlane/common/toolbox.py

@@ -39,7 +39,7 @@
 from deprecated.sphinx import deprecated
 
 
-from .misc import mkdirp
+from .misc import mkdirp, gzopen
 from .types import Path
 from .metrics import aggregate_metrics
 from .generic_dict import GenericImmutableDict, is_string
@@ -388,9 +388,9 @@ class State(IntEnum):
         excluded_cells_filter = Filter(excluded_cells)
 
         for file in input_lib_files:
-            input_lib_stream = open(file)
-            out_filename = f"{uuid.uuid4().hex}.lib"
-            out_path = os.path.join(self.tmp_dir, out_filename)
+            input_lib_stream = gzopen(file)
+            # can't be gzip -- abc cannot read gzipped lib files
+            out_path = os.path.join(self.tmp_dir, f"{uuid.uuid4().hex}.lib")
 
             state = State.initial
             brace_count = 0

openlane/scripts/pyosys/synthesize.py

@@ -32,7 +32,6 @@
 #  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
 import os
-import sys
 import json
 import shutil
 
@@ -280,9 +279,8 @@ def synthesize(
         d.run_pass("plugin", "-i", "ghdl")
         d.run_pass("ghdl", *vhdl_files, "-e", config["DESIGN_NAME"])
     else:
-        print(
+        ys.log_error(
             "Script called inappropriately: config must include either VERILOG_FILES or VHDL_FILES.",
-            file=sys.stderr,
         )
         exit(1)
 

openlane/scripts/pyosys/ys_common.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import os
 import sys
 from typing import Iterable, List, Union
 
@@ -20,7 +21,7 @@
     try:
         from pyosys import libyosys as ys
     except ImportError:
-        print(
+        ys.log_error(
             "Could not find pyosys in 'PYTHONPATH'-- make sure Yosys is compiled with ENABLE_PYTHON set to 1.",
             file=sys.stderr,
         )
@@ -122,11 +123,16 @@ def _Design_add_blackbox_models(
     define_args = [f"-D{define}" for define in defines]
 
     for model in models:
-        if model.endswith(".v") or model.endswith(".sv") or model.endswith(".vh"):
+        model_path, ext = os.path.splitext(model)
+        if ext == ".gz":
+            # Yosys transparently handles gzip compression
+            model_path, ext = os.path.splitext(model_path)
+
+        if ext in [".v", ".sv", ".vh"]:
             self.run_pass(
                 "read_verilog", "-sv", "-lib", *include_args, *define_args, model
             )
-        elif model.endswith(".lib"):
+        elif ext in [".lib"]:
             self.run_pass(
                 "read_liberty",
                 "-lib",
@@ -136,8 +142,8 @@ def _Design_add_blackbox_models(
                 model,
             )
         else:
-            print(
-                f"[ERROR] Black-box model '{model}' has an unrecognized file extension.",
+            ys.log_error(
+                f"Black-box model '{model}' has an unrecognized file extension: '{ext}'.",
                 file=sys.stderr,
             )
             sys.stderr.flush()

openlane/steps/pyosys.py

@@ -557,6 +557,11 @@ class Synthesis(SynthesisCommon):
     * ``design__instance__count``
     * ``design__instance_unmapped__count``
     * ``design__instance__area``
+
+    Note that Yosys steps do not currently support gzipped standard cell dotlib
+    files. They are however supported for macros:
+
+    https://github.com/YosysHQ/yosys/issues/4830
     """
 
     id = "Yosys.Synthesis"
@@ -576,6 +581,11 @@ class Resynthesis(SynthesisCommon):
     * ``design__instance__count``
     * ``design__instance_unmapped__count``
     * ``design__instance__area``
+
+    Note that Yosys steps do not currently support gzipped standard cell dotlib
+    files. They are however supported for macros:
+
+    https://github.com/YosysHQ/yosys/issues/4830
     """
 
     id = "Yosys.Resynthesis"
@@ -600,6 +610,11 @@ class VHDLSynthesis(SynthesisCommon):
     * ``design__instance__count``
     * ``design__instance_unmapped__count``
     * ``design__instance__area``
+
+    Note that Yosys steps do not currently support gzipped standard cell dotlib
+    files. They are however supported for macros:
+
+    https://github.com/YosysHQ/yosys/issues/4830
     """
 
     id = "Yosys.VHDLSynthesis"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "openlane"
-version = "2.3.1"
+version = "2.3.2"
 description = "An infrastructure for implementing chip design flows"
 authors = ["Efabless Corporation and Contributors <[email protected]>"]
 readme = "Readme.md"
@@ -21,7 +21,7 @@ lxml = ">=4.9.0"
 deprecated = ">=1.2.10,<2"
 libparse = ">=0.3.1,<1"
 psutil = ">=5.9.0"
-httpx = ">=0.22.0,<0.28"
+httpx = ">=0.22.0,<0.29"
 klayout = ">=0.29.0,<0.30.0"
 rapidfuzz = ">=3.9.0,<4"
 ioplace-parser = ">=0.3.0,<0.5.0"