Fix conda runtime on Windows (operating from within conda as well) (#56)

.github/workflows/python-build.yml

@@ -22,7 +22,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.10
+      - name: Set up Python
         uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
@@ -51,9 +51,12 @@ jobs:
       - name: Test building and packaging
         run: |
           (. .venv/build/${{ env.BIN_DIR }}/activate && cd partcad && python -m build && cd .. && deactivate)
-          (. .venv/build-cli/${{ env.BIN_DIR }}/activate && python -m pip install partcad/dist/partcad-[0-9].[0-9]*.[0-9]*-py3-none-any.whl && deactivate)
+          (. .venv/build-cli/${{ env.BIN_DIR }}/activate && python -m pip install -r partcad/requirements.txt && deactivate)
+          (. .venv/build-cli/${{ env.BIN_DIR }}/activate && python -m pip install --no-index --find-links=partcad/dist partcad && deactivate)
           (. .venv/build-cli/${{ env.BIN_DIR }}/activate && cd partcad-cli && python -m build && cd .. && deactivate)
       - name: Test installation
         run: |
-          (. .venv/install/${{ env.BIN_DIR }}/activate && python -m pip install     partcad/dist/partcad-[0-9].[0-9]*.[0-9]*-py3-none-any.whl && deactivate)
-          (. .venv/install/${{ env.BIN_DIR }}/activate && python -m pip install partcad-cli/dist/partcad_cli-[0-9].[0-9]*.[0-9]*-py3-none-any.whl && deactivate)
+          (. .venv/install/${{ env.BIN_DIR }}/activate && python -m pip install -r partcad/requirements.txt     && deactivate)
+          (. .venv/install/${{ env.BIN_DIR }}/activate && python -m pip install --no-index --find-links=partcad/dist     partcad && deactivate)
+          (. .venv/install/${{ env.BIN_DIR }}/activate && python -m pip install -r partcad-cli/requirements.txt && deactivate)
+          (. .venv/install/${{ env.BIN_DIR }}/activate && python -m pip install --no-index --find-links=partcad-cli/dist partcad-cli && deactivate)

.github/workflows/python-test.yml

@@ -14,13 +14,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
         python-version: ["3.9", "3.10", "3.11"]
+        exclude:
+          - os: "windows-latest"
+            python-version: "3.11"
     runs-on: ${{ matrix.os }}
 
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.10
+      - name: Set up Python
         uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
@@ -31,14 +34,19 @@ jobs:
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.*') }}
           restore-keys: |
             ${{ runner.os }}-pip-
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniconda-version: latest
+          activate-environment: pc-conda-env
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pytest
+          pip install -U pytest
           pip install -r partcad/requirements.txt
           # pip install -r partcad-cli/requirements.txt
       - name: Test with pytest
         env:
           PYTHONPATH: partcad/src
+          PYTHONWARNINGS: ignore
         run: |
-          pytest
+          pytest -x -p no:error-for-skips

.gitignore

@@ -1,4 +1,5 @@
 .conda
+.venv
 *.egg-info
 __pycache__
 .pytest_cache

.vscode/settings.json

@@ -20,26 +20,25 @@
     "**/LICENSE.txt": true,
     ".ocp_vscode": true,
     ".conda": true,
+    ".venv": true,
     ".tox": true,
     "**/.partcad": true,
     "partcad*/build": true,
     "partcad*/dist": true
   },
   "markdown.extension.toc.omittedFromToc": {
     "README.md": [
-        "# PartCAD"
-    ] 
+      "# PartCAD"
+    ]
   },
   "[python]": {
     "editor.defaultFormatter": "ms-python.black-formatter"
   },
-  "python.testing.pytestArgs": [
-  ],
+  "python.testing.pytestArgs": [],
   "python.testing.unittestEnabled": false,
   "python.testing.pytestEnabled": true,
   "python.terminal.executeInFileDir": true,
-  "python.terminal.launchArgs": [
-  ],
+  "python.terminal.launchArgs": [],
   "OcpCadViewer.advanced.autostart": false,
   "yaml.format.bracketSpacing": false // prevent YAML formatter from breaking Jinja2 templates
-}
+}

partcad/src/partcad/project_factory_tar.py

@@ -8,6 +8,7 @@
 #
 
 import hashlib
+import inspect
 import os
 import requests
 import tarfile
@@ -67,20 +68,27 @@ def _extract(self, tarball_url, cache_dir=None):
             try:
                 os.makedirs(cache_path)
 
-                if not self.import_rel_path is None:
-                    filter = lambda member, _: (
-                        member if member.name.startswith(self.import_rel_path) else None
-                    )
-                else:
-                    filter = lambda member, _: member
-
                 auth = None
                 if not (self.auth_user is None or self.auth_pass is None):
                     auth = (self.auth_user, self.auth_pass)
                 with requests.get(
                     tarball_url, stream=True, auth=auth
                 ) as rx, tarfile.open(fileobj=rx.raw, mode="r:gz") as tarobj:
-                    tarobj.extractall(cache_path, filter=filter)
+                    args = inspect.getfullargspec(tarobj.extractall)
+
+                    if "filter" in args.args:
+                        if not self.import_rel_path is None:
+                            filter = lambda member, _: (
+                                member
+                                if member.name.startswith(self.import_rel_path)
+                                else None
+                            )
+                        else:
+                            filter = lambda member, _: member
+
+                        tarobj.extractall(cache_path, filter=filter)
+                    else:
+                        tarobj.extractall(cache_path)
             except Exception as e:
                 raise RuntimeError(f"Failed to download the tarball: {e}")
 

partcad/src/partcad/runtime_python.py

@@ -11,7 +11,7 @@
 import pathlib
 import subprocess
 import sys
-
+import threading
 
 from . import runtime
 
@@ -23,6 +23,8 @@ def __init__(self, ctx, sandbox, version=None):
         super().__init__(ctx, "python-" + sandbox + "-" + version)
         self.version = version
 
+        self.lock = threading.Lock()
+
     def run(self, cmd, stdin=""):
         p = subprocess.Popen(
             cmd,

partcad/src/partcad/runtime_python_conda.py

@@ -17,33 +17,53 @@ class CondaPythonRuntime(runtime_python.PythonRuntime):
     def __init__(self, ctx, version=None):
         super().__init__(ctx, "conda", version)
 
-        if not self.initialized:
-            if shutil.which("conda") is None:
-                raise Exception(
-                    "ERROR: PartCAD is configured to use missing conda to execute Python scripts (CadQuery, build123d etc)"
-                )
-
-            try:
-                os.makedirs(self.path)
-                subprocess.run(
-                    ["conda", "create", "-y", "-p", self.path, "python=%s" % version]
-                )
-                subprocess.run(["conda", "install", "-y", "-p", self.path, "pip"])
-                self.initialized = True
-            except Exception as e:
-                shutil.rmtree(self.path)
-                raise e
+        self.conda_path = shutil.which("conda")
 
     def run(self, cmd, stdin=""):
-        return super().run(
-            [
-                "conda",
-                "run",
-                "--no-capture-output",
-                "-p",
-                self.path,
-                "python%s" % self.version,
-            ]
-            + cmd,
-            stdin,
-        )
+        with self.lock:
+            if not self.initialized:
+                if self.conda_path is None:
+                    raise Exception(
+                        "ERROR: PartCAD is configured to use conda, but conda is missing"
+                    )
+
+                try:
+                    os.makedirs(self.path)
+                    subprocess.run(
+                        [
+                            self.conda_path,
+                            "create",
+                            "-y",
+                            "-p",
+                            self.path,
+                            "python=%s" % self.version,
+                        ]
+                    )
+                    subprocess.run(
+                        [
+                            self.conda_path,
+                            "install",
+                            "-y",
+                            "-p",
+                            self.path,
+                            "pip",
+                        ]
+                    )
+                    self.initialized = True
+                except Exception as e:
+                    shutil.rmtree(self.path)
+                    raise e
+
+            return super().run(
+                [
+                    self.conda_path,
+                    "run",
+                    "--no-capture-output",
+                    "-p",
+                    self.path,
+                    "python",
+                    # "python%s" % self.version,  # This doesn't work on Windows
+                ]
+                + cmd,
+                stdin,
+            )

partcad/src/partcad/runtime_python_none.py

@@ -20,4 +20,11 @@ def __init__(self, ctx, version=None):
             self.initialized = True
 
     def run(self, cmd, stdin=""):
-        return super().run(["python" + self.version] + cmd, stdin)
+        return super().run(
+            [
+                "python",
+                # "python%s" % self.version, # This doesn't work on Windows
+            ]
+            + cmd,
+            stdin,
+        )

partcad/src/partcad/wrappers/wrapper_common.py

@@ -10,7 +10,8 @@
 # This script contains code shared by all wrapper scripts.
 
 import base64
-import fcntl
+
+# import fcntl # TODO(clairbee): replace it with whatever works on Windows if needed
 import os
 import pickle
 import sys
@@ -27,9 +28,9 @@ def handle_input():
     # - Comand line parameters
     path = sys.argv[1]
     # - Content passed via stdin
-    #   - Make stdin blocking so that we can read until EOF
-    flag = fcntl.fcntl(sys.stdin, fcntl.F_GETFL)
-    fcntl.fcntl(sys.stdin, fcntl.F_SETFL, flag & ~os.O_NONBLOCK)
+    # #   - Make stdin blocking so that we can read until EOF
+    # flag = fcntl.fcntl(sys.stdin, fcntl.F_GETFL)
+    # fcntl.fcntl(sys.stdin, fcntl.F_SETFL, flag & ~os.O_NONBLOCK)
     #   - Read until EOF
     input_str = sys.stdin.read()
     #   - Unpack the content received via stdin