Merge pull request #114 from FragileTech/ci

Fix codecov

.github/workflows/push.yml

@@ -137,9 +137,9 @@ jobs:
       - name: Install build dependencies
         run: |
           set -x
+          pip install uv
           rye install bump2version
           rye install twine
-          rye install uv
 
       - name: Create unique version for test.pypi
         run: |
@@ -166,23 +166,26 @@ jobs:
 #          skip-existing: true
 
       - name: Install dependencies
+        env:
+          UV_SYSTEM_PYTHON: 1
         run: |
           set -x
           sudo apt-get install -y xvfb
           sudo MUJOCO_PATH=/home/runner/.mujoco/ make install-envs
           rye lock --all-features
           uv pip install -r requirements.lock
           uv pip install dist/*.whl
-          ROM_PASSWORD=${{ secrets.ROM_PASSWORD }} uv run import_retro_roms.py
+          ROM_PASSWORD=${{ secrets.ROM_PASSWORD }} python -m plangym.scripts.import_retro_roms
           
 
       - name: Test package
+        env:
+          UV_SYSTEM_PYTHON: 1
         run: |
           set -x
           rm -rf $PROJECT_DIR
           find . -name "*.pyc" -delete
-          PYVIRTUALDISPLAY_DISPLAYFD=0 SKIP_CLASSIC_CONTROL=1 xvfb-run -s "-screen 0 1400x900x24" uv run pytest -n auto -s -o log_cli=true -o log_cli_level=info tests
-          PYVIRTUALDISPLAY_DISPLAYFD=0 xvfb-run -s "-screen 0 1400x900x24" uv run pytest -s -o log_cli=true -o log_cli_level=info tests/control/test_classic_control.py
+          make test
 
   bump-version:
     name: Bump package version

Dockerfile

@@ -1,35 +1,20 @@
-FROM ubuntu:20.04
-ARG JUPYTER_PASSWORD="plangym"
-ARG ROM_PASSWORD=""
-
-ENV BROWSER=/browser \
-    LC_ALL=en_US.UTF-8 \
-    LANG=en_US.UTF-8
-COPY Makefile.docker Makefile
-
-RUN apt-get update && \
-	apt-get install -y --no-install-suggests --no-install-recommends make cmake && \
-    make install-python3.8 && \
-    make install-common-dependencies && \
-    make install-python-libs && \
-    make install-env-deps
-
-COPY . plangym/
-
-RUN cd plangym  \
-    && make install-mujoco \
-    && python3 -m pip install -r requirements-lint.txt \
-    && python3 -m pip install -r requirements-test.txt \
-    && python3 -m pip install -r requirements.txt \
-    && python3 -m pip install ipython jupyter \
-    && python3 -m pip install -e . \
-    && ROM_PASSWORD=${ROM_PASSWORD} make import-roms \
-    && git config --global init.defaultBranch master \
-    && git config --global user.name "Whoever" \
-    && git config --global user.email "[email protected]"
-
-RUN make remove-dev-packages
-
-RUN mkdir /root/.jupyter && \
-    echo 'c.NotebookApp.token = "'${JUPYTER_PASSWORD}'"' > /root/.jupyter/jupyter_notebook_config.py
-CMD jupyter notebook --allow-root --port 8080 --ip 0.0.0.0
+FROM ubuntu:22.04
+
+
+RUN apt-get update && \
+	apt-get install -y --no-install-suggests --no-install-recommends \
+    make cmake git xvfb build-essential curl ssh wget ca-certificates swig \
+    python3 python3-dev && \
+    wget -O - https://bootstrap.pypa.io/get-pip.py | python3
+
+RUN python3 -m pip install uv
+COPY src ./src
+COPY requirements.lock .
+COPY pyproject.toml .
+COPY src/plangym/scripts/import_retro_roms.py .
+COPY LICENSE .
+COPY README.md .
+COPY ROMS.zip .
+
+RUN uv pip install --no-cache --system -r requirements.lock
+RUN python3 src/plangym/scripts/import_retro_roms.py

Makefile

@@ -35,7 +35,7 @@ ifeq (${ROM_PASSWORD}, "NO_PASSWORD")
 else
 	unzip -o -P ${ROM_PASSWORD} ${ROM_FILE}
 endif
-	python3 import_retro_roms.py
+	python3 src/plangym/scripts/import_retro_roms.py
 
 .PHONY: install-envs
 install-envs:

_old_Dockerfile

@@ -0,0 +1,35 @@
+FROM ubuntu:20.04
+ARG JUPYTER_PASSWORD="plangym"
+ARG ROM_PASSWORD=""
+
+ENV BROWSER=/browser \
+    LC_ALL=en_US.UTF-8 \
+    LANG=en_US.UTF-8
+COPY Makefile.docker Makefile
+
+RUN apt-get update && \
+	apt-get install -y --no-install-suggests --no-install-recommends make cmake curl ssh && \
+    make install-python3.8 && \
+    make install-common-dependencies && \
+    make install-python-libs && \
+    make install-env-deps
+
+COPY . plangym/
+
+RUN cd plangym  \
+    && make install-mujoco \
+    && python3 -m pip install -r requirements-lint.txt \
+    && python3 -m pip install -r requirements-test.txt \
+    && python3 -m pip install -r requirements.txt \
+    && python3 -m pip install ipython jupyter \
+    && python3 -m pip install -e . \
+    && ROM_PASSWORD=${ROM_PASSWORD} make import-roms \
+    && git config --global init.defaultBranch master \
+    && git config --global user.name "Whoever" \
+    && git config --global user.email "[email protected]"
+
+RUN make remove-dev-packages
+
+RUN mkdir /root/.jupyter && \
+    echo 'c.NotebookApp.token = "'${JUPYTER_PASSWORD}'"' > /root/.jupyter/jupyter_notebook_config.py
+CMD jupyter notebook --allow-root --port 8080 --ip 0.0.0.0

pyproject.toml

@@ -69,7 +69,7 @@ style = { chain = ["ruff check --fix-only  --unsafe-fixes tests src", "ruff form
 check = { chain = ["ruff check --diff tests src", "ruff format --diff tests src"]}  #,"mypy src tests" ] }
 test = { chain = ["test:doctest", "test:parallel", "test:singlecore"] }
 codecov = { chain = ["codecov:parallel", "codecov:singlecore"] }
-import-roms = { cmd = "python3 import_retro_roms.py" }
+import-roms = { cmd = "python3 src/plangym/scripts/import_retro_roms.py" }
 "test:parallel" = { cmd = "pytest -n auto -s -o log_cli=true -o log_cli_level=info tests", env-file = ".multicore.env" }
 "test:singlecore" = { cmd = "pytest -s -o log_cli=true -o log_cli_level=info tests/control/test_classic_control.py", env-file = ".onecore.env" }
 "test:doctest" = { cmd = "pytest --doctest-modules -n 0 -s -o log_cli=true -o log_cli_level=info src", env-file = ".multicore.env" }

import_retro_roms.py → src/plangym/scripts/import_retro_roms.py

@@ -1,9 +1,12 @@
 import os
 import sys
 import zipfile
+import logging
 
 import retro.data
 
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+
 
 def _check_zipfile(f, process_f):
     with zipfile.ZipFile(f) as zf:
@@ -36,14 +39,18 @@ def main():
     }
     EMU_EXTENSIONS.update(emu_extensions)
     paths = sys.argv[1:] or ["."]
+    logging.info(f"Importing ROMs from: {paths}")
+    logging.info("Fetching known hashes")
     known_hashes = retro.data.get_known_hashes()
+    logging.info(f"Found {len(known_hashes)} known hashes")
     imported_games = 0
 
     def save_if_matches(filename, f):
         nonlocal imported_games
         try:
             data, hash = retro.data.groom_rom(filename, f)
         except (OSError, ValueError):
+            logging.warning(f"Failed to process file: {filename}")
             return
         if hash in known_hashes:
             game, ext, curpath = known_hashes[hash]
@@ -53,20 +60,23 @@ def save_if_matches(filename, f):
             with open(rompath, "wb") as file:  # noqa: FURB103
                 file.write(data)
             imported_games += 1
+            logging.info(f"Imported game: {game} to {rompath}")
 
     for path in paths:  # noqa: PLR1702
         for root, dirs, files in os.walk(path):
             for filename in files:
+                logging.info(f"Processing file: {root}/{filename}")
                 filepath = os.path.join(root, filename)  # noqa: PTH118
                 with open(filepath, "rb") as f:
                     _root, ext = os.path.splitext(filename)  # noqa: PTH122
                     if ext == ".zip":
                         try:
                             _check_zipfile(f, save_if_matches)
                         except (zipfile.BadZipFile, RuntimeError, OSError):
-                            pass
+                            logging.error(f"Failed to process zip file: {filepath}")
                     else:
                         save_if_matches(filename, f)
+    logging.info(f"Total imported games: {imported_games}")
 
 
 if __name__ == "__main__":