From b3f66d2626215410dbe1cfb29ac775125c974977 Mon Sep 17 00:00:00 2001 From: Adam Gleave Date: Tue, 19 Nov 2019 15:04:29 -0800 Subject: [PATCH] Migrate to CircleCI (#121) * Prototype CircleCI and Docker config * Update Docker and setup * Fix typo in Dockerfile * Include mujoco_py support * Install right version of MuJoCo * CircleCI: fix cache saving * Fix linting * Replace local code checks script * Fix typechecking * envs: close environment after testing * Fix Dockerfile and setup.py * Bugfix: include AIRL example envs assets in install * CircleCI: codespell - skip notebooks * test_scripts: reduce Ray resource consumption * setup.py: fix pytype ignore comment * test_scripts: limit # of trials running in parallel * Remove Travis config * Update status badge in README * Address code review * README.md: Bump python version --- .circleci/config.yml | 119 +++++++++++++++++++++++ .dockerignore | 1 + .gitignore | 14 ++- .travis.yml | 47 --------- Dockerfile | 78 +++++++++++++++ README.md | 4 +- ci/build_venv.sh | 8 ++ ci/code_checks.sh | 26 +++++ ci/lint.sh | 34 ------- ci/type_check.sh | 6 -- setup.cfg | 8 ++ setup.py | 19 ++-- src/imitation/scripts/config/parallel.py | 1 + src/imitation/scripts/parallel.py | 5 +- src/imitation/testing/envs.py | 6 +- tests/test_scripts.py | 18 +++- 16 files changed, 290 insertions(+), 104 deletions(-) create mode 100644 .circleci/config.yml create mode 120000 .dockerignore delete mode 100644 .travis.yml create mode 100644 Dockerfile create mode 100755 ci/build_venv.sh create mode 100755 ci/code_checks.sh delete mode 100755 ci/lint.sh delete mode 100755 ci/type_check.sh diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..87a1cfcea --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,119 @@ +version: 2.1 + +orbs: + codecov: codecov/codecov@1.0.5 + +executors: + my-executor: + docker: + - image: humancompatibleai/imitation:base + working_directory: /imitation + environment: + # If you change these, also change ci/code_checks.sh + SRC_FILES: src/ tests/ experiments/ setup.py + +commands: + dependencies: + # You must still manually update the Docker image if any + # binary (non-Python) dependencies change. + description: "Check out and update Python dependencies." + steps: + - checkout + # Download and cache dependencies + - restore_cache: + keys: + - v1-dependencies-{{ checksum "setup.py" }} + + - run: + name: install dependencies + # MUJOCO_KEY is defined in a CircleCI context + # Do some sanity checks to make sure key works + command: | + curl -o /root/.mujoco/mjkey.txt ${MUJOCO_KEY} + md5sum /root/.mujoco/mjkey.txt + # Only create venv if it's not been restored from cache + [[ -d venv ]] || USE_MPI=True ./ci/build_venv.sh + python -c "import mujoco_py" + + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum "setup.py" }} + + - run: + name: install imitation + # Build a wheel then install to avoid copying whole directory (pip issue #2195) + command: | + python setup.py sdist bdist_wheel + pip install --upgrade dist/imitation-*.whl + +jobs: + lintandtype: + executor: my-executor + + steps: + - dependencies + - run: + name: flake8 + command: flake8 ${SRC_FILES} + + - run: + name: codespell + command: codespell -I .codespell.skip --skip='*.pyc,tests/data/*,*.ipynb,*.csv' ${SRC_FILES} + + - run: + name: sphinx + command: pushd docs/ && make clean && make html && popd + + - run: + name: pytype + command: pytype ${SRC_FILES} + + unit-test: + executor: my-executor + parallelism: 3 + steps: + - dependencies + + - run: + name: Memory Monitor + command: | + mkdir /tmp/resource-usage + export FILE=/tmp/resource-usage/memory.txt + while true; do + ps -u root eo pid,%cpu,%mem,args,uname --sort=-%mem >> $FILE + echo "----------" >> $FILE + sleep 1 + done + background: true + + - run: + name: run tests + command: | + export DISPLAY=:0 + pytest --cov=venv/lib/python3.7/site-packages/imitation --cov=tests \ + --junitxml=/tmp/test-reports/junit.xml \ + --shard-id=${CIRCLE_NODE_INDEX} --num-shards=${CIRCLE_NODE_TOTAL} \ + -vv tests/ + mv .coverage .coverage.imitation + coverage combine # rewrite paths from virtualenv to src/ + - codecov/upload + + - store_artifacts: + path: /tmp/test-reports + destination: test-reports + - store_test_results: + path: /tmp/test-reports + unit-test: + - store_artifacts: + path: /tmp/resource-usage + destination: resource-usage + +workflows: + version: 2 + test: + jobs: + - lintandtype: + context: MuJoCo + - unit-test: + context: MuJoCo diff --git a/.dockerignore b/.dockerignore new file mode 120000 index 000000000..3e4e48b0b --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/.gitignore b/.gitignore index 9aadc1f6d..81b3930c0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ # Project-specific -.*/ .#* *.html output/ @@ -55,6 +54,7 @@ htmlcov/ nosetests.xml coverage.xml *.cover +*.py,cover .hypothesis/ .pytest_cache/ @@ -98,8 +98,12 @@ ipython_config.py # install all needed dependencies. #Pipfile.lock -# celery beat schedule file +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff celerybeat-schedule +celerybeat.pid # SageMath parsed files *.sage.py @@ -120,6 +124,9 @@ venv.bak/ # Rope project settings .ropeproject +# PyCharm project settings +.idea + # mkdocs documentation /site @@ -131,6 +138,9 @@ dmypy.json # Pyre type checker .pyre/ +# Pytype type checker +.pytype/ + # Mac OSX .DS_STORE diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 530f37f70..000000000 --- a/.travis.yml +++ /dev/null @@ -1,47 +0,0 @@ -sudo: required -dist: xenial - -branches: - only: - - master - -language: python -cache: pip -before_install: - - sudo apt-get install -y libopenmpi-dev -install: - - pip install '.[test,cpu]' -script: - - pytest -vv tests/ - -matrix: - include: - - name: "Linting" - python: "3.7" - install: - - pip install '.[dev,cpu]' - script: - - ci/lint.sh - - - name: "3.7 Coverage Tests" - python: "3.7" - install: - # Install in developer mode so codecov can match paths - - pip install -e '.[test,cpu]' - script: - - py.test -vv --cov-report=xml --cov=. tests/ - after_success: - - codecov - - - name: "3.7 Static Type Checking" - python: "3.7" - install: - - pip install '.[test,cpu]' - - pip install pytype - script: - - ci/type_check.sh - - - name: "3.6 Unit Tests" - python: "3.6" - - name: "3.7 Unit Tests" - python: "3.7" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..fa22024d1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,78 @@ +# Based on OpenAI's mujoco-py Dockerfile + +# base stage contains just binary dependencies. +# This is used in the CI build. +FROM nvidia/cuda:10.0-runtime-ubuntu18.04 AS base +ARG DEBIAN_FRONTEND=noninteractive + +RUN echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | debconf-set-selections \ + && apt-get update -q \ + && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + ffmpeg \ + git \ + libgl1-mesa-dev \ + libgl1-mesa-glx \ + libglew-dev \ + libosmesa6-dev \ + net-tools \ + parallel \ + python3.7 \ + python3.7-dev \ + python3-pip \ + rsync \ + software-properties-common \ + unzip \ + vim \ + virtualenv \ + xpra \ + xserver-xorg-dev \ + ttf-mscorefonts-installer \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN curl -o /usr/local/bin/patchelf https://s3-us-west-2.amazonaws.com/openai-sci-artifacts/manual-builds/patchelf_0.9_amd64.elf \ + && chmod +x /usr/local/bin/patchelf + +ENV LANG C.UTF-8 + +RUN mkdir -p /root/.mujoco \ + && curl -o mjpro150.zip https://www.roboti.us/download/mjpro150_linux.zip \ + && unzip mjpro150.zip -d /root/.mujoco \ + && rm mjpro150.zip + +# Set the PATH to the venv before we create the venv, so it's visible in base. +# This is since we may create the venv outside of Docker, e.g. in CI +# or by binding it in for local development. +ENV PATH="/imitation/venv/bin:$PATH" +ENV LD_LIBRARY_PATH /usr/local/nvidia/lib64:/root/.mujoco/mjpro150/bin:${LD_LIBRARY_PATH} + +# python-req stage contains Python venv, but not code. +# It is useful for development purposes: you can mount +# code from outside the Docker container. +FROM base as python-req + +WORKDIR /imitation +# Copy over just setup.py and __init__.py (including version) +# to avoid rebuilding venv when requirements have not changed. +COPY ./setup.py ./setup.py +COPY ./src/imitation/__init__.py ./src/imitation/__init__.py +COPY ./ci/build_venv.sh ./ci/build_venv.sh +# mjkey.txt needs to exist for build, but doesn't need to be a real key +RUN touch /root/.mujoco/mjkey.txt \ + && ci/build_venv.sh \ + && rm -rf $HOME/.cache/pip + +# full stage contains everything. +# Can be used for deployment and local testing. +FROM python-req as full + +# Delay copying (and installing) the code until the very end +COPY . /imitation +# Build a wheel then install to avoid copying whole directory (pip issue #2195) +RUN python3 setup.py sdist bdist_wheel +RUN pip install --upgrade dist/imitation-*.whl + +# Default entrypoints +CMD ["pytest", "-n", "auto", "-vv", "tests/"] diff --git a/README.md b/README.md index 35e79275e..d8d90759b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.com/HumanCompatibleAI/imitation.svg?branch=master)](https://travis-ci.com/HumanCompatibleAI/imitation) +[![CircleCI](https://circleci.com/gh/HumanCompatibleAI/imitation.svg?style=svg)](https://circleci.com/gh/HumanCompatibleAI/imitation) [![codecov](https://codecov.io/gh/HumanCompatibleAI/imitation/branch/master/graph/badge.svg)](https://codecov.io/gh/HumanCompatibleAI/imitation) # Imitation Learning Baseline Implementations @@ -10,7 +10,7 @@ Currently we have implementations of [AIRL](https://arxiv.org/abs/1710.11248) an To install: ``` sudo apt install libopenmpi-dev -conda create -n imitation python=3.7 # python 3.6 and virtualenv are also okay. +conda create -n imitation python=3.8 # python 3.7 and virtualenv are also okay. conda activate imitation pip install -e '.[dev]' # install `imitation` in developer mode ``` diff --git a/ci/build_venv.sh b/ci/build_venv.sh new file mode 100755 index 000000000..87595c50c --- /dev/null +++ b/ci/build_venv.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -e # exit immediately on any error + +venv=venv +virtualenv -p python3.7 ${venv} +source ${venv}/bin/activate +pip install .[cpu,dev,test] gym[mujoco] diff --git a/ci/code_checks.sh b/ci/code_checks.sh new file mode 100755 index 000000000..66b84014c --- /dev/null +++ b/ci/code_checks.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# If you change these, also change .circle/config.yml. +SRC_FILES=(src/ tests/ experiments/ setup.py) + +set -x # echo commands +set -e # quit immediately on error + +echo "Source format checking" +flake8 ${SRC_FILES} +codespell -I .codespell.skip --skip='*.pyc,tests/data/*,*.ipynb,*.csv' ${SRC_FILES} + +if [ -x "`which circleci`" ]; then + circleci config validate +fi + +if [ "$skipexpensive" != "true" ]; then + echo "Building docs (validates docstrings)" + pushd docs/ + make clean + make html + popd + + echo "Type checking" + pytype ${SRC_FILES} +fi diff --git a/ci/lint.sh b/ci/lint.sh deleted file mode 100755 index 04605d3f2..000000000 --- a/ci/lint.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -SOURCE_DIRS=(src/ tests/ experiments/) - -RET=0 - -echo "PEP8 compliance" -echo "flake8 --version" -flake8 --version - -echo "flake8" -flake8 ${SOURCE_DIRS[@]} -RET=$(($RET + $?)) - -echo "Check for common typos" -echo "codespell --version" -codespell --version - -echo "codespell" -codespell -I .codespell.skip --skip='*.pyc,tests/data/*,*.ipynb,*.csv' ${SOURCE_DIRS[@]} -RET=$((RET + $?)) - -echo "Building docs (validates docstrings)" -pushd docs/ -make clean -RET=$((RET + $?)) -make html -RET=$((RET + $?)) -popd - -if [ $RET -ne 0 ]; then - echo "Linting failed." -fi -exit $RET diff --git a/ci/type_check.sh b/ci/type_check.sh deleted file mode 100755 index 64f55d246..000000000 --- a/ci/type_check.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -SOURCE_DIRS=(src/ tests/ experiments/) - -echo "pytype ${SOURCE_DIRS[@]}" -pytype "${SOURCE_DIRS[@]}" diff --git a/setup.cfg b/setup.cfg index 440f4ad2d..b25a5aaca 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,6 +35,9 @@ markers = [coverage:run] source = imitation +include= + src/* + tests/* [coverage:report] exclude_lines = @@ -44,3 +47,8 @@ exclude_lines = if __name__ == .__main__.: omit = setup.py + +[coverage:paths] +source = + src/imitation + *venv/lib/python*/site-packages/imitation diff --git a/setup.py b/setup.py index 9c4bafd9b..f0e5042f9 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ from setuptools import find_packages, setup -import src.imitation +import src.imitation # pytype:disable=import-error # TF 1.14.0 is not compatible with sacred because of a TF bug. TF_VERSION = '>=1.13.1,<2.0,!=1.14.0' @@ -8,6 +8,8 @@ 'codespell', 'pytest', 'pytest-cov', + 'pytest-shard', + 'pytest-xdist', ] setup( @@ -16,18 +18,23 @@ description=( 'Implementation of modern IRL and imitation learning algorithms.'), author='Center for Human-Compatible AI and Google', - python_requires='>=3.6.0', + python_requires='>=3.7.0', packages=find_packages('src'), package_dir={'': 'src'}, - package_data={'evaluating_rewards': ['py.typed']}, + package_data={ + 'imitation': [ + 'py.typed', + 'envs/examples/airl_envs/assets/*.xml', + ], + }, install_requires=[ 'awscli', - 'gym', + 'gym[classic_control]', 'numpy>=1.15', 'ray[debug]==0.7.4', 'tqdm', 'scikit-learn>=0.21.2', - # TODO(adam): Change to >=2.9.0 once 2.9.0 is released, until then track master + # TODO(adam): Change to >=2.9.0 once 2.9.0 released 'stable-baselines @ git+https://github.com/hill-a/stable-baselines.git', 'jax!=0.1.37', 'jaxlib~=0.1.20', @@ -75,8 +82,8 @@ 'License :: OSI Approved :: MIT License', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', ], diff --git a/src/imitation/scripts/config/parallel.py b/src/imitation/scripts/config/parallel.py index bf6c30af2..d4e99b1f9 100644 --- a/src/imitation/scripts/config/parallel.py +++ b/src/imitation/scripts/config/parallel.py @@ -20,6 +20,7 @@ @parallel_ex.config def config(): sacred_ex_name = "expert_demos" # The experiment to parallelize + init_kwargs = {} # Keyword arguments to pass to ray.init() _uuid = make_unique_timestamp() run_name = ( f"DEFAULT_{_uuid}") # CLI --name option. For analysis grouping. diff --git a/src/imitation/scripts/parallel.py b/src/imitation/scripts/parallel.py index 09232a960..6c3c9204e 100644 --- a/src/imitation/scripts/parallel.py +++ b/src/imitation/scripts/parallel.py @@ -1,5 +1,5 @@ import os.path as osp -from typing import Callable, Optional +from typing import Any, Callable, Dict, Optional import ray import ray.tune @@ -15,6 +15,7 @@ def parallel(sacred_ex_name: str, base_named_configs: list, base_config_updates: dict, resources_per_trial: dict, + init_kwargs: Dict[str, Any], local_dir: Optional[str], upload_dir: Optional[str], ) -> None: @@ -57,7 +58,7 @@ def parallel(sacred_ex_name: str, # dashboard. ray_loggers = () - ray.init() + ray.init(**init_kwargs) try: ray.tune.run(trainable, config=search_space, name=run_name, diff --git a/src/imitation/testing/envs.py b/src/imitation/testing/envs.py index 444a7df18..c1e3440a5 100644 --- a/src/imitation/testing/envs.py +++ b/src/imitation/testing/envs.py @@ -12,14 +12,18 @@ def make_env_fixture(skip_fn): def f(env_name): + env = None try: env = gym.make(env_name) + yield env except gym.error.DependencyNotInstalled as e: # pragma: no cover if e.args[0].find('mujoco_py') != -1: skip_fn("Requires `mujoco_py`, which isn't installed.") else: raise - return env + finally: + if env is not None: + env.close() return f diff --git a/tests/test_scripts.py b/tests/test_scripts.py index 0270c577d..f718959f3 100644 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -174,9 +174,20 @@ def test_transfer_learning(tmpdir): ] +PARALLEL_CONFIG_LOW_RESOURCE = { + # CI server only has 2 cores. + "init_kwargs": {"num_cpus": 2}, + # Memory is low enough we only want to run one job at a time. + "resources_per_trial": {"cpu": 2}, +} + + @pytest.mark.parametrize("config_updates", PARALLEL_CONFIG_UPDATES) def test_parallel(config_updates): """Hyperparam tuning smoke test.""" + # CI server only has 2 cores + config_updates = dict(config_updates) + config_updates.update(PARALLEL_CONFIG_LOW_RESOURCE) # No need for TemporaryDirectory because the hyperparameter tuning script # itself generates no artifacts, and "debug_log_root" sets inner experiment's # log_root="/tmp/parallel_debug/". @@ -204,11 +215,10 @@ def test_analyze_imitation(tmpdir: str, def test_analyze_gather_tb(tmpdir: str): + config_updates = dict(local_dir=tmpdir, run_name="test") + config_updates.update(PARALLEL_CONFIG_LOW_RESOURCE) parallel_run = parallel_ex.run(named_configs=["generate_test_data"], - config_updates=dict( - local_dir=tmpdir, - run_name="test", - )) + config_updates=config_updates) assert parallel_run.status == 'COMPLETED' run = analysis_ex.run(