diff --git a/.coveragerc b/.coveragerc index 4987b253e..b2920e991 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,7 +1,12 @@ # .coveragerc to control coverage.py [run] +branch = True omit = - src/scenic/simulators/webots/WBT*.py + src/scenic/simulators/carla/* + src/scenic/simulators/gta/* + src/scenic/simulators/lgsvl/* + src/scenic/simulators/webots/* + src/scenic/simulators/xplane/* [report] # Regexes for lines to exclude from consideration @@ -20,3 +25,5 @@ exclude_lines = @(abc\.)?abstractmethod ignore_errors = True +show_missing = True +precision = 2 \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/1-bug.yml b/.github/ISSUE_TEMPLATE/1-bug.yml new file mode 100644 index 000000000..d84707fdb --- /dev/null +++ b/.github/ISSUE_TEMPLATE/1-bug.yml @@ -0,0 +1,69 @@ +name: Bug Report +description: Create a report to help us reproduce and fix a bug +labels: + - "type: bug" + - "status: triage" + +body: + - type: markdown + attributes: + value: > + #### Thank you for contributing! Before reporting a bug, please make sure it has no duplicate and has not already been addressed by searching through [the existing and past issues](https://github.com/BerkeleyLearnVerify/Scenic/issues) + + - type: textarea + attributes: + label: System Details + description: | + Please provide the following system information to help us diagnose the bug. + 1. Python Version (e.g. Python 3.11.7) + 2. Scenic Version (e.g. Scenic 3.0.0b2) + 3. Operating System / Platform (e.g. Apple M2 Pro macOS Sonoma 14.2.1) + 4. Simulator Version (Optional, only applies if bug is simulator specific) (e.g. CARLA 0.9.14) + + placeholder: | + 1. Python Version: `python --version` + 2. Scenic Version: `scenic --version` + 3. Operating System / Platform: get from system preferences + 4. Simulator Version: simulator version + validations: + required: true + + - type: textarea + attributes: + label: Detailed Description + description: | + Please provide a clear and concise description of what the bug is and paste the error log below. Please rerun your simulation error with the **`-b`** parameter for a full stack trace. + It helps improving readability if the error log is wrapped in ```triple quotes blocks```. + placeholder: | + A clear and concise description of what the bug is. You can rerun your Scenic simulation with `-b` to get a full stack trace. + + ``` + # error full stack trace + ``` + validations: + required: true + + - type: textarea + attributes: + label: Steps To Reproduce + description: | + Please provide a minimal example to help us reproduce the bug. Code should be wrapped with ```triple quotes blocks``` to improve readability. + If it is not possible to reproduce the bug with a short self-contained Scenic or Python file, for example if a specific mesh is required, you can attach any required files below or include a link to them. + placeholder: | + 1. First step to reproduce bug + 2. Second step, etc + ... + ``` + # sample code to reproduce the bug + ``` + validations: + required: true + + - type: checkboxes + attributes: + label: Issue Submission Checklist + options: + - label: I am reporting an issue, not asking a question + required: true + - label: I checked the open issues, forum, etc. and have not found any solution + - label: I have provided all necessary code, etc. to reproduce the issue diff --git a/.github/ISSUE_TEMPLATE/2-docs.yml b/.github/ISSUE_TEMPLATE/2-docs.yml new file mode 100644 index 000000000..6c7be09f1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/2-docs.yml @@ -0,0 +1,35 @@ +name: Documentation +description: Report an issue or enhancement related to https://scenic-lang.readthedocs.io/ +labels: + - "type: documentation" + - "status: triage" + +body: + - type: markdown + attributes: + value: > + #### Thank you for contributing! Before submitting a doc issue, please make sure it has no duplicate and has not already been addressed by searching through [the existing and past issues](https://github.com/BerkeleyLearnVerify/Scenic/issues) + + - type: textarea + attributes: + label: Describe the doc issue or enhancement + description: > + Please provide a clear and concise description of what content in https://scenic-lang.readthedocs.io/ has an issue or needs enhancement. + placeholder: | + Link to location in the docs: https://scenic-lang.readthedocs.io/ + validations: + required: true + + - type: textarea + attributes: + label: Fix suggestion + description: > + Tell us how we could improve the documentation in this regard. + + - type: checkboxes + attributes: + label: Issue Submission Checklist + options: + - label: I am reporting an issue, not asking a question + required: true + - label: I checked the open issues, forum, etc. and have not found any solution diff --git a/.github/ISSUE_TEMPLATE/3-feature.yml b/.github/ISSUE_TEMPLATE/3-feature.yml new file mode 100644 index 000000000..301315521 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/3-feature.yml @@ -0,0 +1,31 @@ +name: Feature Request +description: Submit a request for new Scenic features +labels: + - "type: feature" + - "status: triage" + +body: + - type: markdown + attributes: + value: > + #### Thank you for contributing! Before submitting a feature request, please make sure it has no duplicate and has not already been addressed by searching through [the existing and past issues](https://github.com/BerkeleyLearnVerify/Scenic/issues) + + - type: textarea + attributes: + label: Describe the feature and motivation + description: | + Please provide a clear and concise proposal of the feature and outline the motivation. + validations: + required: true + + - type: textarea + attributes: + label: Additional context + description: | + Add any other context, such as pseudo code, links, diagrams, screenshots, to help the community better understand the feature request. + + - type: checkboxes + attributes: + label: Issue Submission Checklist + options: + - label: I checked the open issues, forum, etc. and have not found any solution \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..427eb214e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: true +contact_links: + - name: Questions + url: https://forms.gle/uUhQNuPzQrvvBFJX9 + about: Send your questions via Google Form \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..7dc981b71 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,15 @@ +### Description + + +### Issue Link + + +### Checklist +- [ ] I have tested the changes locally via `pytest` and/or other means +- [ ] I have added or updated relevant documentation +- [ ] I have autoformatted the code with black and isort +- [ ] I have added test cases (if applicable) + +### Additional Notes + + \ No newline at end of file diff --git a/.github/workflows/run-coverage.yml b/.github/workflows/run-coverage.yml new file mode 100644 index 000000000..7ec93f84f --- /dev/null +++ b/.github/workflows/run-coverage.yml @@ -0,0 +1,65 @@ +name: run_coverage + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + workflow_dispatch: + inputs: + ref: + description: Git ref on which to run the tests. + type: string + required: true + workflow_call: + inputs: + ref: + description: Git ref on which to run the tests. + type: string + +jobs: + coverage: + strategy: + fail-fast: true + matrix: + python-version: ["3.11"] + os: [ubuntu-latest] + extras: ["test-full"] + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout given ref + uses: actions/checkout@v3 + if: inputs.ref != '' + with: + ref: ${{ inputs.ref }} + + - name: Checkout current branch + uses: actions/checkout@v3 + if: inputs.ref == '' + with: + ref: ${{ github.ref }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: Update pip + run: | + python -m pip install --upgrade pip + + - name: Install Scenic and dependencies + run: | + python -m pip install -e ".[${{ matrix.extras }}]" + + - name: Run and report code coverage + run: | + pytest --cov --cov-report json + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4.0.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: BerkeleyLearnVerify/Scenic \ No newline at end of file diff --git a/.gitignore b/.gitignore index 702d62a1f..31bd1e964 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,9 @@ poetry.lock *.sublime-project *.sublime-workspace +# VSCode files +*.vscode + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -66,6 +69,7 @@ coverage.xml *.cover .hypothesis/ .pytest_cache/ +coverage.json # Translations *.mo diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..5cecf4809 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,21 @@ +codecov: + coverage: + precision: 2 + round: down + range: "70...100" + status: + project: + default: + target: 80% + threshold: 1% + changes: + target: 80% + threshold: 1% + comment: + layout: "reach, diff, flags, files" + behavior: default + require_ci_to_pass: true +cli: + plugins: + pycoverage: + report_type: "json" \ No newline at end of file diff --git a/docs/developing.rst b/docs/developing.rst index 044f88ced..97bdfe8e0 100644 --- a/docs/developing.rst +++ b/docs/developing.rst @@ -58,6 +58,21 @@ If you're running the test suite on a headless server or just want to stop windo popping up during testing, use the :command:`--no-graphics` option to skip graphical tests. +Prior to finalizing a PR or other substantial changes, it's a good idea to run the test suite under all major versions of Python that Scenic supports, in fresh virtual environments. +You can do this automatically with the command :command:`tox`, which by default will test all supported major versions both with and without optional dependencies (this will take a long time). +Some variations: + +* :command:`tox -p` will run the various combinations in parallel. + +* :command:`tox -m basic` skips testing installations with the optional dependencies. + +* :command:`tox -- --fast` only runs the "fast" tests. In general, any arguments after the :command:`--` will get passed to ``pytest``. For example, + +* :command:`tox -- tests/syntax/test_specifiers.py` only runs the tests in the given file. + +See the `Tox `_ website for more information about the available options and how to configure Tox. + + .. _debugging: Debugging diff --git a/pyproject.toml b/pyproject.toml index 796b96bd9..e69b47d05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ guideways = [ ] test = [ # minimum dependencies for running tests (used for tox virtualenvs) "pytest >= 7.0.0, <8", + "pytest-cov >= 3.0.0", "pytest-randomly ~= 3.2", ] test-full = [ # like 'test' but adds dependencies for optional features @@ -79,7 +80,7 @@ dev = [ "isort ~= 5.11", "pre-commit ~= 3.0", "pytest-cov >= 3.0.0", - "tox ~= 3.14", + "tox ~= 4.0", ] [project.urls] @@ -121,3 +122,6 @@ extend_skip_glob = [ [tool.pytest.ini_options] norecursedirs = ["examples"] + +[tool.coverage.run] +source = ["src"] \ No newline at end of file diff --git a/src/scenic/core/dynamics/behaviors.py b/src/scenic/core/dynamics/behaviors.py index afb33ca93..7c12ef7c2 100644 --- a/src/scenic/core/dynamics/behaviors.py +++ b/src/scenic/core/dynamics/behaviors.py @@ -95,7 +95,8 @@ def _step(self): assert self._runningIterator def alarmHandler(signum, frame): - if sys.gettrace(): + # NOTE: if using pytest-cov, sys.gettrace() set to CTracer(), but we still want timeout warnings enabled + if sys.gettrace() and "coverage" not in str(type(sys.gettrace())): return # skip the warning if we're in the debugger warnings.warn( f"the behavior {self} is taking a long time to take an action; " diff --git a/tox.ini b/tox.ini index 365ce524a..e08482b81 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,7 @@ [tox] -isolated_build = true envlist = py{38,39,310,311,312}{,-extras} +labels = + basic = py{38,39,310,311,312} [testenv] extras =