Add basic cli tests using Bats

.github/workflows/tests.yml

@@ -26,17 +26,20 @@ jobs:
           - ubuntu-latest
           - macos-latest
           - windows-latest
+    env:
+      SKIP_ENVS: ${{ (matrix.py != '3.12' || matrix.os == 'windows-latest') && '--skip-env=cli' || '' }}
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          submodules: ${{ matrix.py == '3.12' && matrix.os != 'windows-latest' && 'true' || 'false' }}
       - name: Setup python for test ${{ matrix.py }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.py }}
-      - name: Install tox
+      - name: Install tox-gh
         run: python -m pip install tox-gh>=1.2
       - name: Setup test suite
         run: tox -vv --notest
       - name: Run test suite
-        run: tox --skip-pkg-install
+        run: tox --skip-pkg-install ${{ env.SKIP_ENVS }}

.gitmodules

@@ -0,0 +1,9 @@
+[submodule "tests/cli/bats"]
+	path = tests/cli/bats
+	url = https://github.com/bats-core/bats-core.git
+[submodule "tests/cli/test_helper/bats-support"]
+	path = tests/cli/test_helper/bats-support
+	url = https://github.com/bats-core/bats-support.git
+[submodule "tests/cli/test_helper/bats-assert"]
+	path = tests/cli/test_helper/bats-assert
+	url = https://github.com/bats-core/bats-assert.git

CONTRIBUTING.md

@@ -40,7 +40,8 @@ Test coverage for new functionality is helpful, but not always required. Feel fr
 Make sure you've installed [tox](https://tox.wiki/) and a supported python version, then run
 
 ```shell
+git submodule update --init
 tox
 ```
 
-You can follow up in the issue tracker to ask questions or report issues.
+See [tests/README.md](tests/README.md) for more info on the tests.

pyproject.toml

@@ -58,6 +58,7 @@ universal = true
 
 [tool.ruff]
 line-length = 80
+extend-exclude = ["tests/cli/"]
 
 [tool.ruff.lint]
 # Enable Errors, Warnings, Flakes

tests/README.md

@@ -0,0 +1,53 @@
+# Tests for gcalcli
+
+This directory contains unit tests and functional tests for gcalcli. To run them all, make sure
+you've installed [tox](https://tox.wiki/) and a supported python version, then in the repository root dir run
+
+```shell
+git submodule update --init
+tox
+```
+
+Or run individual configurations like
+
+```shell
+tox -e py38,cli
+```
+
+The supported tox testing envs are listed and configured in ../tox.ini.
+
+They're also configured to run on GitHub pull requests for various platforms and python versions
+(config: ../.github/workflows/tests.yml).
+
+## Linters and type checking
+
+Tox will also run [Ruff](https://docs.astral.sh/ruff/) linters and [mypy](https://mypy-lang.org/)
+type checking on the code (configured in the root dir, not under tests/).
+
+If a weird Ruff check is giving you grief, you might need to
+[ignore it](https://docs.astral.sh/ruff/settings/#lint_extend-per-file-ignores) in the pyproject.toml.
+
+The type checking can also give you grief, particularly for library types it can't resolve. See
+https://mypy.readthedocs.io/en/stable/common_issues.html for troubleshooting info. Some issues can
+be resolved by using their stubgen tool to generate more type stubs under stubs/, or tweaking the
+existing .pyi files there to better reflect reality.
+
+## Unit tests
+
+The python test files under tests/ are unit tests run via [pytest](https://pytest.org/). If you
+hit failures, you can start a debugger with the --pdb flag to troubleshoot (probably also
+specifying an individual test env and test to debug). Example:
+
+```shell
+tox -e py312 -- tests/test_gcalcli.py::test_import --pdb
+```
+
+## Functional cli tests
+
+Under tests/cli/ there are also high-level tests running the cli in a shell using the
+[Bats](https://bats-core.readthedocs.io/) tool. They have a .bats extension. These can be run
+individually with `tox -e cli`.
+
+NOTE: They'll fail if you haven't initialized the repo submodules for Bats yet, so if you hit
+errors for missing test runner files, make sure you've run `git submodule update --init` in the
+repo.

tests/cli/test.bats

@@ -0,0 +1,14 @@
+setup() {
+  load 'test_helper/bats-support/load'
+  load 'test_helper/bats-assert/load'
+}
+
+@test "can run" {
+  run gcalcli
+  assert_output --regexp 'usage: .*error:.*required: .*command'
+}
+
+@test "prints correct help" {
+  run gcalcli -h
+  assert_output --regexp 'positional arguments:.*list.*search.*edit.*options:'
+}

tox.ini

@@ -1,7 +1,7 @@
 [tox]
 requires =
     tox>=4
-envlist = lint, type, py3{8,9,10,11,12}
+envlist = lint, type, py3{8,9,10,11,12}, cli
 
 [testenv]
 usedevelop=true
@@ -31,9 +31,16 @@ deps =
 commands =
     mypy {posargs:gcalcli}
 
+[testenv:cli]
+description = run functional tests using Bats
+allowlist_externals =
+    ./tests/cli/bats/bin/bats
+commands =
+    ./tests/cli/bats/bin/bats tests/cli/test.bats
+
 [gh]
 python =
-    3.12 = py312, lint, type
+    3.12 = py312, lint, type, cli
     3.11 = py311
     3.10 = py310
     3.9 = py39