Spin-up CKAN for testing in Github Actions (#192)

ropensci · Jan 27, 2023 · a5cca7e · a5cca7e
1 parent 7f6e2dc
commit a5cca7e
Show file tree

Hide file tree

Showing 12 changed files with 59 additions and 42 deletions.
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
@@ -13,15 +13,15 @@
 * Test your changes locally. You need Docker Compose for this (installation instructions are available at <https://docs.docker.com/compose/install/>). You can spin up a local CKAN site with
 
     ```bash
-    docker compose up
+    CKAN_VERSION=2.8 docker compose up
     ```
 
     Enter http://localhost:5000/ in a browser to see CKAN running.
 
     There is a sysadmin user created by default with `username=ckan_admin` and `password=test1234`. You can retrieve the user details including the API KEY (for the R environment variable `TEST_API_KEY`) with
 
     ```bash
-    docker exec ckanr-ckan-1 paster --plugin=ckan user ckan_admin
+    docker exec ckan paster --plugin=ckan user ckan_admin
     ```
 * Push up to your account
 * Submit a pull request to home base (likely master branch, but check to make sure) at `ropensci/ckanr`

diff --git a/.github/workflows/R-check.yaml b/.github/workflows/R-check.yaml
@@ -6,36 +6,35 @@ jobs:
   R-check:
     runs-on: ${{ matrix.config.os }}
 
-    name: ${{ matrix.config.os }} (${{ matrix.config.r }})
+    name: ${{ matrix.config.os }} (${{ matrix.config.r }}) - CKAN ${{ matrix.config.ckan-version }}
 
     strategy:
       fail-fast: false
       matrix:
         config:
         - { os: windows-latest, r: 'release'}
-        - { os: windows-latest, r: 'devel'}
         - { os: macOS-latest, r: 'release'}
-        - { os: ubuntu-latest,  r: 'release'}
+        - { os: ubuntu-latest,  r: 'release', ckan-version: 2.7}
+        - { os: ubuntu-latest,  r: 'release', ckan-version: 2.8}
+        - { os: ubuntu-latest,  r: 'release', ckan-version: 2.9}
+        - { os: ubuntu-latest, r: 'devel', ckan-version: 2.9}
 
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
-      R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
-      CRAN: ${{ matrix.config.rspm }}
-      CKANR_DEFAULT_KEY: ${{secrets.CKANR_DEFAULT_KEY}}
-      CKANR_DEFAULT_URL: ${{secrets.CKANR_DEFAULT_URL}}
-      CKANR_TEST_BEHAVIOUR: ${{secrets.CKANR_TEST_BEHAVIOUR}}
-      CKANR_TEST_DID: ${{secrets.CKANR_TEST_DID}}
-      CKANR_TEST_GID: ${{secrets.CKANR_TEST_GID}}
-      CKANR_TEST_KEY: ${{secrets.CKANR_TEST_KEY}}
-      CKANR_TEST_OID: ${{secrets.CKANR_TEST_OID}}
-      CKANR_TEST_RID: ${{secrets.CKANR_TEST_RID}}
       CKANR_TEST_URL: ${{secrets.CKANR_TEST_URL}}
-      TEST_API_KEY: ${{secrets.TEST_API_KEY}}
+      CKANR_TEST_BEHAVIOUR: ${{secrets.CKANR_TEST_BEHAVIOUR}}
+      R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
       R_KEEP_PKG_SOURCE: yes
+      CKAN_VERSION: ${{ matrix.config.ckan-version }} # for docker compose up
 
     steps:
       - uses: actions/checkout@v3
-
+      - name: Configure /etc/hosts
+        run: sudo echo "127.0.0.1 ckan" | sudo tee -a /etc/hosts
+        if: matrix.config.os == 'ubuntu-latest'
+      - name: Start CKAN
+        run: docker compose up -d
+        if: matrix.config.os == 'ubuntu-latest'
       - uses: r-lib/actions/setup-r@v2
         with:
           r-version: ${{ matrix.config.r }}
@@ -51,15 +50,22 @@ jobs:
           needs: |
             check
             coverage
-
+      - name: Generate CKAN API token
+        run: |
+          echo "TEST_API_KEY=$(docker exec ckan ckan user token add ckan_admin dev_token | sed 's/API Token created://' | tr -d '\n\t')" >> $GITHUB_ENV
+        if: matrix.config.ckan-version == '2.9' && matrix.config.os == 'ubuntu-latest'
+      - name: Generate CKAN API key
+        run: |
+          echo "TEST_API_KEY"=$(docker exec ckan paster --plugin=ckan user ckan_admin | grep -o -P '(?<=apikey=).*(?= created)') >> $GITHUB_ENV
+        if: matrix.config.ckan-version != '2.9' && matrix.config.os == 'ubuntu-latest'
       - name: Check
         env:
           _R_CHECK_CRAN_INCOMING_: false
         run: rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran"), error_on = "error", check_dir = "check")
         shell: Rscript {0}
 
       - name: Test coverage
-        if: matrix.config.os == 'macOS-latest' && matrix.config.r == 'release'
+        if: matrix.config.os == 'ubuntu-latest' && matrix.config.r == 'release' &&  matrix.config.ckan-version == '2.8'
         run: |
           covr::codecov(
             quiet = FALSE,
@@ -76,7 +82,7 @@ jobs:
         shell: bash
 
       - name: Upload test results
-        if: failure()
+        if: failure() && matrix.config.os == 'ubuntu-latest'
         uses: actions/upload-artifact@v3
         with:
           name: coverage-test-failures

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,7 +1,8 @@
 version: "3.9"
 services:
   ckan:
-    image: "openknowledge/ckan-base:2.8"
+    image: "openknowledge/ckan-base:${CKAN_VERSION}"
+    container_name: ckan
     environment:
       - CKAN_SITE_URL=http://ckan:5000
       - CKAN_SQLALCHEMY_URL=postgresql://ckan_default:pass@postgres/ckan_test
@@ -18,20 +19,24 @@ services:
     ports:
       - "5000:5000"
   postgres:
-    image: "ckan/ckan-postgres-dev:2.8"
+    image: "ckan/ckan-postgres-dev:${CKAN_VERSION}"
+    container_name: postgres
     environment:
       - POSTGRES_DB=postgres
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=postgres
   datapusher:
     image: ghcr.io/keitaroinc/datapusher:0.0.17
+    container_name: datapusher
     environment:
       - DATAPUSHER_MAX_CONTENT_LENGTH=10485760
       - DATAPUSHER_CHUNK_SIZE=16384
       - DATAPUSHER_CHUNK_INSERT_ROWS=250
       - DATAPUSHER_DOWNLOAD_TIMEOUT=30
       - DATAPUSHER_SSL_VERIFY=False
   solr: 
-    image: "ckan/ckan-solr:2.8"
+    image: "ckan/ckan-solr:${CKAN_VERSION}"
+    container_name: solr
   redis:
     image: "redis:5.0.14"
+    container_name: redis
diff --git a/tests/testthat/helper-ckanr.R b/tests/testthat/helper-ckanr.R
@@ -153,4 +153,10 @@ check_organization <- function(url, x){
   }
 }
 
-prepare_test_ckan()
+u <- get_test_url()
+
+if (ping(u)) {
+  prepare_test_ckan()
+} else {
+  message("CKAN is offline. Running tests that don't depend on CKAN.")
+}
diff --git a/tests/testthat/test-changes.R b/tests/testthat/test-changes.R
@@ -3,9 +3,9 @@ context("changes")
 skip_on_cran()
 
 u <- get_test_url()
+check_ckan(u)
 
 test_that("changes gives back expected class types", {
-  check_ckan(u)
   cat(u, sep = "\n")
   a <- changes(url = u)
 
@@ -16,7 +16,6 @@ test_that("changes gives back expected class types", {
 })
 
 test_that("changes works giving back json output", {
-  check_ckan(u)
   b <- changes(url = u, as = 'json')
   b_df <- jsonlite::fromJSON(b)
 
@@ -26,7 +25,6 @@ test_that("changes works giving back json output", {
 })
 
 test_that("changes fails correctly", {
-  check_ckan(u)
 
   expect_error(changes("adf"), "offset Invalid integer")
   expect_error(changes(limit = "Adf"), "limit Invalid integer")

diff --git a/tests/testthat/test-ckan_fetch.R b/tests/testthat/test-ckan_fetch.R
@@ -11,6 +11,9 @@ test_that("ckan_fetch returns error when file format can't be determined from UR
   )
 })
 
+u <- get_test_url()
+check_ckan(u)
+
 test_that("ckan_fetch doesn't write any files to working directory when session = TRUE", {
   expect_identical(list.files(test_path()), {
     res <- resource_show(id = rid, as = "table")

diff --git a/tests/testthat/test-group_show.R b/tests/testthat/test-group_show.R
@@ -3,14 +3,13 @@ context("group_show")
 skip_on_cran()
 
 u <- get_test_url()
+check_ckan(u)
 g <- get_test_gid()
 if (g == "") {
   g <- group_list(url = u, limit = 1)[[1]]$name
 }
 
 test_that("group_show gives back expected class types", {
-  check_ckan(u)
-  # check_group(u,g)
   if (!ok_group(u, g))
     group_create("ckanr_test_group", url = u, key = get_test_key())
   a <- group_show(g, url=u)
@@ -20,8 +19,6 @@ test_that("group_show gives back expected class types", {
 })
 
 test_that("group_show works giving back json output", {
-  check_ckan(u)
-  # check_group(u,g)
   if (!ok_group(u, g))
     group_create("ckanr_test_group", url = u, key = get_test_key())
   b <- group_show(g, url=u, as='json')
@@ -33,7 +30,6 @@ test_that("group_show works giving back json output", {
 })
 
 test_that("group_show fails correctly", {
-  check_ckan(u)
   expect_error(group_show("adf", url=u), "404 - Not Found Error")
   expect_error(group_show(limit = "Adf", url=u), "argument \"id\" is missing")
 })
diff --git a/tests/testthat/test-license_list.R b/tests/testthat/test-license_list.R
@@ -1,12 +1,11 @@
 context("license_list")
 
 skip_on_cran()
+check_ckan(u)
 
 u = get_test_url()
 
 test_that("license_list gives back expected type", {
-  check_ckan(u)
-
   expect_equal(class(license_list(url=u)), "list")
-  expect_equal(class(license_list(as = "table")), "data.frame")
+  expect_equal(class(license_list(url=u, as = "table")), "data.frame")
 })
diff --git a/tests/testthat/test-organization_list.R b/tests/testthat/test-organization_list.R
@@ -3,9 +3,9 @@ context("organization_list")
 skip_on_cran()
 
 u <- get_test_url()
+check_ckan(u)
 
 test_that("organization_list gives back expected class types", {
-  check_ckan(u)
   a <- organization_list(url=u, limit=10)
 
   expect_is(a, "list")

diff --git a/tests/testthat/test-package_activity_list.R b/tests/testthat/test-package_activity_list.R
@@ -4,10 +4,11 @@ skip_on_cran()
 skip_on_ci()
 
 u <- get_test_url()
+check_ckan(u)
+
 id <- package_list(limit = 1, url=u)[[1]]
 
 package_activity_num <- local({
-  check_ckan(u)
   res <- crul::HttpClient$new(file.path(u, "dataset/activity", id))$get()
   res$raise_for_status()
   txt <- res$parse("UTF-8")

diff --git a/tests/testthat/test-ping.R b/tests/testthat/test-ping.R
@@ -2,10 +2,14 @@ context("ping")
 
 skip_on_cran()
 
-test_that("ping return true", {
-  expect_true(ping())
-})
+u = get_test_url()
 
 test_that("ping returns false when run against non-CKAN URLS", {
   expect_false(ping(url = "http://www.google.com"))
 })
+
+check_ckan(u)
+
+test_that("ping return true", {
+  expect_true(ping(url = u))
+})
diff --git a/tests/testthat/test-tag_show.R b/tests/testthat/test-tag_show.R
@@ -4,6 +4,7 @@ skip_on_cran()
 skip_on_ci()
 
 u <- get_test_url()
+check_ckan(u)
 
 tag_test_num <- local({
   t <- "api"
@@ -16,7 +17,6 @@ tag_test_num <- local({
 })
 
 test_that("tag_show gives back expected class types", {
-  check_ckan(u)
   t <- tag_list(url=u)[[1]]
   a <- tag_show(t$name, include_datasets = TRUE, url=u)
 
@@ -26,7 +26,6 @@ test_that("tag_show gives back expected class types", {
 })
 
 test_that("tag_show works giving back json output", {
-  check_ckan(u)
   t <- tag_list(url=u)[[1]]
   b <- tag_show(t$name, include_datasets = TRUE, url=u, as = 'json')
   b_df <- jsonlite::fromJSON(b)