Merge branch 'ropensci:main' into main

CONTRIBUTING.md

@@ -51,19 +51,18 @@ If you have found a bug, follow the issue template to create a minimal
 Some changes have intricate internal and external dependencies, which are easy
 to miss and break. These checklists aim to avoid these pitfalls.
 
-Test and update reverse dependencies (wastdr, urODK, etlTurtleNesting, etc.).
+Test and update reverse dependencies (wastdr, etlTurtleNesting, etc.).
 
 #### Adding a dependency
 * Update DESCRIPTION
 * Update GH Actions install workflows - do R package deps have system deps? Can GHA install them in all environments?
 * Update Dockerfile
-* Update urODK binder install.R
+* Update binder install.R
 * Update installation instructions
 
 #### Renaming a vignette
 * Search-replace all links to the vignette throughout 
   * ruODK, 
-  * urODK, 
   * ODK Central "OData" modal
   * ODK Central docs
 

DESCRIPTION

@@ -1,7 +1,7 @@
 Type: Package
 Package: ruODK
 Title: An R Client for the ODK Central API
-Version: 1.3.10
+Version: 1.3.11
 Authors@R: 
     c(person(given = c("Florian", "W."),
              family = "Mayer",
@@ -48,7 +48,6 @@ Imports:
     dplyr (>= 1.0.7),
     fs (>= 1.5.0),
     glue (>= 1.4.2),
-    here (>= 1.0.1),
     httr (>= 1.4.2),
     janitor (>= 2.1.0),
     lifecycle (>= 1.0.0),
@@ -65,6 +64,7 @@ Suggests:
     covr (>= 3.5.1),
     DT (>= 0.18),
     ggplot2 (>= 3.3.5),
+    here (>= 1.0.1),
     knitr (>= 1.33),
     lattice (>= 0.20-44),
     leaflet (>= 2.0.4.1),
@@ -86,7 +86,7 @@ RdMacros:
 Encoding: UTF-8
 Language: en-AU
 LazyData: true
-RoxygenNote: 7.1.2
+RoxygenNote: 7.2.0
 X-schema.org-applicationCategory: Data Access
 X-schema.org-keywords: database, open-data, opendatakit, odk, api, data, dataset
 Remotes: 

NAMESPACE

@@ -30,6 +30,7 @@ export(get_default_url)
 export(get_one_attachment)
 export(get_one_submission)
 export(get_one_submission_attachment_list)
+export(get_one_submission_audit)
 export(get_retries)
 export(get_ru_verbose)
 export(get_test_fid)
@@ -69,6 +70,7 @@ export(ru_setup)
 export(split_geopoint)
 export(split_geoshape)
 export(split_geotrace)
+export(submission_audit_get)
 export(submission_detail)
 export(submission_export)
 export(submission_get)

NEWS.md

@@ -1,3 +1,8 @@
+# ruODK 1.3.11
+* Update `project_list`, `submission_list` and `` columns to reflect Central 1.5 API output.
+* Update `submission_list` columns to reflect Central 1.5 API output.
+
+
 # ruODK 1.3.10
 
 # ruODK 1.3.9

R/form_list.R

@@ -6,9 +6,15 @@
 #' @template param-url
 #' @template param-auth
 #' @template param-retries
-#' @return A tibble with one row per form and all form metadata as columns.
+#' @template param-orders
+#' @template param-tz
+#' @return A tibble with one row per form and all given form metadata as columns.
+#'   Column names are sanitized into `snake_case`.
+#'   Nested columns (review start and created by) are flattened and prefixed.
+#'   The column `xml_form_id` is replicated as `fid` according to `ruODK` naming
+#'   standards.
 # nolint start
-#' @seealso \url{https://odkcentral.docs.apiary.io/#reference/forms-and-submissions/forms}
+#' @seealso \url{https://odkcentral.docs.apiary.io/#reference/forms/forms/list-all-forms}
 # nolint end
 #' @family form-management
 #' @importFrom httr add_headers authenticate content GET
@@ -38,7 +44,16 @@ form_list <- function(pid = get_default_pid(),
                       url = get_default_url(),
                       un = get_default_un(),
                       pw = get_default_pw(),
-                      retries = get_retries()) {
+                      retries = get_retries(),
+                      orders = c(
+                        "YmdHMS",
+                        "YmdHMSz",
+                        "Ymd HMS",
+                        "Ymd HMSz",
+                        "Ymd",
+                        "ymd"
+                      ),
+                      tz = get_default_tz()) {
   yell_if_missing(url, un, pw, pid = pid)
   httr::RETRY(
     "GET",
@@ -52,26 +67,22 @@ form_list <- function(pid = get_default_pid(),
   ) %>%
     yell_if_error(., url, un, pw) %>%
     httr::content(.) %>%
-    { # nolint
-      tibble::tibble(
-        name = purrr::map_chr(., "name"),
-        fid = purrr::map_chr(., "xmlFormId"),
-        version = purrr::map_chr(., "version", .default = NA),
-        state = purrr::map_chr(., "state"),
-        submissions = purrr::map_chr(., "submissions"),
-        created_at = purrr::map_chr(., "createdAt", .default = NA) %>%
-          isodt_to_local(),
-        created_by_id = purrr::map_int(., c("createdBy", "id")),
-        created_by = purrr::map_chr(., c("createdBy", "displayName")),
-        updated_at = purrr::map_chr(., "updatedAt", .default = NA) %>%
-          isodt_to_local(),
-        published_at = purrr::map_chr(., "publishedAt", .default = NA) %>%
-          isodt_to_local(),
-        last_submission = purrr::map_chr(., "lastSubmission", .default = NA) %>%
-          isodt_to_local(),
-        hash = purrr::map_chr(., "hash", .default = NA)
-      )
-    }
+    tibble::tibble(.) %>%
+    tidyr::unnest_wider(".", names_repair = "universal") %>%
+    tidyr::unnest_wider(
+      "reviewStates",
+      names_repair = "universal", names_sep = "_"
+    ) %>%
+    tidyr::unnest_wider(
+      "createdBy",
+      names_repair = "universal", names_sep = "_"
+    ) %>%
+    janitor::clean_names() %>%
+    dplyr::mutate_at(
+      dplyr::vars(dplyr::contains("_at")), # assume datetimes are named "_at"
+      ~ isodt_to_local(., orders = orders, tz = tz)
+    ) %>%
+    dplyr::mutate(fid = xml_form_id)
 }
 
 # usethis::use_test("form_list") # nolint

R/odata_submission_get.R

@@ -72,9 +72,9 @@
 #'   Default: FALSE.
 #' @param filter If provided, will filter responses to those matching the query.
 #'   For an `odkc_version` below 1.1, this parameter will be discarded.
-#'   In ODK Central v1.1, only the fields `system/submitterId` and
-#'   `system/submissionDate` are available to reference.
-#'   In ODK Central v1.2, other fields may become available.
+#'   As of ODK Central v1.5, the fields `system/submitterId`,
+#'   `system/submissionDate`, `__system/updatedAt`  and `__system/reviewState`
+#'   are available to reference.
 #'   The operators `lt`, `lte`, `eq`, `neq`, `gte`, `gt`, `not`, `and`, and `or`
 #'   are supported, and the built-in functions
 #'   `now`, `year`, `month`, `day`, `hour`, `minute`, `second.`
@@ -154,6 +154,12 @@
 #'   table = form_tables$url[1],
 #'   filter = "year(__system/submissionDate) lt 2020"
 #' )
+#'
+#' # To include all of the month of January, you need to filter by either
+#' # filter = "__system/submissionDate le 2020-01-31T23:59:59.999Z"
+#' # or
+#' # filter = "__system/submissionDate lt 2020-02-01".
+#' # Instead of timezone UTC ("Z"), you can also filter by any other timezone.
 #' }
 odata_submission_get <- function(table = "Submissions",
                                  skip = NULL,

R/project_list.R

@@ -64,9 +64,13 @@ project_list <- function(url = get_default_url(),
     httr::content(.) %>%
     tibble::tibble(.) %>%
     tidyr::unnest_wider(".", names_repair = "universal") %>%
+    # tidyr::unnest_wider(
+    #   "verbs",
+    #   names_repair = "universal", names_sep = "_"
+    # ) %>%
     janitor::clean_names(.) %>%
     dplyr::mutate_at(
-      dplyr::vars("last_submission", "created_at", "updated_at"),
+      dplyr::vars("last_submission", "created_at", "updated_at", "deleted_at"),
       ~ isodt_to_local(., orders = orders, tz = tz)
     ) %>%
     {

R/ruODK.R

@@ -17,5 +17,6 @@ utils::globalVariables(c(
   "path",
   "type",
   "variable",
-  "xx"
+  "xx",
+  "xml_form_id"
 ))

R/submission_audit_get.R

@@ -0,0 +1,133 @@
+#' Download server audit logs for one submission.
+#'
+#' `r lifecycle::badge("experimental")`
+#'
+#' This function is the workhorse for the vectorised function
+#' submission_audit_get,
+#' which gets all server audit logs for a list of submission IDs.
+#'
+#' Note this function returns a nested list containing any repeating subgroups.
+#' As the presence and length of repeating subgroups is non-deterministic and
+#' entirely depends on the completeness of the submission data, we cannot
+#' rectangle them any further here. Rectangling requires knowledge of the form
+#' schema and the completeness of submission data.
+#'
+#' `r lifecycle::badge("stable")`
+#'
+#' @template param-iid
+#' @template param-pid
+#' @template param-fid
+#' @template param-url
+#' @template param-auth
+#' @template param-retries
+#' @return A nested list of submission data.
+# nolint start
+#' @seealso \url{https://odkcentral.docs.apiary.io/#reference/forms-and-submissions/submissions/retrieving-submission-xml}
+# nolint end
+#' @family utilities
+#' @export
+#' @examples
+#' \dontrun{
+#' # See vignette("setup") for setup and authentication options
+#' # ruODK::ru_setup(svc = "....svc", un = "[email protected]", pw = "...")
+#'
+#' # With explicit credentials, see tests
+#' sl <- submission_list()
+#'
+#' sub <- get_one_submission_audit(sl$instance_id[[1]])
+#' listviewer::jsonedit(sub)
+#'
+#' # The details for one submission depend on the form fields
+#' length(sub)
+#' # > 11
+#'
+#' # The items are the field names. Repeated groups have the same name.
+#' names(sub)
+#' # > "meta"                     "encounter_start_datetime" "reporter"
+#' # > "device_id"                "location"                 "habitat"
+#' # > "vegetation_structure"     "perimeter"                "taxon_encounter"
+#' # > "taxon_encounter"          "encounter_end_datetime"
+#' }
+get_one_submission_audit <- function(iid,
+                                     pid = get_default_pid(),
+                                     fid = get_default_fid(),
+                                     url = get_default_url(),
+                                     un = get_default_un(),
+                                     pw = get_default_pw(),
+                                     retries = get_retries()) {
+  yell_if_missing(url, un, pw, pid = pid, fid = fid, iid = iid)
+  httr::RETRY(
+    "GET",
+    glue::glue(
+      "{url}/v1/projects/{pid}/forms/",
+      "{URLencode(fid, reserved = TRUE)}/submissions/{iid}/audits"
+    ),
+    httr::authenticate(un, pw),
+    times = retries
+  ) %>%
+    yell_if_error(., url, un, pw) %>%
+    httr::content(.)
+  # %>%
+  #   tibble::as_tibble() %>%
+  #   tidyr::unnest_wider()
+}
+
+#' Get submission audits for a list of submission instance IDs.
+#'
+#' `r lifecycle::badge("experimental")`
+#'
+#' Uses \code{\link{get_one_submission_audit}} on a list of submission
+#' instance IDs
+#' (`iid`) as returned from \code{\link{submission_list}$instance_id}.
+#' By giving the list of `iid` to download explicitly, that list can be
+#' modified using information not accessible to `ruODK`,
+#' e.g. `iid` can be restricted to "only not already downloaded submissions".
+#'
+#' To get the combined submission audit logs for one form
+#' as one single, concatenated `audit.csv` file, use `submission_export`.
+#'
+#' @param iid A list of submission instance IDs, e.g. from
+#'   \code{\link{submission_list}$instance_id}.
+#' @template param-pid
+#' @template param-fid
+#' @template param-url
+#' @template param-auth
+#' @template param-retries
+#' @return A nested list of submission audit logs.
+# nolint start
+#' @seealso \url{https://odkcentral.docs.apiary.io/#reference/forms-and-submissions/submissions/retrieving-submission-xml}
+# nolint end
+#' @family submission-management
+#' @export
+#' @examples
+#' \dontrun{
+#' # Step 1: Setup ruODK with OData Service URL (has url, pid, fid)
+#' ruODK::ru_setup(svc = "...")
+#'
+#' # Step 2: List all submissions of form
+#' sl <- submission_list()
+#'
+#' # Step 3: Get submission audit logs
+#' sa <- submission_audit_get(sl$instance_id)
+#' }
+submission_audit_get <- function(iid,
+                                 pid = get_default_pid(),
+                                 fid = get_default_fid(),
+                                 url = get_default_url(),
+                                 un = get_default_un(),
+                                 pw = get_default_pw(),
+                                 retries = get_retries()) {
+  yell_if_missing(url, un, pw, pid = pid, fid = fid, iid = iid)
+  tibble::tibble(
+    iid = iid,
+    pid = pid,
+    fid = fid,
+    url = url,
+    un = un,
+    pw = pw,
+    retries = retries
+  ) %>%
+    purrr::pmap(ruODK::get_one_submission_audit)
+}
+
+# usethis::use_test("submission_get_audit") # nolint

R/submission_get.R

@@ -80,6 +80,9 @@ get_one_submission <- function(iid,
 #' as `audit.csv`. This will overwrite all previous `audit.csv` files.
 #' To get the combined submission audit logs as one single, concatenated
 #' `audit.csv` file, use `submission_export`.
+#' Note that the submission audit is different from the server audit.
+#' The submission audit logs the actions of the enumerator in ODK Collect,
+#' whereas the form audit logs the actions of the project manager in ODK Central.
 #'
 #' @param iid A list of submission instance IDs, e.g. from
 #'   \code{\link{submission_list}$instance_id}.

R/submission_list.R

@@ -25,10 +25,7 @@
 #' \dontrun{
 #' # Set default credentials, see vignette("setup")
 #' ruODK::ru_setup(
-#'   svc = paste0(
-#'     "https://sandbox.central.getodk.org/v1/projects/14/",
-#'     "forms/build_Flora-Quadrat-0-2_1558575936.svc"
-#'   ),
+#'   svc = ...,
 #'   un = "[email protected]",
 #'   pw = "..."
 #' )
@@ -96,7 +93,7 @@ submission_list <- function(pid = get_default_pid(),
     ) %>%
     janitor::clean_names() %>%
     dplyr::mutate_at(
-      dplyr::vars(dplyr::contains("at")), # assume datetimes are named "_at"
+      dplyr::vars(dplyr::contains("_at")), # assume datetimes are named "_at"
       ~ isodt_to_local(., orders = orders, tz = tz)
     )
 }

data-raw/make_release.R

@@ -4,7 +4,7 @@
 # Implement new features
 #
 # Starting from a clean code base, create new dev version
-usethis::use_version(which = "dev")
+usethis::use_version(which = "patch")
 # Write code, add tests
 # Add new feature to news if user-facing
 usethis::edit_file("NEWS.md")