version 1.1.0

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: httr2
 Title: Perform HTTP Requests and Process the Responses
-Version: 1.0.7
+Version: 1.1.0
 Authors@R: c(
     person("Hadley", "Wickham", , "[email protected]", role = c("aut", "cre")),
     person("Posit Software, PBC", role = c("cph", "fnd")),
@@ -14,7 +14,7 @@ License: MIT + file LICENSE
 URL: https://httr2.r-lib.org, https://github.com/r-lib/httr2
 BugReports: https://github.com/r-lib/httr2/issues
 Depends: R (>= 4.0)
-Imports: cli (>= 3.0.0), curl (>= 6.0.1), glue, lifecycle, magrittr,
+Imports: cli (>= 3.0.0), curl (>= 6.1.0), glue, lifecycle, magrittr,
         openssl, R6, rappdirs, rlang (>= 1.1.0), vctrs (>= 0.6.3),
         withr
 Suggests: askpass, bench, clipr, covr, docopt, httpuv, jose, jsonlite,
@@ -24,14 +24,14 @@ VignetteBuilder: knitr
 Config/Needs/website: tidyverse/tidytemplate
 Config/testthat/edition: 3
 Config/testthat/parallel: true
-Config/testthat/start-first: resp-stream, req-perform
+Config/testthat/start-first: multi-req, resp-stream, req-perform
 Encoding: UTF-8
 RoxygenNote: 7.3.2
 NeedsCompilation: no
-Packaged: 2024-11-26 13:45:28 UTC; hadleywickham
+Packaged: 2025-01-18 14:13:16 UTC; hadleywickham
 Author: Hadley Wickham [aut, cre],
   Posit Software, PBC [cph, fnd],
   Maximilian Girlich [ctb]
 Maintainer: Hadley Wickham <[email protected]>
 Repository: CRAN
-Date/Publication: 2024-11-26 23:00:07 UTC
+Date/Publication: 2025-01-18 16:20:02 UTC

NAMESPACE

@@ -18,6 +18,7 @@ export(curl_help)
 export(curl_translate)
 export(example_github_client)
 export(example_url)
+export(is_online)
 export(iterate_with_cursor)
 export(iterate_with_link_url)
 export(iterate_with_offset)
@@ -67,6 +68,7 @@ export(req_cookies_set)
 export(req_dry_run)
 export(req_error)
 export(req_headers)
+export(req_headers_redacted)
 export(req_method)
 export(req_oauth)
 export(req_oauth_auth_code)
@@ -95,6 +97,7 @@ export(req_url)
 export(req_url_path)
 export(req_url_path_append)
 export(req_url_query)
+export(req_url_relative)
 export(req_user_agent)
 export(req_verbose)
 export(request)
@@ -115,10 +118,12 @@ export(resp_headers)
 export(resp_is_error)
 export(resp_link_url)
 export(resp_raw)
+export(resp_request)
 export(resp_retry_after)
 export(resp_status)
 export(resp_status_desc)
 export(resp_stream_aws)
+export(resp_stream_is_complete)
 export(resp_stream_lines)
 export(resp_stream_raw)
 export(resp_stream_sse)
@@ -143,7 +148,12 @@ export(secret_write_rds)
 export(signal_total_pages)
 export(throttle_status)
 export(url_build)
+export(url_modify)
+export(url_modify_query)
+export(url_modify_relative)
 export(url_parse)
+export(url_query_build)
+export(url_query_parse)
 export(with_mock)
 export(with_mocked_responses)
 export(with_verbosity)

NEWS.md

@@ -1,3 +1,63 @@
+# httr2 1.1.0
+
+## Lifecycle changes
+
+* `req_perform_stream()` is superseded in favor of `req_perform_connection()`,
+  which is no longer experimental (#625).
+
+* `with_mock()` and `local_mock()` are defunct and will be removed in the next
+  release.
+
+## New features
+
+* `is_online()` wraps `curl::has_internet()`, making it easy to tell if you're
+  currently online (#512).
+
+* `req_headers_redacted()` makes it easier to redact sensitive headers (#561).
+
+* `req_retry()` implements "circuit breaking", which immediatelys error after
+  multiple failures to the same server (e.g. because the server is down)
+  (#370).
+
+* `req_url_relative()` navigates to a relative URL (#449).
+
+* `resp_request()` returns the request associated with a response; this can
+   be useful when debugging (#604).
+
+* `resp_stream_is_complete()` checks if data remains in the stream (#559).
+
+* `url_modify()`, `url_modify_query()`, and `url_modify_relative()` modify
+  URLs (#464); `url_query_parse()` and `url_query_build()` parse and build
+  query strings (#425).
+
+## Bug fixes and minor improvements
+
+* OAuth response parsing errors now have a dedicated `httr2_oauth_parse` error
+  class that includes the original response object (@atheriel, #596).
+
+* `curl_translate()` converts cookie headers to `req_cookies_set()` (#431)
+  and JSON data to `req_body_json_modify()` calls (#258).
+
+* `print.request()` escapes `{}` in headers (#586).
+
+* `req_auth_aws_v4()` formats the AWS Authorization header correctly (#627).
+
+* `req_retry()` defaults to `max_tries = 2` when nethier `max_tries` nor
+  `max_seconds` is set. If you want to disable retries, set `max_tries = 1`.
+
+* `req_perform_connection()` gains a `verbosity` argument, which is useful for
+  understanding exactly how data is streamed back to you (#599).
+  `req_perform_promise()` also gains a `verbosity` argument.
+
+* `req_url_query()` can control how spaces are encoded with `.space` (#432).
+
+* `resp_link_url()` handles multiple `Link` headers (#587).
+
+* `resp_stream_sse()` will warn if it recieves a partial event.
+
+* `url_parse()` parses relative URLs with new `base_url` argument (#449) and
+  the uses faster and more correct `curl::curl_parse_url()` (#577).
+
 # httr2 1.0.7
 
 * `req_perform_promise()` upgraded to use event-driven async based on waiting efficiently on curl socket activity (#579).

R/curl.R

@@ -54,17 +54,32 @@ curl_translate <- function(cmd, simplify_headers = TRUE) {
 
   steps <- add_curl_step(steps, "req_url_query", dots = query)
 
+  # Cookies
+  cookies <- data$headers$`Cookie`
+  data$headers$`Cookie` <- NULL
+  if (!is.null(cookies)) {
+    steps <- add_curl_step(steps, "req_cookies_set", dots = cookies_parse(cookies))
+  }
+
   # Content type set with data
   type <- data$headers$`Content-Type`
-  data$headers$`Content-Type` <- NULL
+  if (!identical(data$data, "")) {
+    data$headers$`Content-Type` <- NULL
+  }
 
   headers <- curl_simplify_headers(data$headers, simplify_headers)
   steps <- add_curl_step(steps, "req_headers", dots = headers)
 
   if (!identical(data$data, "")) {
     type <- type %||% "application/x-www-form-urlencoded"
-    body <- data$data
-    steps <- add_curl_step(steps, "req_body_raw", main_args = c(body, type))
+    if (type == "application/json" && idempotent_json(data$data)) {
+      json <- jsonlite::parse_json(data$data)
+      args <- list(data = I(deparse1(json)))
+      steps <- add_curl_step(steps, "req_body_json", dots = args)
+    } else {
+      body <- data$data
+      steps <- add_curl_step(steps, "req_body_raw", main_args = c(body, type))
+    }
   }
 
   steps <- add_curl_step(steps, "req_auth_basic", main_args = unname(data$auth))
@@ -146,6 +161,11 @@ curl_normalize <- function(cmd, error_call = caller_env()) {
     method <- NULL
   }
 
+  if (has_name(args, "--json")) {
+    args <- c(args, list(`--data-raw` = args[["--json"]]))
+    headers[["Content-Type"]] <- "application/json"
+  }
+
   # https://curl.se/docs/manpage.html#-d
   # --data-ascii, --data
   #   * if first element is @, treat as path to read from, stripping CRLF
@@ -206,6 +226,7 @@ curl_opts <- "Usage: curl [<url>] [-H <header> ...] [-d <data> ...] [options] [<
       --data-ascii <data>      HTTP POST ASCII data
       --data-binary <data>     HTTP POST binary data
       --data-urlencode <data>  HTTP POST data url encoded
+      --json <data>            HTTP POST JSON
   -G, --get                    Put the post data in the URL and use GET
   -I, --head                   Show document info only
   -H, --header <header>        Pass custom header(s) to server
@@ -262,19 +283,17 @@ quote_name <- function(x) {
 
 add_curl_step <- function(steps,
                           f,
-                          ...,
                           main_args = NULL,
                           dots = NULL,
                           keep_if_empty = FALSE) {
-  check_dots_empty0(...)
   args <- c(main_args, dots)
 
   if (is_empty(args) && !keep_if_empty) {
     return(steps)
   }
 
   names <- quote_name(names2(args))
-  string <- vapply(args, is.character, logical(1L))
+  string <- map_lgl(args, function(x) is.character(x) && !inherits(x, "AsIs"))
   values <- unlist(args)
   values <- ifelse(string, encode_string2(values), values)
 
@@ -316,3 +335,22 @@ encode_string2 <- function(x) {
   names(out) <- names(x)
   out
 }
+
+cookies_parse <- function(x) {
+  pairs <- strsplit(x, "; ?")[[1]]
+  cookies <- parse_name_equals_value(pairs)
+
+  if (length(cookies) == 0) {
+    return(NULL)
+  }
+
+  out <- as.list(curl::curl_unescape(cookies))
+  names(out) <- curl::curl_unescape(names(cookies))
+  out
+}
+
+idempotent_json <- function(old) {
+  args <- formals(req_body_json)[c("auto_unbox", "null", "digits")]
+  new <- exec(jsonlite::toJSON, jsonlite::parse_json(old), !!!args)
+  jsonlite::minify(old) == jsonlite::minify(new)
+}

R/headers.R

@@ -1,15 +1,11 @@
 as_headers <- function(x, error_call = caller_env()) {
   if (is.character(x) || is.raw(x)) {
-    headers <- curl::parse_headers(x)
-    headers <- headers[grepl(":", headers, fixed = TRUE)]
+    parsed <- curl::parse_headers(x)
+    valid <- parsed[grepl(":", parsed, fixed = TRUE)]
+    halves <- parse_in_half(valid, ":")
 
-    equals <- regexpr(":", headers, fixed = TRUE)
-    pieces <- regmatches(headers, equals, invert = TRUE)
-
-    names <- map_chr(pieces, "[[", 1)
-    values <- as.list(trimws(map_chr(pieces, "[[", 2)))
-
-    new_headers(set_names(values, names), error_call = error_call)
+    headers <- set_names(trimws(halves$right), halves$left)
+    new_headers(as.list(headers), error_call = error_call)
   } else if (is.list(x)) {
     new_headers(x, error_call = error_call)
   } else {

R/httr2-package.R

@@ -11,6 +11,7 @@ NULL
 
 the <- new_environment()
 the$throttle <- list()
+the$breaker <- new_environment()
 the$cache_throttle <- list()
 the$token_cache <- new_environment()
 the$last_response <- NULL

R/is-online.R

@@ -0,0 +1,12 @@
+#' Is your computer currently online?
+#'
+#' This function uses some cheap heuristics to determine if your computer is
+#' currently online. It's a simple wrapper around [curl::has_internet()]
+#' exported from httr2 for convenience.
+#'
+#' @export
+#' @examples
+#' is_online()
+is_online <- function() {
+  curl::has_internet()
+}

R/iterate-responses.R

@@ -16,6 +16,11 @@
 #' @param resps A list of responses (possibly including errors).
 #' @param resp_data A function that takes a response (`resp`) and
 #'   returns the data found inside that response as a vector or data frame.
+#'
+#'   NB: If you're using [resp_body_raw()], you're likely to want to wrap its
+#'   output in `list()` to avoid combining all the bodies into a single raw
+#'   vector, e.g. `resps |> resps_data(\(resp) list(resp_body_raw(resp)))`.
+#'
 #' @examples
 #' reqs <- list(
 #'   request(example_url()) |> req_url_path("/ip"),
@@ -29,10 +34,14 @@
 #' resps |> resps_successes()
 #'
 #' # collect all their data
-#' resps |> resps_successes() |> resps_data(\(resp) resp_body_json(resp))
+#' resps |>
+#'   resps_successes() |>
+#'   resps_data(\(resp) resp_body_json(resp))
 #'
 #' # find requests corresponding to failure responses
-#' resps |> resps_failures() |> resps_requests()
+#' resps |>
+#'   resps_failures() |>
+#'   resps_requests()
 resps_successes <- function(resps) {
   resps[resps_ok(resps)]
 }

R/oauth-flow-auth-code.R

@@ -7,30 +7,33 @@
 #' This flow is the most commonly used OAuth flow where the user
 #' opens a page in their browser, approves the access, and then returns to R.
 #' When possible, it redirects the browser back to a temporary local webserver
-#' to capture the authorization code. When this is not possible (e.g. when
+#' to capture the authorization code. When this is not possible (e.g., when
 #' running on a hosted platform like RStudio Server), provide a custom
 #' `redirect_uri` and httr2 will prompt the user to enter the code manually.
 #'
 #' Learn more about the overall OAuth authentication flow in
-#' <https://httr2.r-lib.org/articles/oauth.html>.
+#' <https://httr2.r-lib.org/articles/oauth.html>, and more about the motivations
+#' behind this flow in
+#' <https://stack-auth.com/blog/oauth-from-first-principles>.
 #'
 #' # Security considerations
 #'
 #' The authorization code flow is used for both web applications and native
 #' applications (which are equivalent to R packages). `r rfc(8252)` spells out
 #' important considerations for native apps. Most importantly there's no way
 #' for native apps to keep secrets from their users. This means that the
-#' server should either not require a `client_secret` (i.e. a public client
-#' not an confidential client) or ensure that possession of the `client_secret`
-#' doesn't bestow any meaningful rights.
+#' server should either not require a `client_secret` (i.e. it should be a
+#' public client and not a confidential client) or ensure that possession of
+#' the `client_secret` doesn't grant any significant privileges.
 #'
-#' Only modern APIs from the bigger players (Azure, Google, etc) explicitly
-#' native apps. However, in most cases, even for older APIs, possessing the
-#' `client_secret` gives you no ability to do anything harmful, so our
-#' general principle is that it's fine to include it in an R package, as long
-#' as it's mildly obfuscated to protect it from credential scraping. There's
-#' no incentive to steal your client credentials if it takes less time to
-#' create a new client than find your client secret.
+#' Only modern APIs from major providers (like Azure and Google) explicitly
+#' support native apps. However, in most cases, even for older APIs, possessing
+#' the `client_secret` provides limited ability to perform harmful actions.
+#' Therefore, our general principle is that it's acceptable to include it in an
+#' R package, as long as it's mildly obfuscated to protect against credential
+#' scraping attacks (which aim to acquire large numbers of client secrets by
+#' scanning public sites like GitHub). The goal is to ensure that obtaining your
+#' client credentials is more work than just creating a new client.
 #'
 #' @export
 #' @family OAuth flows
@@ -362,7 +365,7 @@ oauth_flow_auth_code_listen <- function(redirect_uri = "http://localhost:1410")
 # https://www.w3.org/TR/html401/interact/forms.html#h-17.13.4.1
 # Spaces are first replaced by +
 parse_form_urlencoded <- function(query) {
-  query <- query_parse(query)
+  query <- url_query_parse(query)
   query[] <- gsub("+", " ", query, fixed = TRUE)
   query
 }

R/oauth-flow.R

@@ -12,6 +12,8 @@ oauth_flow_parse <- function(resp, source, error_call = caller_env()) {
       cli::cli_abort(
         "Failed to parse response from {.arg {source}} OAuth url.",
         parent = err,
+        resp = resp,
+        class = "httr2_oauth_parse",
         call = error_call
       )
     }
@@ -40,6 +42,8 @@ oauth_flow_parse <- function(resp, source, error_call = caller_env()) {
         "Failed to parse response from {.arg {source}} OAuth url.",
         "*" = "Did not contain {.code access_token}, {.code device_code}, or {.code error} field."
       ),
+      resp = resp,
+      class = "httr2_oauth_parse",
       call = error_call
     )
   }