diff --git a/DESCRIPTION b/DESCRIPTION index dff94d1..35282b8 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: httr2 Title: Perform HTTP Requests and Process the Responses -Version: 1.0.7 +Version: 1.1.0 Authors@R: c( person("Hadley", "Wickham", , "hadley@posit.co", role = c("aut", "cre")), person("Posit Software, PBC", role = c("cph", "fnd")), @@ -14,7 +14,7 @@ License: MIT + file LICENSE URL: https://httr2.r-lib.org, https://github.com/r-lib/httr2 BugReports: https://github.com/r-lib/httr2/issues Depends: R (>= 4.0) -Imports: cli (>= 3.0.0), curl (>= 6.0.1), glue, lifecycle, magrittr, +Imports: cli (>= 3.0.0), curl (>= 6.1.0), glue, lifecycle, magrittr, openssl, R6, rappdirs, rlang (>= 1.1.0), vctrs (>= 0.6.3), withr Suggests: askpass, bench, clipr, covr, docopt, httpuv, jose, jsonlite, @@ -24,14 +24,14 @@ VignetteBuilder: knitr Config/Needs/website: tidyverse/tidytemplate Config/testthat/edition: 3 Config/testthat/parallel: true -Config/testthat/start-first: resp-stream, req-perform +Config/testthat/start-first: multi-req, resp-stream, req-perform Encoding: UTF-8 RoxygenNote: 7.3.2 NeedsCompilation: no -Packaged: 2024-11-26 13:45:28 UTC; hadleywickham +Packaged: 2025-01-18 14:13:16 UTC; hadleywickham Author: Hadley Wickham [aut, cre], Posit Software, PBC [cph, fnd], Maximilian Girlich [ctb] Maintainer: Hadley Wickham Repository: CRAN -Date/Publication: 2024-11-26 23:00:07 UTC +Date/Publication: 2025-01-18 16:20:02 UTC diff --git a/MD5 b/MD5 index 6e1b9c1..34b9f08 100644 --- a/MD5 +++ b/MD5 @@ -1,79 +1,81 @@ -0d75514cf746a231e092af3764657e9c *DESCRIPTION +e6e86345963a58ee76d77ff813f077e3 *DESCRIPTION 67c32986371e772e798cc5d9f2dcc504 *LICENSE -36f2f39c39236643e54f18829b8d71d6 *NAMESPACE -376a33c2fa28a3539a9ad7cb38918ea3 *NEWS.md +9ac49d8446e70d982280916433c7a340 *NAMESPACE +253445ff6cdb70f079b34f4d1132d503 *NEWS.md 1658ee72d6e44a999ae79903bcd25229 *R/content-type.R -000654278bff779897500fd3ecf69e17 *R/curl.R -3dd7ea652be40280bb2f0727674c88b3 *R/headers.R -9f36afc7c707845ac1ce98694b3b5b70 *R/httr2-package.R +6514ffb9abc6617d02d7c94e53578d3b *R/curl.R +d93afa7275aedc64ef3e71d57f655adc *R/headers.R +10a8901c555fcc9fc38df0eb0a097fee *R/httr2-package.R c80a9eb1427c585807cecf618b6f3870 *R/import-standalone-obj-type.R 17bb123964057b839a42eda1c3da214b *R/import-standalone-purrr.R c40f882046a958444c6058a9e2cb9a3b *R/import-standalone-types-check.R +507abe6ac4547126f8fa5d727bed378e *R/is-online.R 9b30eaec39a5108b8d433a20f9634c02 *R/iterate-helpers.R -6a481f63dd4378d148d3fead8e0f170e *R/iterate-responses.R +e27a45dc3c3644ad419b94e9c0ea12de *R/iterate-responses.R 2d943bdad354e34cbd4a45100b0d1e6b *R/iterate.R b9d534f99dcda91551c3fcd746e2a445 *R/jwt.R 89a47587293ef9a5c2a290cdda214944 *R/multi-req.R b1ae849ccaeae58d217eed06275a9b16 *R/oauth-client.R -d9db7d2520133a6ac3aeac7f9119ae70 *R/oauth-flow-auth-code.R +7a7e997e32344970cfb1fe7092fbe049 *R/oauth-flow-auth-code.R a83498a469807c9f97bb6ccc4ea35a1a *R/oauth-flow-client-credentials.R 8fb1c52ebc692107974ec313bde9dcdb *R/oauth-flow-device.R 5c091bad88025c39d795d0c50a69997a *R/oauth-flow-jwt.R c7d09f1525f2f600e9b55631e34e8cbe *R/oauth-flow-password.R 4861c984d0dfe47cb1a0db1ce21b4b71 *R/oauth-flow-refresh.R 947401cd8e1d67a790685757392fb50c *R/oauth-flow-token-exchange.R -69b08195db7d117f9aa252d243a4446a *R/oauth-flow.R +304974c85e1a0dec4738a95e9548fa6b *R/oauth-flow.R e61af568972d4a7c08105046de860163 *R/oauth-token.R 81a6bc1821551a54e013134e75e022a0 *R/oauth.R 750539f93e92d55bcccfcf9960f78b8d *R/parse.R -9b92feda3bc2daee5fd60f5d1418fe03 *R/progress-bars.R +3019a5a632f44a7a0035cb59d5b206ca *R/progress-bars.R ce08586490a871eff3cc5154fb92f8f1 *R/progress.R -9d74c74340e929a4ef1d926f147f639b *R/req-auth-aws.R +95bd5190615dff5795fecc3f7f56772e *R/req-auth-aws.R d783da3750bbb4858a7a1593a0836f3c *R/req-auth-sign.R -0ad8a12bf7f3993e45e4e4df582f5bea *R/req-auth.R -8c7a2b332b3da42ca98c110ee4845028 *R/req-body.R +ef23d2e4aff0606bddf477cf00bf20d1 *R/req-auth.R +c70563cd2f402a195ae514e4d9122e55 *R/req-body.R ab0f0916cc7dd698b9c7719a6dd7445e *R/req-cache.R fc33319973b4af684d277c84f6b05613 *R/req-cookies.R 57cdca5cee4acb32444edc8f31dcb308 *R/req-error.R -f541ab8aed99da114e2587ea95740268 *R/req-headers.R +ec15960db4e67c23e6ca3fb48488964c *R/req-headers.R c64e21ca81ec2590c380e74b89a9119f *R/req-method.R -b2d2fbde7e21f1998a435a1030a9e6d8 *R/req-mock.R -432edc5de1c29c3713cac719d59283fa *R/req-options.R -01c39cfad634d71fbdec1d8322288cdd *R/req-perform-connection.R -cf459caaf28612288d4049f8fd184431 *R/req-perform-stream.R -603b107c17ac8cd8965db9002e5a75ae *R/req-perform.R +c892d24ea5aea680b8d203a2a5718e3b *R/req-mock.R +f5a0f0cff888758e195d984cc6d441e8 *R/req-options.R +6f9368749624d4539e94231783bab509 *R/req-perform-connection.R +c3169dc84805a1e40510761aa2718c6f *R/req-perform-stream.R +d69b03c55a0d390c5314c4ee2419240c *R/req-perform.R 06612e2c3c9ca2995093c9174a3761ed *R/req-policy.R -5a4147ff54d90b5d2ae2527ea1abaab8 *R/req-promise.R -750078e7065abbe239d52a45b56d5b6e *R/req-retries.R +3cab57060a89290541e3a3677cd8cb0a *R/req-promise.R +5f8e4063ee130b4a9c1d5589d97e671f *R/req-retries.R 55ed14b760ba2f18d64d4c69ba064611 *R/req-template.R 9ce715edfb171a80d520c538f25e29ed *R/req-throttle.R -09e8a5d0e0b1ea1bf62d8cc7ec66b8ed *R/req-url.R +39df0c5d2c8b0fa179e421a49d7d95fc *R/req-url.R 13c13cea034751a029afaa80f629fa6c *R/req.R a3998f317ba316c845455ce58a73951c *R/resp-body.R -f7ce50805d872ea60ef6ba41f6c24aa6 *R/resp-headers.R +d36d5bd0e31867e8f580fee242c06fb5 *R/resp-headers.R +e3273082cbfdb95d21e7d3090a286d54 *R/resp-request.R 458cc0a7eb6a6f17289bb64f5ba3be5e *R/resp-status.R -b13c7d2cd4f8481a9579e6c7641a8587 *R/resp-stream-aws.R -ac236c510797c9e8123efe6a727cbeb9 *R/resp-stream.R +929ad53e91d9fe3f59b399a78d223952 *R/resp-stream-aws.R +c58e9126564b463084bb440e4b4312c8 *R/resp-stream.R 4de6118af4838858409a4f91bf6294b6 *R/resp-url.R ff36eca1c2f4c905713060138ff33c2c *R/resp.R 8f02cbe77f0b346ddaf3a7e5158e387c *R/roxygen2.R c50272978398261ca3af6973a716f457 *R/secret.R -b120c3e290c54c23ca898a16afbbaa75 *R/sequential.R +4dd469f37a74b4892d2f42b7e05f751c *R/sequential.R 18fbe8fc2497eef4d6f13c7e45f5508f *R/sysdata.rda -afd0b7ec7c23651c632a0db4cdf17a98 *R/test.R -9956c44afcedd46015e3f850fffda96a *R/url.R -3e468224dbe2421903303c28f1912a4f *R/utils-multi.R +1c33e6a0ddd76ebe77a2cd6fffb32f86 *R/test.R +9e85d6175d0dec3b64f93d5d538b74a9 *R/url.R +46898a02a6ba2dfeb06d0dd363279b06 *R/utils-multi.R b56ff2b193ecba52f9ec7f3b10cca008 *R/utils-pipe.R -4dc1cfb0f44bbaad06909586afb0259b *R/utils.R +53fed380459feb8c073b1b197069f987 *R/utils.R 059a3c630cbf1d8cba089f78486fa4d9 *R/zzz.R -21eb0f1ba57098c6a253787dc3b72f79 *README.md +f9f24424e238b15c3a186d47a79f444b *README.md ed7097e327faf147198f31bb3553c326 *build/vignette.rds de3fc7fc6d5f0ec1ddf799c84d04a63e *cleanup b547518249e0da4528616c8ed2382036 *configure 757ef8281ef2a1ebcbcc8235aeee2d0e *configure.win 62d1743a605c7aa827b5742a25950087 *inst/doc/httr2.R 5cc7d3c34b6a4fcfde1bde0e741c7823 *inst/doc/httr2.Rmd -eadde17e78df0b46485869084b0065f0 *inst/doc/httr2.html +9044e751c5441047bf9ff6efcf205d0e *inst/doc/httr2.html 8ac6a7b7aac968e8a0e93c58049261ef *man/curl_translate.Rd fc06ec6c9f17492418496a7875111a55 *man/example_url.Rd a1cbaf3f328e8d74e747faacf640c7fc *man/figures/lifecycle-archived.svg @@ -87,10 +89,11 @@ ed42e3fbd7cc30bc6ca8fa9b658e24a8 *man/figures/lifecycle-stable.svg bf2f1ad432ecccee3400afe533404113 *man/figures/lifecycle-superseded.svg 5533c65601c00e2027a5b4f2f11d460c *man/figures/logo.png ebab256683f284e8b780f139c2c0f0cc *man/httr2-package.Rd +49d626df322d7cf662377e1c01b06f2d *man/is_online.Rd a6b2fc640f68348bb1df1e055a55710a *man/iterate_with_offset.Rd 36e14f11e9656c1a80e51c6f137ea052 *man/jwt_claim.Rd bbdae5dea29b21d47d7c7069c0d46ab1 *man/last_response.Rd -3a9bb92dd4c4edee672a4ec7d2ec71c6 *man/multi_req_perform.Rd +6a3dc58851540795e0dec9b8d2d975b6 *man/multi_req_perform.Rd 04804ae5afb484f583664a8ef5bf51e4 *man/oauth_cache_clear.Rd 9b54af46994b24405f4751fd1086c06f *man/oauth_cache_path.Rd 28127c166aacb0222ee462c49822f29e *man/oauth_client.Rd @@ -101,40 +104,40 @@ e8432ee09c99bc777397e20e6181d444 *man/oauth_token.Rd 2af58cfe96d548ef9086a791d1570a91 *man/oauth_token_cached.Rd 622607c5f5df647186110c9b1ce435b5 *man/obfuscate.Rd 8f4aad003a999fae004ba9361f9a99d6 *man/pipe.Rd -4cfc1fa751b1d593aad4a6911539bfc2 *man/progress_bars.Rd +e68629ab3d3b207dcf7346a6385b0393 *man/progress_bars.Rd a785e57eddb1cd148bc0b0070bace521 *man/req_auth_aws_v4.Rd -50ed8972f31a273d13ddfe39e634a7b6 *man/req_auth_basic.Rd +b9258b3b58269decb6e84b3d0e9d544b *man/req_auth_basic.Rd e62ad6ff1724af6693421e1722d323ee *man/req_auth_bearer_token.Rd -bab65521b1355bba1ffe6f4358387dc5 *man/req_body.Rd +3921f52aec7ba33f0f5c91151ecdc70a *man/req_body.Rd 58ddbc800a0f9676c146ff3a9cfb964c *man/req_cache.Rd 9ff896ff8069ceb86a9dd115d454aedc *man/req_cookie_preserve.Rd -c3511461abf203e6051d0cce349fe475 *man/req_dry_run.Rd +9cae1589f3996495f6acfdbc38a43481 *man/req_dry_run.Rd 20f4b9126f19b176e452de860a8f427c *man/req_error.Rd -014d664aff294cdc7906257a1c5c92b8 *man/req_headers.Rd +38a9080eba606368cde72933e029bcd7 *man/req_headers.Rd 016775046384c8579f22ad7b8cd54f6a *man/req_method.Rd bb00e2e79745986b4cef04e6b8e56e5c *man/req_oauth.Rd -ce7d4687bbafeb001c6fa3ee8fd02658 *man/req_oauth_auth_code.Rd +ee2167ae16d025385e6eb9f09204dd39 *man/req_oauth_auth_code.Rd c474b629b73cde26b2184c763aef00ae *man/req_oauth_bearer_jwt.Rd c537e872c7e8daa6e9d2aeae0c1cf119 *man/req_oauth_client_credentials.Rd f625f409275c2bd63080bb856cb37d0e *man/req_oauth_device.Rd -2c7320823145170c2102cc72e4679209 *man/req_oauth_password.Rd +9ee09168c57126cd537ab66e3328b2f4 *man/req_oauth_password.Rd 74b982010cb6f5505de3888f7dd79461 *man/req_oauth_refresh.Rd 551e3d4aead599d0f7794a73cc52368d *man/req_oauth_token_exchange.Rd d0bfcfa3163dd92ba9dc87393077fda3 *man/req_options.Rd 538e1b1c5a396431a6ab35259e5c7af6 *man/req_perform.Rd -b37b4a851f08a3ff36b47921fa8927b5 *man/req_perform_connection.Rd -89f5a7fcb0463641f5dfce8ddb817df0 *man/req_perform_iterative.Rd -7059e093f4f97de7e5999914843c2185 *man/req_perform_parallel.Rd -39c42ba22649fb8e8fedd03ec055f2d3 *man/req_perform_promise.Rd -d4785d6379db8d180e7e3b5b1830f588 *man/req_perform_sequential.Rd -e83ae46099e9766ee7bd0cb2d982070d *man/req_perform_stream.Rd +1c7114ac7f6c37f0256df98a910ad977 *man/req_perform_connection.Rd +66dd7c639de6e281e409c49bd35e4083 *man/req_perform_iterative.Rd +612a79beb81dddee30e3938010dbf58b *man/req_perform_parallel.Rd +08791d0c4086425135222aeb2fa3f325 *man/req_perform_promise.Rd +1ef11a8be998045ba273a592247205ba *man/req_perform_sequential.Rd +f702b9e086842784d3d5d682f31b9840 *man/req_perform_stream.Rd b41d5e8f316515288ff886df0e4881d7 *man/req_progress.Rd 01cf8fa1cbbe0443d56be0b3ca48dfb4 *man/req_proxy.Rd -1f14f440aa429db5435d120486770fd2 *man/req_retry.Rd +de20dee6cc2aec1e82d723789e9eb9d8 *man/req_retry.Rd e99c0b063c7a0493acd4000fa7de6690 *man/req_template.Rd 3095f9a645fbcb15c5cf42a02ff13e53 *man/req_throttle.Rd 00f3f81313e9c0587e9449a7973fa117 *man/req_timeout.Rd -6139a2835d121de7435de9655b73eeb8 *man/req_url.Rd +6881d56f9d08c5819d3855fb87a0abdd *man/req_url.Rd 04f71709a8a1f84dc93cc18cad22363e *man/req_user_agent.Rd bf9457f0a9d87135253910d6eb2571ef *man/req_verbose.Rd da6d32cfb889bd0cf0857b565ccd2594 *man/request.Rd @@ -145,21 +148,25 @@ eda7b85b891d9d84c032fd47b7700129 *man/resp_date.Rd fedeb826affa2bceabef362c4e25d9e3 *man/resp_headers.Rd 1c66073d260f82878bb3c2f342717def *man/resp_link_url.Rd db26222bc778e6cedf51a806c65ab392 *man/resp_raw.Rd +affdd5eba3a4b5d5aa1727b6a0998faa *man/resp_request.Rd 2ce94385e1e1652829a0f366f9631189 *man/resp_retry_after.Rd 2b0209c459a4595d3624cbcd2e23618f *man/resp_status.Rd -695b76dc3c3aa2fd47a3a7bfcee2bea0 *man/resp_stream_raw.Rd +da1ea5336af84d41afedcf1227f0690f *man/resp_stream_raw.Rd 38f37482fa2a5fac8a3e743e342336fd *man/resp_url.Rd 712d1929f961ef8b93224aab5238fd30 *man/response.Rd -7149070f6fbc8cbf62db975f8c0c468e *man/resps_successes.Rd +2b10262b289e5556af9332f3bbd281a3 *man/resps_successes.Rd 06114c024f153dea1dfdb3b2d373eadf *man/secrets.Rd ddef6b65c32aabc60f757ed193a88134 *man/signal_total_pages.Rd 3045c501a77cbffeda91c520dfccfe1f *man/throttle_status.Rd -8cb5433fa4b42103e4039005fc54278c *man/url_parse.Rd +56d8688106b975ba2be8a09c3682adc8 *man/url_build.Rd +73b86e58e8d148fd4f962884890fd271 *man/url_modify.Rd +6ba9eb8f20eadbe80eecc2e479ba2133 *man/url_parse.Rd +6ede730cfcced60bbcfbdd87b60cd00f *man/url_query_parse.Rd 1db5ba24b3f55a0ab371859f591c26ba *man/with_mocked_responses.Rd ee022e3c977f1c527ec0ec29a0c22eb6 *man/with_verbosity.Rd c2b2cef37a5921be7c43b192f6ab8cae *tests/testthat.R 2abbb09e4412155bb8c71d2f5d01f842 *tests/testthat/_snaps/content-type.md -1d96277aefa951c04ab5e854cfc54aa9 *tests/testthat/_snaps/curl.md +2d22c26ed0f194f6a4a6a9997afafe0e *tests/testthat/_snaps/curl.md bd0372ba1a8a3abbcd4986ce7b05fb59 *tests/testthat/_snaps/headers.md 80f7f8dd80eaec7d4dd7dcbf843b5522 *tests/testthat/_snaps/iterate-helpers.md 7fed77e81c3f22fa048ff542e406e40d *tests/testthat/_snaps/iterate.md @@ -177,77 +184,79 @@ e69cb8912ae3ebe10d09c87ed1a286a9 *tests/testthat/_snaps/req-cache.md 3e31359c6520222447d6233f39f415b9 *tests/testthat/_snaps/req-cookies.md f9c831a6fd93a36222a2f8b026dcb933 *tests/testthat/_snaps/req-error.md bc00406611a4379e24b3a5e20c84ba1e *tests/testthat/_snaps/req-headers.md -f9590c20318dd7d97c35ad3c6410040d *tests/testthat/_snaps/req-mock.md +ec8a652884a6bd3a561f705a2cb1f57f *tests/testthat/_snaps/req-mock.md 5197a993461f4b825ee8a646be0d96cf *tests/testthat/_snaps/req-options.md 7320994deee7fe8397fe38984a289ab5 *tests/testthat/_snaps/req-perform-connection.md 304742dbf0ddb5549a47bddcb27b8b6b *tests/testthat/_snaps/req-perform-stream.md 94f451fd8a53349fc62f3d59e6f4f53c *tests/testthat/_snaps/req-perform.md 9b07d019e4eab37f0f6d97245e61535f *tests/testthat/_snaps/req-policy.md -9f7760e1e3a993b03ed824c5d6f6ef92 *tests/testthat/_snaps/req-promise.md -6ad8c0cece748dec6f7a511414408857 *tests/testthat/_snaps/req-retries.md +709f80c1ac1823018b98f2f6416afe4c *tests/testthat/_snaps/req-promise.md +2e1a950eef2130c8e6f55831779589be *tests/testthat/_snaps/req-retries.md a6aa2b17597e91ded969ff83d8ae1aa7 *tests/testthat/_snaps/req-template.md -76cbea12ce426ece85dbe3e03a9a12ef *tests/testthat/_snaps/req-url.md -05349394ee4361f76aaf4c3519bccfb0 *tests/testthat/_snaps/req.md +90f8f6da1dae21b0bc53afbb60bfe02f *tests/testthat/_snaps/req-url.md +1cc1dbb126597203dd6a734385cd462e *tests/testthat/_snaps/req.md 60e6374c33a9a6251ee9662ec1eab2cb *tests/testthat/_snaps/resp-body.md dd83ab51d56154baedd0fce1794d4b78 *tests/testthat/_snaps/resp-status.md 5d9dcaab39402b44c55500061ecadb10 *tests/testthat/_snaps/resp-stream-aws.md -f417eaf75c2d9b7ed51d22f6423e3260 *tests/testthat/_snaps/resp-stream.md +1beefe7ee4fdd0a088645ae3cad3a7bc *tests/testthat/_snaps/resp-stream.md 2f39783d8d60a0b9e37069ea62beb1b5 *tests/testthat/_snaps/resp.md 9a2997f35ff08553b17ad7ffd5445ee2 *tests/testthat/_snaps/secret.md 23197b884a9489a8ba9f505cf7502382 *tests/testthat/_snaps/sequential.md -a09476510fc0c89f7e10f102561a24a0 *tests/testthat/_snaps/url.md +14e117a66571f3c4ae385b0f8e1e368b *tests/testthat/_snaps/url.md 44c5d6d7c37249a3c71939b83eafa622 *tests/testthat/_snaps/utils-multi.md 5e637a98c47c01cc54e95cbdf515cc49 *tests/testthat/_snaps/utils.md da0824f5ba70df4cb80983244d7a07d6 *tests/testthat/azure-cert.rds 6d46dd9362a66c2dc7a7b7ca04e24b66 *tests/testthat/azure-key.rds +5f7373b1f0b2b11efdb832831fc475d3 *tests/testthat/helper-promise.R 6dda7928deb28c8a2939a2fcbc54e082 *tests/testthat/helper-webfakes.R -a7242fd178fb212ec4159995f8aa724e *tests/testthat/helper.R +5655db9da51ccc1133a92f50bb4adfe0 *tests/testthat/helper.R 63864fa9e8c9e485bdf6026c2aeb5f1a *tests/testthat/test-content-type.R -8708c86d4aef6d15f08a6ca669bc549d *tests/testthat/test-curl.R +c9586ba099126e1819ba10cb2ab4dc46 *tests/testthat/test-curl.R bb05f379e71b020064f3b0db5a23922f *tests/testthat/test-headers.R 5be02d74f062de3487e54c8d4bfa25d7 *tests/testthat/test-iterate-helpers.R 6d3ec8a9144133090eb42a027c84d50b *tests/testthat/test-iterate-responses.R 6c02a6d5d34e2b3c1c33906f40a90bc5 *tests/testthat/test-iterate.R -6d1d3570ef8748497c45a4ddd788ac5a *tests/testthat/test-multi-req.R +c184e494aa79e82876f5c41993c55cb8 *tests/testthat/test-multi-req.R da1c04bca53e199b761be5b06efde1a8 *tests/testthat/test-oauth-client.R -b9f2d0c46a1f0c0c6cbc35c3538c3845 *tests/testthat/test-oauth-flow-auth-code.R +c40907499399974a0d15a724c14f1486 *tests/testthat/test-oauth-flow-auth-code.R 29b4f80b4448eb91aa5e19966bd890b6 *tests/testthat/test-oauth-flow-jwt-google.rds bca4aa0a15213a62d6a70f6f8597ae75 *tests/testthat/test-oauth-flow-jwt.R 1a61b2fbcbfbf3e08fc4c3c3f5a86e76 *tests/testthat/test-oauth-flow-refresh.R -38e56020e2cd8f27a920c997515c41a6 *tests/testthat/test-oauth-flow.R +fa281ad7b70b8987d371d1dc08faeb7c *tests/testthat/test-oauth-flow.R 3b6a5ecf363085dbaf2e9bb26f82bdb9 *tests/testthat/test-oauth-token.R b4eb40ea8f3923f2a17c426b1accb683 *tests/testthat/test-oauth.R 7aafd3f87651aad2096256f730ca3f96 *tests/testthat/test-parse.R -dfb8665cea4d355c8023f94f867aa58a *tests/testthat/test-req-auth-aws.R +038d9fe8380aba66894070cf5d8be287 *tests/testthat/test-req-auth-aws.R 9731ce2fae5c9084c1797c99dd5245a3 *tests/testthat/test-req-auth.R fec6f6bf4456ce1452a73f9050a0d409 *tests/testthat/test-req-body.R 7b77b91744d179da2e77f6961e3bfe79 *tests/testthat/test-req-cache.R 7ea4295cb8bfc2f4b127191dc374193b *tests/testthat/test-req-cookies.R 94aad378e2187ab554ee0b46cf005ac3 *tests/testthat/test-req-error.R -4ca49c15d21c65abc3a132754e40139b *tests/testthat/test-req-headers.R +298c3a5c3f52cab75afc2cd8ad6f85e8 *tests/testthat/test-req-headers.R abe3ec676219640db627c485562b04c3 *tests/testthat/test-req-method.R -1fc31f502e426bc8f24fd38551bb23b5 *tests/testthat/test-req-mock.R +a0b363b87327500f0d924478f5043d85 *tests/testthat/test-req-mock.R da98932f68102ee2b1cf07eca3ae614b *tests/testthat/test-req-options.R 0ae9ae926545591c692f33c9c46c96f0 *tests/testthat/test-req-perform-connection.R 622a71b5652729264b4b858f5b4ff8bf *tests/testthat/test-req-perform-stream.R -307eb769ec31e16a78046956bc91f6fd *tests/testthat/test-req-perform.R +f10989ae82cde825d9d80dbe60e2f485 *tests/testthat/test-req-perform.R 8e5b70954049fa379b900743857d68d0 *tests/testthat/test-req-policy.R -8805578b840c04aa5a67a68b754a768b *tests/testthat/test-req-promise.R -c3c7553bc1472f7138b7cc4d9b59b01f *tests/testthat/test-req-retries.R +4b27b8fe612a15a8cd17ae2970f8591e *tests/testthat/test-req-promise.R +8d441c2ab83574236eb2a380b33e46da *tests/testthat/test-req-retries.R d8ebf8812ec156c0c00443c71d6e65e8 *tests/testthat/test-req-template.R 7f8382c81e7208a3adbaf6e0beb58946 *tests/testthat/test-req-throttle.R -7dbc34e12b4fc2507f844c9fff1edf24 *tests/testthat/test-req-url.R -6ad8ca4a0cdaa0d99da0ea8f43387d92 *tests/testthat/test-req.R +986d06bf1e471a3c14eb7b30f3a1517f *tests/testthat/test-req-url.R +75cd4425b038e917ec04921e5dd051e5 *tests/testthat/test-req.R df705107d13e79dfb0401bd655df7f65 *tests/testthat/test-resp-body.R -02cfdcfb6810eaddb3c4dd329eaf9cbc *tests/testthat/test-resp-headers.R +e0cfb159e674c5cd3ecfdee5e2e94b4d *tests/testthat/test-resp-headers.R +f0c4119da95f8859efd0a6580f240750 *tests/testthat/test-resp-request.R 306705998dbf3b108cbeb46449a01ec5 *tests/testthat/test-resp-status.R 8823fcea7255ac7e0f69b26413435c82 *tests/testthat/test-resp-stream-aws.R -977f97b683e621a2477553c397be413d *tests/testthat/test-resp-stream.R +7a4caeb485d9c995f9950bb12455bfe6 *tests/testthat/test-resp-stream.R 2a891e35620fe4003801eb3be2faceed *tests/testthat/test-resp-url.R 73c6a817d1876b11d5e1f972837f3c54 *tests/testthat/test-resp.R 6e1cf597de2078da4f7cad93112d6b9e *tests/testthat/test-secret.R 57013c2bbff9e24e5d76e5f86b163217 *tests/testthat/test-sequential.R -22cd2141eafbe522695d75e4e7006375 *tests/testthat/test-url.R +4872fc4a9ef74c2161da4f7d4abcb10a *tests/testthat/test-url.R 2f68cfd0571464f93d4e04e46824605a *tests/testthat/test-utils-multi.R f060f901cda6a431960eebc30e636906 *tests/testthat/test-utils.R 04b0a09ebe1847f15c2b36527ba834ca *tools/examples.R diff --git a/NAMESPACE b/NAMESPACE index 0613c62..05a7599 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -18,6 +18,7 @@ export(curl_help) export(curl_translate) export(example_github_client) export(example_url) +export(is_online) export(iterate_with_cursor) export(iterate_with_link_url) export(iterate_with_offset) @@ -67,6 +68,7 @@ export(req_cookies_set) export(req_dry_run) export(req_error) export(req_headers) +export(req_headers_redacted) export(req_method) export(req_oauth) export(req_oauth_auth_code) @@ -95,6 +97,7 @@ export(req_url) export(req_url_path) export(req_url_path_append) export(req_url_query) +export(req_url_relative) export(req_user_agent) export(req_verbose) export(request) @@ -115,10 +118,12 @@ export(resp_headers) export(resp_is_error) export(resp_link_url) export(resp_raw) +export(resp_request) export(resp_retry_after) export(resp_status) export(resp_status_desc) export(resp_stream_aws) +export(resp_stream_is_complete) export(resp_stream_lines) export(resp_stream_raw) export(resp_stream_sse) @@ -143,7 +148,12 @@ export(secret_write_rds) export(signal_total_pages) export(throttle_status) export(url_build) +export(url_modify) +export(url_modify_query) +export(url_modify_relative) export(url_parse) +export(url_query_build) +export(url_query_parse) export(with_mock) export(with_mocked_responses) export(with_verbosity) diff --git a/NEWS.md b/NEWS.md index 6396c60..b8aa670 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,63 @@ +# httr2 1.1.0 + +## Lifecycle changes + +* `req_perform_stream()` is superseded in favor of `req_perform_connection()`, + which is no longer experimental (#625). + +* `with_mock()` and `local_mock()` are defunct and will be removed in the next + release. + +## New features + +* `is_online()` wraps `curl::has_internet()`, making it easy to tell if you're + currently online (#512). + +* `req_headers_redacted()` makes it easier to redact sensitive headers (#561). + +* `req_retry()` implements "circuit breaking", which immediatelys error after + multiple failures to the same server (e.g. because the server is down) + (#370). + +* `req_url_relative()` navigates to a relative URL (#449). + +* `resp_request()` returns the request associated with a response; this can + be useful when debugging (#604). + +* `resp_stream_is_complete()` checks if data remains in the stream (#559). + +* `url_modify()`, `url_modify_query()`, and `url_modify_relative()` modify + URLs (#464); `url_query_parse()` and `url_query_build()` parse and build + query strings (#425). + +## Bug fixes and minor improvements + +* OAuth response parsing errors now have a dedicated `httr2_oauth_parse` error + class that includes the original response object (@atheriel, #596). + +* `curl_translate()` converts cookie headers to `req_cookies_set()` (#431) + and JSON data to `req_body_json_modify()` calls (#258). + +* `print.request()` escapes `{}` in headers (#586). + +* `req_auth_aws_v4()` formats the AWS Authorization header correctly (#627). + +* `req_retry()` defaults to `max_tries = 2` when nethier `max_tries` nor + `max_seconds` is set. If you want to disable retries, set `max_tries = 1`. + +* `req_perform_connection()` gains a `verbosity` argument, which is useful for + understanding exactly how data is streamed back to you (#599). + `req_perform_promise()` also gains a `verbosity` argument. + +* `req_url_query()` can control how spaces are encoded with `.space` (#432). + +* `resp_link_url()` handles multiple `Link` headers (#587). + +* `resp_stream_sse()` will warn if it recieves a partial event. + +* `url_parse()` parses relative URLs with new `base_url` argument (#449) and + the uses faster and more correct `curl::curl_parse_url()` (#577). + # httr2 1.0.7 * `req_perform_promise()` upgraded to use event-driven async based on waiting efficiently on curl socket activity (#579). diff --git a/R/curl.R b/R/curl.R index 6016638..a978965 100644 --- a/R/curl.R +++ b/R/curl.R @@ -54,17 +54,32 @@ curl_translate <- function(cmd, simplify_headers = TRUE) { steps <- add_curl_step(steps, "req_url_query", dots = query) + # Cookies + cookies <- data$headers$`Cookie` + data$headers$`Cookie` <- NULL + if (!is.null(cookies)) { + steps <- add_curl_step(steps, "req_cookies_set", dots = cookies_parse(cookies)) + } + # Content type set with data type <- data$headers$`Content-Type` - data$headers$`Content-Type` <- NULL + if (!identical(data$data, "")) { + data$headers$`Content-Type` <- NULL + } headers <- curl_simplify_headers(data$headers, simplify_headers) steps <- add_curl_step(steps, "req_headers", dots = headers) if (!identical(data$data, "")) { type <- type %||% "application/x-www-form-urlencoded" - body <- data$data - steps <- add_curl_step(steps, "req_body_raw", main_args = c(body, type)) + if (type == "application/json" && idempotent_json(data$data)) { + json <- jsonlite::parse_json(data$data) + args <- list(data = I(deparse1(json))) + steps <- add_curl_step(steps, "req_body_json", dots = args) + } else { + body <- data$data + steps <- add_curl_step(steps, "req_body_raw", main_args = c(body, type)) + } } steps <- add_curl_step(steps, "req_auth_basic", main_args = unname(data$auth)) @@ -146,6 +161,11 @@ curl_normalize <- function(cmd, error_call = caller_env()) { method <- NULL } + if (has_name(args, "--json")) { + args <- c(args, list(`--data-raw` = args[["--json"]])) + headers[["Content-Type"]] <- "application/json" + } + # https://curl.se/docs/manpage.html#-d # --data-ascii, --data # * if first element is @, treat as path to read from, stripping CRLF @@ -206,6 +226,7 @@ curl_opts <- "Usage: curl [] [-H
...] [-d ...] [options] [< --data-ascii HTTP POST ASCII data --data-binary HTTP POST binary data --data-urlencode HTTP POST data url encoded + --json HTTP POST JSON -G, --get Put the post data in the URL and use GET -I, --head Show document info only -H, --header
Pass custom header(s) to server @@ -262,11 +283,9 @@ quote_name <- function(x) { add_curl_step <- function(steps, f, - ..., main_args = NULL, dots = NULL, keep_if_empty = FALSE) { - check_dots_empty0(...) args <- c(main_args, dots) if (is_empty(args) && !keep_if_empty) { @@ -274,7 +293,7 @@ add_curl_step <- function(steps, } names <- quote_name(names2(args)) - string <- vapply(args, is.character, logical(1L)) + string <- map_lgl(args, function(x) is.character(x) && !inherits(x, "AsIs")) values <- unlist(args) values <- ifelse(string, encode_string2(values), values) @@ -316,3 +335,22 @@ encode_string2 <- function(x) { names(out) <- names(x) out } + +cookies_parse <- function(x) { + pairs <- strsplit(x, "; ?")[[1]] + cookies <- parse_name_equals_value(pairs) + + if (length(cookies) == 0) { + return(NULL) + } + + out <- as.list(curl::curl_unescape(cookies)) + names(out) <- curl::curl_unescape(names(cookies)) + out +} + +idempotent_json <- function(old) { + args <- formals(req_body_json)[c("auto_unbox", "null", "digits")] + new <- exec(jsonlite::toJSON, jsonlite::parse_json(old), !!!args) + jsonlite::minify(old) == jsonlite::minify(new) +} diff --git a/R/headers.R b/R/headers.R index 6fdc5dd..c5c4057 100644 --- a/R/headers.R +++ b/R/headers.R @@ -1,15 +1,11 @@ as_headers <- function(x, error_call = caller_env()) { if (is.character(x) || is.raw(x)) { - headers <- curl::parse_headers(x) - headers <- headers[grepl(":", headers, fixed = TRUE)] + parsed <- curl::parse_headers(x) + valid <- parsed[grepl(":", parsed, fixed = TRUE)] + halves <- parse_in_half(valid, ":") - equals <- regexpr(":", headers, fixed = TRUE) - pieces <- regmatches(headers, equals, invert = TRUE) - - names <- map_chr(pieces, "[[", 1) - values <- as.list(trimws(map_chr(pieces, "[[", 2))) - - new_headers(set_names(values, names), error_call = error_call) + headers <- set_names(trimws(halves$right), halves$left) + new_headers(as.list(headers), error_call = error_call) } else if (is.list(x)) { new_headers(x, error_call = error_call) } else { diff --git a/R/httr2-package.R b/R/httr2-package.R index a88b7dd..be60360 100644 --- a/R/httr2-package.R +++ b/R/httr2-package.R @@ -11,6 +11,7 @@ NULL the <- new_environment() the$throttle <- list() +the$breaker <- new_environment() the$cache_throttle <- list() the$token_cache <- new_environment() the$last_response <- NULL diff --git a/R/is-online.R b/R/is-online.R new file mode 100644 index 0000000..8c970b4 --- /dev/null +++ b/R/is-online.R @@ -0,0 +1,12 @@ +#' Is your computer currently online? +#' +#' This function uses some cheap heuristics to determine if your computer is +#' currently online. It's a simple wrapper around [curl::has_internet()] +#' exported from httr2 for convenience. +#' +#' @export +#' @examples +#' is_online() +is_online <- function() { + curl::has_internet() +} diff --git a/R/iterate-responses.R b/R/iterate-responses.R index 09a8224..e8a2723 100644 --- a/R/iterate-responses.R +++ b/R/iterate-responses.R @@ -16,6 +16,11 @@ #' @param resps A list of responses (possibly including errors). #' @param resp_data A function that takes a response (`resp`) and #' returns the data found inside that response as a vector or data frame. +#' +#' NB: If you're using [resp_body_raw()], you're likely to want to wrap its +#' output in `list()` to avoid combining all the bodies into a single raw +#' vector, e.g. `resps |> resps_data(\(resp) list(resp_body_raw(resp)))`. +#' #' @examples #' reqs <- list( #' request(example_url()) |> req_url_path("/ip"), @@ -29,10 +34,14 @@ #' resps |> resps_successes() #' #' # collect all their data -#' resps |> resps_successes() |> resps_data(\(resp) resp_body_json(resp)) +#' resps |> +#' resps_successes() |> +#' resps_data(\(resp) resp_body_json(resp)) #' #' # find requests corresponding to failure responses -#' resps |> resps_failures() |> resps_requests() +#' resps |> +#' resps_failures() |> +#' resps_requests() resps_successes <- function(resps) { resps[resps_ok(resps)] } diff --git a/R/oauth-flow-auth-code.R b/R/oauth-flow-auth-code.R index f089305..7abda74 100644 --- a/R/oauth-flow-auth-code.R +++ b/R/oauth-flow-auth-code.R @@ -7,12 +7,14 @@ #' This flow is the most commonly used OAuth flow where the user #' opens a page in their browser, approves the access, and then returns to R. #' When possible, it redirects the browser back to a temporary local webserver -#' to capture the authorization code. When this is not possible (e.g. when +#' to capture the authorization code. When this is not possible (e.g., when #' running on a hosted platform like RStudio Server), provide a custom #' `redirect_uri` and httr2 will prompt the user to enter the code manually. #' #' Learn more about the overall OAuth authentication flow in -#' . +#' , and more about the motivations +#' behind this flow in +#' . #' #' # Security considerations #' @@ -20,17 +22,18 @@ #' applications (which are equivalent to R packages). `r rfc(8252)` spells out #' important considerations for native apps. Most importantly there's no way #' for native apps to keep secrets from their users. This means that the -#' server should either not require a `client_secret` (i.e. a public client -#' not an confidential client) or ensure that possession of the `client_secret` -#' doesn't bestow any meaningful rights. +#' server should either not require a `client_secret` (i.e. it should be a +#' public client and not a confidential client) or ensure that possession of +#' the `client_secret` doesn't grant any significant privileges. #' -#' Only modern APIs from the bigger players (Azure, Google, etc) explicitly -#' native apps. However, in most cases, even for older APIs, possessing the -#' `client_secret` gives you no ability to do anything harmful, so our -#' general principle is that it's fine to include it in an R package, as long -#' as it's mildly obfuscated to protect it from credential scraping. There's -#' no incentive to steal your client credentials if it takes less time to -#' create a new client than find your client secret. +#' Only modern APIs from major providers (like Azure and Google) explicitly +#' support native apps. However, in most cases, even for older APIs, possessing +#' the `client_secret` provides limited ability to perform harmful actions. +#' Therefore, our general principle is that it's acceptable to include it in an +#' R package, as long as it's mildly obfuscated to protect against credential +#' scraping attacks (which aim to acquire large numbers of client secrets by +#' scanning public sites like GitHub). The goal is to ensure that obtaining your +#' client credentials is more work than just creating a new client. #' #' @export #' @family OAuth flows @@ -362,7 +365,7 @@ oauth_flow_auth_code_listen <- function(redirect_uri = "http://localhost:1410") # https://www.w3.org/TR/html401/interact/forms.html#h-17.13.4.1 # Spaces are first replaced by + parse_form_urlencoded <- function(query) { - query <- query_parse(query) + query <- url_query_parse(query) query[] <- gsub("+", " ", query, fixed = TRUE) query } diff --git a/R/oauth-flow.R b/R/oauth-flow.R index 068ae87..4fcdcd4 100644 --- a/R/oauth-flow.R +++ b/R/oauth-flow.R @@ -12,6 +12,8 @@ oauth_flow_parse <- function(resp, source, error_call = caller_env()) { cli::cli_abort( "Failed to parse response from {.arg {source}} OAuth url.", parent = err, + resp = resp, + class = "httr2_oauth_parse", call = error_call ) } @@ -40,6 +42,8 @@ oauth_flow_parse <- function(resp, source, error_call = caller_env()) { "Failed to parse response from {.arg {source}} OAuth url.", "*" = "Did not contain {.code access_token}, {.code device_code}, or {.code error} field." ), + resp = resp, + class = "httr2_oauth_parse", call = error_call ) } diff --git a/R/progress-bars.R b/R/progress-bars.R index 31c2a73..59d46df 100644 --- a/R/progress-bars.R +++ b/R/progress-bars.R @@ -30,9 +30,6 @@ #' By default the same as `format`. #' * `name`: progress bar name. This is by default the empty string and it #' is displayed at the beginning of the progress bar. -#' * `show_after`: numeric scalar. Only show the progress bar after this -#' number of seconds. It overrides the `cli.progress_show_after` -#' global option. #' * `type`: progress bar type. Currently supported types are: #' * `iterator`: the default, a for loop or a mapping function, #' * `tasks`: a (typically small) number of tasks, diff --git a/R/req-auth-aws.R b/R/req-auth-aws.R index 7d32e69..c6713bb 100644 --- a/R/req-auth-aws.R +++ b/R/req-auth-aws.R @@ -129,7 +129,7 @@ aws_v4_signature <- function(method, CanonicalQueryString <- "" } else { sorted_query <- url$query[order(names(url$query))] - CanonicalQueryString <- query_build(CanonicalQueryString) + CanonicalQueryString <- url_query_build(CanonicalQueryString) } headers$host <- url$hostname @@ -186,7 +186,7 @@ aws_v4_signature <- function(method, credential <- file.path(aws_access_key_id, CredentialScope) Authorization <- paste0( - Algorithm, ",", + Algorithm, " ", "Credential=", credential, ",", "SignedHeaders=", SignedHeaders, ",", "Signature=", signature diff --git a/R/req-auth.R b/R/req-auth.R index 9811f0a..6459349 100644 --- a/R/req-auth.R +++ b/R/req-auth.R @@ -5,8 +5,8 @@ #' #' @inheritParams req_perform #' @param username User name. -#' @param password Password. You avoid entering the password directly when -#' calling this function as it will be captured by `.Rhistory`. Instead, +#' @param password Password. You should avoid entering the password directly +#' when calling this function as it will be captured by `.Rhistory`. Instead, #' leave it unset and the default behaviour will prompt you for it #' interactively. #' @returns A modified HTTP [request]. diff --git a/R/req-body.R b/R/req-body.R index f978673..6f63e67 100644 --- a/R/req-body.R +++ b/R/req-body.R @@ -137,7 +137,8 @@ req_body_json_modify <- function(req, ...) { #' data in the body. #' #' * For `req_body_form()`, the values must be strings (or things easily -#' coerced to strings); +#' coerced to strings). Vectors are convertd to strings using the +#' value of `.multi`. #' * For `req_body_multipart()` the values must be strings or objects #' produced by [curl::form_file()]/[curl::form_data()]. #' * For `req_body_json_modify()`, any simple data made from atomic vectors @@ -226,7 +227,7 @@ req_body_get <- function(req) { raw = req$body$data, form = { data <- unobfuscate(req$body$data) - query_build(data) + url_query_build(data) }, json = exec(jsonlite::toJSON, req$body$data, !!!req$body$params), cli::cli_abort("Unsupported request body type {.str {req$body$type}}.") diff --git a/R/req-headers.R b/R/req-headers.R index e943587..3d099cc 100644 --- a/R/req-headers.R +++ b/R/req-headers.R @@ -1,7 +1,12 @@ #' Modify request headers #' +#' @description #' `req_headers()` allows you to set the value of any header. #' +#' `req_headers_redacted()` is a variation that adds "redacted" headers, which +#' httr2 avoids printing on the console. This is good practice for +#' authentication headers to avoid accidentally leaking them in log files. +#' #' @param .req A [request]. #' @param ... <[`dynamic-dots`][rlang::dyn-dots]> Name-value pairs of headers #' and their values. @@ -46,13 +51,16 @@ #' # If you have headers in a list, use !!! #' headers <- list(HeaderOne = "one", HeaderTwo = "two") #' req |> -#' req_headers(!!!headers, HeaderThree = "three") |> -#' req_dry_run() -#' -#' # Use `.redact` to hide a header in the output -#' req |> -#' req_headers(Secret = "this-is-private", Public = "but-this-is-not", .redact = "Secret") |> +#' req_headers(!!!headers, HeaderThree = "three") |> #' req_dry_run() +#' +#' # Use `req_headers_redacted()`` to hide a header in the output +#' req_secret <- req |> +#' req_headers_redacted(Secret = "this-is-private") |> +#' req_headers(Public = "but-this-is-not") +#' +#' req_secret +#' req_secret |> req_dry_run() req_headers <- function(.req, ..., .redact = NULL) { check_request(.req) @@ -68,3 +76,12 @@ req_headers <- function(.req, ..., .redact = NULL) { .req } + +#' @export +#' @rdname req_headers +req_headers_redacted <- function(.req, ...) { + check_request(.req) + + dots <- list(...) + req_headers(.req, !!!dots, .redact = names(dots)) +} diff --git a/R/req-mock.R b/R/req-mock.R index 56d9be0..b13b286 100644 --- a/R/req-mock.R +++ b/R/req-mock.R @@ -40,7 +40,7 @@ with_mocked_responses <- function(mock, code) { #' @rdname with_mocked_responses #' @usage NULL with_mock <- function(mock, code) { - lifecycle::deprecate_warn("1.0.0", "with_mock()", "with_mocked_responses()") + lifecycle::deprecate_stop("1.1.0", "with_mock()", "with_mocked_responses()") with_mocked_responses(mock, code) } @@ -55,7 +55,7 @@ local_mocked_responses <- function(mock, env = caller_env()) { #' @rdname with_mocked_responses #' @usage NULL local_mock <- function(mock, env = caller_env()) { - lifecycle::deprecate_warn("1.0.0", "local_mock()", "local_mocked_responses()") + lifecycle::deprecate_warn("1.1.0", "local_mock()", "local_mocked_responses()") local_mocked_responses(mock, env) } diff --git a/R/req-options.R b/R/req-options.R index 08939d2..4e8fd04 100644 --- a/R/req-options.R +++ b/R/req-options.R @@ -207,7 +207,7 @@ verbose_header <- function(prefix, x, redact = TRUE, to_redact = NULL) { lines <- unlist(strsplit(x, "\r?\n", useBytes = TRUE)) for (line in lines) { - if (grepl(":", line, fixed = TRUE)) { + if (grepl("^[-a-zA-z0-9]+:", line)) { header <- headers_redact(as_headers(line), redact, to_redact = to_redact) cli::cat_line(prefix, cli::style_bold(names(header)), ": ", header) } else { diff --git a/R/req-perform-connection.R b/R/req-perform-connection.R index d8baf47..066aa68 100644 --- a/R/req-perform-connection.R +++ b/R/req-perform-connection.R @@ -1,9 +1,6 @@ - #' Perform a request and return a streaming connection #' #' @description -#' `r lifecycle::badge("experimental")` -#' #' Use `req_perform_connection()` to perform a request if you want to stream the #' response body. A response returned by `req_perform_connection()` includes a #' connection as the body. You can then use [resp_stream_raw()], @@ -16,10 +13,21 @@ #' than providing callbacks that the data is pushed to. This is useful if you #' want to do other work in between handling inputs from the stream. #' -#' @inheritParams req_perform_stream +#' @inheritParams req_perform #' @param blocking When retrieving data, should the connection block and wait #' for the desired information or immediately return what it has (possibly #' nothing)? +#' @param verbosity How much information to print? This is a wrapper +#' around [req_verbose()] that uses an integer to control verbosity: +#' +#' * `0`: no output +#' * `1`: show headers +#' * `2`: show headers and bodies as they're streamed +#' * `3`: show headers, bodies, curl status messages, and stream buffer +#' management +#' +#' Use [with_verbosity()] to control the verbosity of requests that +#' you can't affect directly. #' @export #' @examples #' req <- request(example_url()) |> @@ -33,10 +41,19 @@ #' #' # Always close the response when you're done #' close(resp) -req_perform_connection <- function(req, blocking = TRUE) { +#' +#' # You can loop until complete with resp_stream_is_complete() +#' resp <- req_perform_connection(req) +#' while (!resp_stream_is_complete(resp)) { +#' print(length(resp_stream_raw(resp, kb = 12))) +#' } +#' close(resp) +req_perform_connection <- function(req, blocking = TRUE, verbosity = NULL) { check_request(req) check_bool(blocking) + # verbosity checked in req_verbosity_connection + req <- req_verbosity_connection(req, verbosity %||% httr2_verbosity()) req <- auth_sign(req) req_prep <- req_prepare(req) handle <- req_handle(req_prep) @@ -49,6 +66,7 @@ req_perform_connection <- function(req, blocking = TRUE) { deadline <- Sys.time() + retry_max_seconds(req) resp <- NULL while (tries < max_tries && Sys.time() < deadline) { + retry_check_breaker(req, tries) sys_sleep(delay, "for retry backoff") if (!is.null(resp)) { @@ -58,6 +76,7 @@ req_perform_connection <- function(req, blocking = TRUE) { if (retry_is_transient(req, resp)) { tries <- tries + 1 + delay <- retry_after(req, resp, tries) signal(class = "httr2_retry", tries = tries, delay = delay) } else { @@ -78,6 +97,30 @@ req_perform_connection <- function(req, blocking = TRUE) { resp } +# Like req_verbosity() but we want to print the streaming body when it's +# requested not when curl actually receives it +req_verbosity_connection <- function(req, verbosity, error_call = caller_env()) { + if (!is_integerish(verbosity, n = 1) || verbosity < 0 || verbosity > 3) { + cli::cli_abort("{.arg verbosity} must 0, 1, 2, or 3.", call = error_call) + } + + req <- switch(verbosity + 1, + req, + req_verbose(req), + req_verbose(req, body_req = TRUE), + req_verbose(req, body_req = TRUE, info = TRUE) + ) + if (verbosity > 1) { + req <- req_policies( + req, + show_streaming_body = verbosity >= 2, + show_streaming_buffer = verbosity >= 3 + ) + } + req +} + + req_perform_connection1 <- function(req, handle, blocking = TRUE) { stream <- curl::curl(req$url, handle = handle) diff --git a/R/req-perform-stream.R b/R/req-perform-stream.R index 0f9cd42..9c6aa6d 100644 --- a/R/req-perform-stream.R +++ b/R/req-perform-stream.R @@ -1,6 +1,12 @@ #' Perform a request and handle data as it streams back #' #' @description +#' `r lifecycle::badge("superseded")` +#' +#' We now recommend [req_perform_connection()] since it has a considerably more +#' flexible interface. Unless I hear compelling reasons otherwise, I'm likely +#' to deprecate `req_perform_stream()` in a future release. +#' #' After preparing a request, call `req_perform_stream()` to perform the request #' and handle the result with a streaming callback. This is useful for #' streaming HTTP APIs where potentially the stream never ends. diff --git a/R/req-perform.R b/R/req-perform.R index 5eb8915..8da8896 100644 --- a/R/req-perform.R +++ b/R/req-perform.R @@ -107,6 +107,7 @@ req_perform <- function( delay <- 0 while (tries < max_tries && Sys.time() < deadline) { + retry_check_breaker(req, tries, error_call = error_call) sys_sleep(delay, "for retry backoff") n <- n + 1 @@ -161,6 +162,7 @@ handle_resp <- function(req, resp, error_call = caller_env()) { req_perform1 <- function(req, path = NULL, handle = NULL) { the$last_request <- req the$last_response <- NULL + signal(class = "httr2_perform") if (!is.null(path)) { res <- curl::curl_fetch_disk(req$url, path, handle) @@ -229,6 +231,10 @@ last_request <- function() { #' works by sending the real HTTP request to a local webserver, thanks to #' the magic of [curl::curl_echo()]. #' +#' ## Limitations +#' +#' * The `Host` header is not respected. +#' #' @inheritParams req_verbose #' @param quiet If `TRUE` doesn't print anything. #' @returns Invisibly, a list containing information about the request, diff --git a/R/req-promise.R b/R/req-promise.R index f157e13..0c0d216 100644 --- a/R/req-promise.R +++ b/R/req-promise.R @@ -69,9 +69,13 @@ #' } req_perform_promise <- function(req, path = NULL, - pool = NULL) { + pool = NULL, + verbosity = NULL) { check_installed(c("promises", "later")) + + check_request(req) check_string(path, allow_null = TRUE) + verbosity <- verbosity %||% httr2_verbosity() if (missing(pool)) { if (!identical(later::current_loop(), later::global_loop())) { @@ -80,7 +84,14 @@ req_perform_promise <- function(req, i = "Do you need {.code pool = curl::new_pool()}?" )) } + } else { + if (!is.null(pool) && !inherits(pool, "curl_multi")) { + stop_input_type(pool, "a {curl} pool", allow_null = TRUE) + } } + # verbosity checked by req_verbosity + + req <- req_verbosity(req, verbosity) promises::promise( function(resolve, reject) { diff --git a/R/req-retries.R b/R/req-retries.R index 202391a..3049cd9 100644 --- a/R/req-retries.R +++ b/R/req-retries.R @@ -1,55 +1,62 @@ -#' Control when a request will retry, and how long it will wait between tries +#' Automatically retry a request on failure #' #' @description -#' `req_retry()` alters [req_perform()] so that it will automatically retry -#' in the case of failure. To activate it, you must specify either the total -#' number of requests to make with `max_tries` or the total amount of time -#' to spend with `max_seconds`. Then `req_perform()` will retry if the error is -#' "transient", i.e. it's an HTTP error that can be resolved by waiting. By -#' default, 429 and 503 statuses are treated as transient, but if the API you -#' are wrapping has other transient status codes (or conveys transient-ness -#' with some other property of the response), you can override the default -#' with `is_transient`. +#' `req_retry()` allows [req_perform()] to automatically retry failing +#' requests. It's particularly important for APIs with rate limiting, but can +#' also be useful when dealing with flaky servers. #' -#' Additionally, if you set `retry_on_failure = TRUE`, the request will retry -#' if either the HTTP request or HTTP response doesn't complete successfully +#' By default, `req_perform()` will retry if the response is a 429 +#' ("too many requests", often used for rate limiting) or 503 +#' ("service unavailable"). If the API you are wrapping has other transient +#' status codes (or conveys transience with some other property of the +#' response), you can override the default with `is_transient`. And +#' if you set `retry_on_failure = TRUE`, the request will retry +#' if either the HTTP request or HTTP response doesn't complete successfully, #' leading to an error from curl, the lower-level library that httr2 uses to -#' perform HTTP request. This occurs, for example, if your wifi is down. +#' perform HTTP requests. This occurs, for example, if your Wi-Fi is down. +#' +#' ## Delay #' #' It's a bad idea to immediately retry a request, so `req_perform()` will #' wait a little before trying again: #' #' * If the response contains the `Retry-After` header, httr2 will wait the #' amount of time it specifies. If the API you are wrapping conveys this -#' information with a different header (or other property of the response) -#' you can override the default behaviour with `retry_after`. +#' information with a different header (or other property of the response), +#' you can override the default behavior with `retry_after`. #' #' * Otherwise, httr2 will use "truncated exponential backoff with full -#' jitter", i.e. it will wait a random amount of time between one second and -#' `2 ^ tries` seconds, capped to at most 60 seconds. In other words, it +#' jitter", i.e., it will wait a random amount of time between one second and +#' `2 ^ tries` seconds, capped at a maximum of 60 seconds. In other words, it #' waits `runif(1, 1, 2)` seconds after the first failure, `runif(1, 1, 4)` #' after the second, `runif(1, 1, 8)` after the third, and so on. If you'd #' prefer a different strategy, you can override the default with `backoff`. #' #' @inheritParams req_perform -#' @param max_tries,max_seconds Cap the maximum number of attempts with -#' `max_tries` or the total elapsed time from the first request with -#' `max_seconds`. If neither option is supplied (the default), [req_perform()] -#' will not retry. +#' @param max_tries,max_seconds Cap the maximum number of attempts +#' (`max_tries`), the total elapsed time from the first request +#' (`max_seconds`), or both. #' -#' `max_tries` is the total number of attempts make, so this should always -#' be greater than one.` +#' `max_tries` is the total number of attempts made, so this should always +#' be greater than one. #' @param is_transient A predicate function that takes a single argument #' (the response) and returns `TRUE` or `FALSE` specifying whether or not #' the response represents a transient error. #' @param retry_on_failure Treat low-level failures as if they are -#' transient errors, and can be retried. +#' transient errors that can be retried. #' @param backoff A function that takes a single argument (the number of failed #' attempts so far) and returns the number of seconds to wait. #' @param after A function that takes a single argument (the response) and -#' returns either a number of seconds to wait or `NULL`, which indicates -#' that a precise wait time is not available that the `backoff` strategy -#' should be used instead.. +#' returns either a number of seconds to wait or `NA`. `NA` indicates +#' that a precise wait time is not available and that the `backoff` strategy +#' should be used instead. +#' @param failure_threshold,failure_timeout,failure_realm +#' Set `failure_threshold` to activate "circuit breaking" where if a request +#' continues to fail after `failure_threshold` times, cause the request to +#' error until a timeout of `failure_timeout` seconds has elapsed. This +#' timeout will persist across all requests with the same `failure_realm` +#' (which defaults to the hostname of the request) and is intended to detect +#' failing servers without needing to wait each time. #' @returns A modified HTTP [request]. #' @export #' @seealso [req_throttle()] if the API has a rate-limit but doesn't expose @@ -61,7 +68,7 @@ #' #' # use a constant 10s delay after every failure #' request("http://example.com") |> -#' req_retry(backoff = ~10) +#' req_retry(backoff = \(resp) 10) #' #' # When rate-limited, GitHub's API returns a 403 with #' # `X-RateLimit-Remaining: 0` and an Unix time stored in the @@ -85,10 +92,22 @@ req_retry <- function(req, retry_on_failure = FALSE, is_transient = NULL, backoff = NULL, - after = NULL) { + after = NULL, + failure_threshold = Inf, + failure_timeout = 30, + failure_realm = NULL) { + check_request(req) - check_number_whole(max_tries, min = 2, allow_null = TRUE) + check_number_whole(max_tries, min = 1, allow_null = TRUE) check_number_whole(max_seconds, min = 0, allow_null = TRUE) + check_number_whole(failure_threshold, min = 1, allow_infinite = TRUE) + check_number_whole(failure_timeout, min = 1) + + if (is.null(max_tries) && is.null(max_seconds)) { + max_tries <- 2 + cli::cli_inform("Setting {.code max_tries = 2}.") + } + check_bool(retry_on_failure) req_policies(req, @@ -97,7 +116,10 @@ req_retry <- function(req, retry_on_failure = retry_on_failure, retry_is_transient = as_callback(is_transient, 1, "is_transient"), retry_backoff = as_callback(backoff, 1, "backoff"), - retry_after = as_callback(after, 1, "after") + retry_after = as_callback(after, 1, "after"), + retry_failure_threshold = failure_threshold, + retry_failure_timeout = failure_timeout, + retry_realm = failure_realm %||% url_parse(req$url)$hostname ) } @@ -110,6 +132,38 @@ retry_max_seconds <- function(req) { req$policies$retry_max_wait %||% Inf } +retry_check_breaker <- function(req, i, error_call = caller_env()) { + realm <- req$policies$retry_realm + if (is.null(realm)) { + return(invisible()) + } + + now <- unix_time() + if (env_has(the$breaker, realm)) { + triggered <- the$breaker[[realm]] + } else if (i > req$policies$retry_failure_threshold) { + the$breaker[[realm]] <- triggered <- now + } else { + return(invisible()) + } + + remaining <- req$policies$retry_failure_timeout - (now - triggered) + if (remaining <= 0) { + env_unbind(the$breaker, realm) + } else { + cli::cli_abort( + c( + "Request failures have exceeded the threshold for realm {.str {realm}}.", + i = "The server behind {.str {realm}} is likely still overloaded or down.", + i = "Wait {remaining} seconds before retrying." + + ), + call = error_call, + class = "httr2_breaker" + ) + } +} + retry_is_transient <- function(req, resp) { if (is_error(resp)) { return(req$policies$retry_on_failure %||% FALSE) diff --git a/R/req-url.R b/R/req-url.R index d168e42..d060ee8 100644 --- a/R/req-url.R +++ b/R/req-url.R @@ -1,13 +1,19 @@ #' Modify request URL #' #' @description -#' * `req_url()` replaces the entire url -#' * `req_url_query()` modifies the components of the query -#' * `req_url_path()` modifies the path -#' * `req_url_path_append()` adds to the path +#' * `req_url()` replaces the entire URL. +#' * `req_url_relative()` navigates to a relative URL. +#' * `req_url_query()` modifies individual query components. +#' * `req_url_path()` modifies just the path. +#' * `req_url_path_append()` adds to the path. #' +#' @seealso +#' * To modify a URL without creating a request, see [url_modify()] and +#' friends. +#' * To use a template like `GET /user/{user}`, see [req_template()]. #' @inheritParams req_perform -#' @param url New URL; completely replaces existing. +#' @param url A new URL; either an absolute URL for `req_url()` or a +#' relative URL for `req_url_relative()`. #' @param ... For `req_url_query()`: <[`dynamic-dots`][rlang::dyn-dots]> #' Name-value pairs that define query parameters. Each value must be either #' an atomic vector or `NULL` (which removes the corresponding parameters). @@ -18,7 +24,14 @@ #' @returns A modified HTTP [request]. #' @export #' @examples +#' # Change complete url #' req <- request("http://example.com") +#' req |> req_url("http://google.com") +#' +#' # Use a relative url +#' req <- request("http://example.com/a/b/c") +#' req |> req_url_relative("..") +#' req |> req_url_relative("/d/e/f") #' #' # Change url components #' req |> @@ -27,9 +40,11 @@ #' req_url_path_append("search.html") |> #' req_url_query(q = "the cool ice") #' -#' # Change complete url -#' req |> -#' req_url("http://google.com") +#' # Modify individual query parameters +#' req <- request("http://example.com?a=1&b=2") +#' req |> req_url_query(a = 10) +#' req |> req_url_query(a = NULL) +#' req |> req_url_query(c = 3) #' #' # Use .multi to control what happens with vector parameters: #' req |> req_url_query(id = 100:105, .multi = "comma") @@ -49,26 +64,21 @@ req_url <- function(req, url) { #' @export #' @rdname req_url -#' @param .multi Controls what happens when an element of `...` is a vector -#' containing multiple values: -#' -#' * `"error"`, the default, throws an error. -#' * `"comma"`, separates values with a `,`, e.g. `?x=1,2`. -#' * `"pipe"`, separates values with a `|`, e.g. `?x=1|2`. -#' * `"explode"`, turns each element into its own parameter, e.g. `?x=1&x=2`. -#' -#' If none of these functions work, you can alternatively supply a function -#' that takes a character vector and returns a string. +req_url_relative <- function(req, url) { + check_request(req) + req_url(req, url_modify_relative(req$url, url)) +} + +#' @export +#' @rdname req_url +#' @inheritParams url_modify_query req_url_query <- function(.req, ..., - .multi = c("error", "comma", "pipe", "explode")) { + .multi = c("error", "comma", "pipe", "explode"), + .space = c("percent", "form")) { check_request(.req) - - dots <- multi_dots(..., .multi = .multi) - - url <- url_parse(.req$url) - url$query <- modify_list(url$query, !!!dots) - req_url(.req, url_build(url)) + url <- url_modify_query(.req$url, ..., .multi = .multi, .space = .space) + req_url(.req, url) } #' @export diff --git a/R/resp-headers.R b/R/resp-headers.R index 0560fc3..d76f18b 100644 --- a/R/resp-headers.R +++ b/R/resp-headers.R @@ -173,7 +173,9 @@ resp_link_url <- function(resp, rel) { return() } - links <- parse_link(resp_header(resp, "Link")) + headers <- resp_headers(resp) + link_headers <- headers[names(headers) == "Link"] + links <- unlist(lapply(link_headers, parse_link), recursive = FALSE) sel <- map_lgl(links, ~ .$rel == rel) if (sum(sel) != 1L) { return() diff --git a/R/resp-request.R b/R/resp-request.R new file mode 100644 index 0000000..d9e346f --- /dev/null +++ b/R/resp-request.R @@ -0,0 +1,16 @@ +#' Find the request responsible for a response +#' +#' To make debugging easier, httr2 includes the request that was used to +#' generate every response. You can use this function to access it. +#' +#' @inheritParams resp_header +#' @export +#' @examples +#' req <- request(example_url()) +#' resp <- req_perform(req) +#' resp_request(resp) +resp_request <- function(resp) { + check_response(resp) + + resp$request +} diff --git a/R/resp-stream-aws.R b/R/resp-stream-aws.R index 7e7f59c..40c9954 100644 --- a/R/resp-stream-aws.R +++ b/R/resp-stream-aws.R @@ -9,11 +9,21 @@ resp_stream_aws <- function(resp, max_size = Inf) { include_trailer = FALSE ) - if (!is.null(event_bytes)) { - parse_aws_event(event_bytes) - } else { - return(NULL) + if (is.null(event_bytes)) { + return() + } + + event <- parse_aws_event(event_bytes) + if (resp_stream_show_body(resp)) { + # Emit header + for (key in names(event$headers)) { + log_stream(cli::style_bold(key), ": ", event$headers[[key]]) + } + # Emit body + log_stream(jsonlite::toJSON(event$body, auto_unbox = TRUE, pretty = TRUE)) + cli::cat_line() } + event } find_aws_event_boundary <- function(buffer) { @@ -57,15 +67,15 @@ parse_aws_event <- function(bytes) { # headers headers <- list() - while(i <= 12 + header_length) { + while (i <= 12 + header_length) { name_length <- as.integer(read_bytes(1)) name <- rawToChar(read_bytes(name_length)) type <- as.integer(read_bytes(1)) delayedAssign("length", parse_int(read_bytes(2))) value <- switch(type_enum(type), - 'TRUE' = TRUE, - 'FALSE' = FALSE, + "TRUE" = TRUE, + "FALSE" = FALSE, BYTE = parse_int(read_bytes(1)), SHORT = parse_int(read_bytes(2)), INTEGER = parse_int(read_bytes(4)), @@ -95,7 +105,7 @@ parse_aws_event <- function(bytes) { # Helpers ---------------------------------------------------------------- parse_int <- function(x) { - sum(as.integer(x) * 256 ^ rev(seq_along(x) - 1)) + sum(as.integer(x) * 256^rev(seq_along(x) - 1)) } parse_int64 <- function(x) { diff --git a/R/resp-stream.R b/R/resp-stream.R index 20f632c..7b2c08c 100644 --- a/R/resp-stream.R +++ b/R/resp-stream.R @@ -1,8 +1,6 @@ #' Read a streaming body a chunk at a time #' #' @description -#' `r lifecycle::badge("experimental")` -#' #' * `resp_stream_raw()` retrieves bytes (`raw` vectors). #' * `resp_stream_lines()` retrieves lines of text (`character` vectors). #' * `resp_stream_sse()` retrieves a single [server-sent @@ -10,6 +8,9 @@ #' * `resp_stream_aws()` retrieves a single event from an AWS stream #' (i.e. mime type `application/vnd.amazon.eventstream``). #' +#' Use `resp_stream_is_complete()` to determine if there is further data +#' waiting on the stream. +#' #' @returns #' * `resp_stream_raw()`: a raw vector. #' * `resp_stream_lines()`: a character vector. @@ -25,11 +26,33 @@ #' @param resp,con A streaming [response] created by [req_perform_connection()]. #' @param kb How many kilobytes (1024 bytes) of data to read. #' @order 1 +#' @examples +#' req <- request(example_url()) |> +#' req_template("GET /stream/:n", n = 5) +#' +#' con <- req |> req_perform_connection() +#' while (!resp_stream_is_complete(con)) { +#' lines <- con |> resp_stream_lines(2) +#' cat(length(lines), " lines received\n", sep = "") +#' } +#' close(con) +#' +#' # You can also see what's happening by setting verbosity +#' con <- req |> req_perform_connection(verbosity = 2) +#' while (!resp_stream_is_complete(con)) { +#' lines <- con |> resp_stream_lines(2) +#' } +#' close(con) resp_stream_raw <- function(resp, kb = 32) { check_streaming_response(resp) conn <- resp$body - readBin(conn, raw(), kb * 1024) + out <- readBin(conn, raw(), kb * 1024) + if (resp_stream_show_body(resp)) { + log_stream("Streamed ", length(out), " bytes") + cli::cat_line() + } + out } #' @export @@ -56,12 +79,18 @@ resp_stream_lines <- function(resp, lines = 1, max_size = Inf, warn = TRUE) { line <- resp_stream_oneline(resp, max_size, warn, encoding) if (length(line) == 0) { # No more data, either because EOF or req_perform_connection(blocking=FALSE). - # Either way, return what we have - return(lines_read) + # Either way we're done + break } lines_read <- c(lines_read, line) lines <- lines - 1 } + + if (resp_stream_show_body(resp)) { + log_stream(lines_read) + cli::cat_line() + } + lines_read } @@ -73,11 +102,25 @@ resp_stream_lines <- function(resp, lines = 1, max_size = Inf, warn = TRUE) { #' @order 1 resp_stream_sse <- function(resp, max_size = Inf) { event_bytes <- resp_boundary_pushback(resp, max_size, find_event_boundary, include_trailer = FALSE) - if (!is.null(event_bytes)) { - parse_event(event_bytes) - } else { - return(NULL) + if (is.null(event_bytes)) { + return() } + + event <- parse_event(event_bytes) + if (resp_stream_show_body(resp)) { + for (key in names(event)) { + log_stream(cli::style_bold(key), ": ", pretty_json(event[[key]])) + } + cli::cat_line() + } + event +} + +#' @export +#' @rdname resp_stream_raw +resp_stream_is_complete <- function(resp) { + check_response(resp) + length(resp$cache$push_back) == 0 && !isIncomplete(resp$body) } #' @export @@ -178,16 +221,16 @@ find_event_boundary <- function(buffer) { boundary_end <- which( (left1 == 0x0A & buffer == 0x0A) | # \n\n - (left1 == 0x0D & buffer == 0x0D) | # \r\r - (left3 == 0x0D & left2 == 0x0A & left1 == 0x0D & buffer == 0x0A) # \r\n\r\n + (left1 == 0x0D & buffer == 0x0D) | # \r\r + (left3 == 0x0D & left2 == 0x0A & left1 == 0x0D & buffer == 0x0A) # \r\n\r\n ) if (length(boundary_end) == 0) { - return(NULL) # No event boundary found + return(NULL) # No event boundary found } - boundary_end <- boundary_end[1] # Take the first occurrence - split_at <- boundary_end + 1 # Split at one after the boundary + boundary_end <- boundary_end[1] # Take the first occurrence + split_at <- boundary_end + 1 # Split at one after the boundary split_at } @@ -208,7 +251,7 @@ split_buffer <- function(buffer, split_at) { # the vector # @param include_trailer If TRUE, at the end of the response, if there are # bytes after the last boundary, then return those bytes; if FALSE, then those -# bytes are silently discarded. +# bytes are discarded with a warning. resp_boundary_pushback <- function(resp, max_size, boundary_func, include_trailer) { check_streaming_response(resp) check_number_whole(max_size, min = 1, allow_infinite = TRUE) @@ -219,8 +262,12 @@ resp_boundary_pushback <- function(resp, max_size, boundary_func, include_traile buffer <- resp$cache$push_back %||% raw() resp$cache$push_back <- raw() - print_buffer <- function(buf, label) { - # cat(label, ":", paste(sprintf("%02X", as.integer(buf)), collapse = " "), "\n", file = stderr()) + if (resp_stream_show_buffer(resp)) { + print_buffer <- function(buf, label) { + cli::cat_line(" * ", label, ": ", paste(as.character(buf), collapse = " ")) + } + } else { + print_buffer <- function(buf, label) {} } # Read chunks until we find an event or reach the end of input @@ -252,24 +299,27 @@ resp_boundary_pushback <- function(resp, max_size, boundary_func, include_traile # one extra byte so we know to error. n = min(chunk_size, max_size - length(buffer) + 1) ) - print_buffer(chunk, "Received chunk") - # If we've reached the end of input, store the buffer and return NULL if (length(chunk) == 0) { if (!isIncomplete(resp$body)) { # We've truly reached the end of the connection; no more data is coming - if (include_trailer && length(buffer) > 0) { - return(buffer) - } else { + if (length(buffer) == 0) { return(NULL) + } else { + if (include_trailer) { + return(buffer) + } else { + cli::cli_warn("Premature end of input; ignoring final partial chunk") + return(NULL) + } } + } else { + # More data might come later; store the buffer and return NULL + print_buffer(buffer, "Storing incomplete buffer") + resp$cache$push_back <- buffer + return(NULL) } - - # More data might come later - print_buffer(buffer, "Storing incomplete buffer") - resp$cache$push_back <- buffer - return(NULL) } # More data was received; combine it with existing buffer and continue the @@ -313,7 +363,6 @@ parse_event <- function(event_data) { check_streaming_response <- function(resp, arg = caller_arg(resp), call = caller_env()) { - check_response(resp, arg = arg, call = call) if (resp_body_type(resp) != "stream") { @@ -344,3 +393,10 @@ isValid <- function(con) { error = function(cnd) FALSE ) } + +resp_stream_show_body <- function(resp) { + resp$request$policies$show_streaming_body %||% FALSE +} +resp_stream_show_buffer <- function(resp) { + resp$request$policies$show_streaming_buffer %||% FALSE +} diff --git a/R/sequential.R b/R/sequential.R index cda5389..db80bd6 100644 --- a/R/sequential.R +++ b/R/sequential.R @@ -6,16 +6,19 @@ #' #' @param reqs A list of [request]s. #' @param paths An optional character vector of paths, if you want to download -#' the request bodies to disk. If supplied, must be the same length as `reqs`. +#' the response bodies to disk. If supplied, must be the same length as +#' `reqs`. #' @param on_error What should happen if one of the requests fails? #' #' * `stop`, the default: stop iterating with an error. #' * `return`: stop iterating, returning all the successful responses #' received so far, as well as an error object for the failed request. #' * `continue`: continue iterating, recording errors in the result. -#' @param progress Display a progress bar? Use `TRUE` to turn on a basic -#' progress bar, use a string to give it a name, or see [progress_bars] to -#' customise it in other ways. +#' @param progress Display a progress bar for the status of all requests? Use +#' `TRUE` to turn on a basic progress bar, use a string to give it a name, +#' or see [progress_bars] to customize it in other ways. Not compatible with +#' [req_progress()], as httr2 can only display a single progress bar at a +#' time. #' @return #' A list, the same length as `reqs`, containing [response]s and possibly #' error objects, if `on_error` is `"return"` or `"continue"` and one of the diff --git a/R/test.R b/R/test.R index 9f973db..7622338 100644 --- a/R/test.R +++ b/R/test.R @@ -50,7 +50,7 @@ example_url <- function() { env_cache(the, "test_app", webfakes::new_app_process( app, - opts = webfakes::server_opts(num_threads = 2) + opts = webfakes::server_opts(num_threads = 6, enable_keep_alive = TRUE) ) ) the$test_app$url() diff --git a/R/url.R b/R/url.R index 24680ab..bf2f8ac 100644 --- a/R/url.R +++ b/R/url.R @@ -1,75 +1,203 @@ -#' Parse and build URLs +#' Parse a URL into its component pieces #' -#' `url_parse()` parses a URL into its component pieces; `url_build()` does -#' the reverse, converting a list of pieces into a string URL. See `r rfc(3986)` -#' for the details of the parsing algorithm. +#' `url_parse()` parses a URL into its component parts, powered by +#' [curl::curl_parse_url()]. The parsing algorithm follows the specifications +#' detailed in `r rfc(3986)`. #' -#' @param url For `url_parse()` a string to parse into a URL; -#' for `url_build()` a URL to turn back into a string. -#' @returns -#' * `url_build()` returns a string. -#' * `url_parse()` returns a URL: a S3 list with class `httr2_url` -#' and elements `scheme`, `hostname`, `port`, `path`, `fragment`, `query`, -#' `username`, `password`. +#' @param url A string containing the URL to parse. +#' @param base_url Use this as a parent, if `url` is a relative URL. +#' @returns An S3 object of class `httr2_url` with the following components: +#' `scheme`, `hostname`, `username`, `password`, `port`, `path`, `query`, and +#' `fragment`. #' @export +#' @family URL manipulation #' @examples #' url_parse("http://google.com/") #' url_parse("http://google.com:80/") #' url_parse("http://google.com:80/?a=1&b=2") #' url_parse("http://username@google.com:80/path;test?a=1&b=2#40") #' -#' url <- url_parse("http://google.com/") -#' url$port <- 80 -#' url$hostname <- "example.com" -#' url$query <- list(a = 1, b = 2, c = 3) -#' url_build(url) -url_parse <- function(url) { +#' # You can parse a relative URL if you also provide a base url +#' url_parse("foo", "http://google.com/bar/") +#' url_parse("..", "http://google.com/bar/") +url_parse <- function(url, base_url = NULL) { check_string(url) + check_string(base_url, allow_null = TRUE) - # https://datatracker.ietf.org/doc/html/rfc3986#appendix-B - pieces <- parse_match(url, "^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\\?([^#]*))?(#(.*))?") + curl <- curl::curl_parse_url(url, baseurl = base_url, decode = FALSE) - scheme <- pieces[[2]] - authority <- pieces[[4]] - path <- pieces[[5]] - query <- pieces[[7]] - if (!is.null(query)) { - query <- query_parse(query) + parsed <- list( + scheme = curl$scheme, + hostname = curl$host, + username = curl$user, + password = curl$password, + port = curl$port, + path = curl$path, + query = if (length(curl$params)) as.list(curl$params), + fragment = curl$fragment + ) + class(parsed) <- "httr2_url" + parsed +} + +#' Modify a URL +#' +#' @description +#' Use `url_modify()` to modify any component of the URL, +#' `url_modify_relative()` to modify with a relative URL, +#' or `url_modify_query()` to modify individual query parameters. +#' +#' For `url_modify()`, components that aren't specified in the +#' function call will be left as is; components set to `NULL` will be removed, +#' and all other values will be updated. Note that removing `scheme` or +#' `hostname` will create a relative URL. +#' +#' @param url,.url A string or [parsed URL][url_parse()]. +#' @param scheme The scheme, typically either `http` or `https`. +#' @param hostname The hostname, e.g., `www.google.com` or `posit.co`. +#' @param username,password Username and password to embed in the URL. +#' Not generally recommended but needed for some legacy applications. +#' @param port An integer port number. +#' @param path The path, e.g., `/search`. Paths must start with `/`, so this +#' will be automatically added if omitted. +#' @param query Either a query string or a named list of query components. +#' @param fragment The fragment, e.g., `#section-1`. +#' @return An object of the same type as `url`. +#' @export +#' @family URL manipulation +#' @examples +#' url_modify("http://hadley.nz", path = "about") +#' url_modify("http://hadley.nz", scheme = "https") +#' url_modify("http://hadley.nz/abc", path = "/cde") +#' url_modify("http://hadley.nz/abc", path = "") +#' url_modify("http://hadley.nz?a=1", query = "b=2") +#' url_modify("http://hadley.nz?a=1", query = list(c = 3)) +#' +#' url_modify_query("http://hadley.nz?a=1&b=2", c = 3) +#' url_modify_query("http://hadley.nz?a=1&b=2", b = NULL) +#' url_modify_query("http://hadley.nz?a=1&b=2", a = 100) +#' +#' url_modify_relative("http://hadley.nz/a/b/c.html", "/d.html") +#' url_modify_relative("http://hadley.nz/a/b/c.html", "d.html") +#' url_modify_relative("http://hadley.nz/a/b/c.html", "../d.html") +url_modify <- function(url, + scheme = as_is, + hostname = as_is, + username = as_is, + password = as_is, + port = as_is, + path = as_is, + query = as_is, + fragment = as_is) { + if (!is_string(url) && !is_url(url)) { + stop_input_type(url, "a string or parsed URL") + } + string_url <- is_string(url) + if (string_url) { + url <- url_parse(url) } - fragment <- pieces[[9]] - # https://datatracker.ietf.org/doc/html/rfc3986#section-3.2 - pieces <- parse_match(authority %||% "", "^(([^@]+)@)?([^:]+)?(:([^#]+))?") + if (!leave_as_is(scheme)) check_string(scheme, allow_null = TRUE) + if (!leave_as_is(hostname)) check_string(hostname, allow_null = TRUE) + if (!leave_as_is(username)) check_string(username, allow_null = TRUE) + if (!leave_as_is(password)) check_string(password, allow_null = TRUE) + if (!leave_as_is(port)) check_number_whole(port, min = 1, allow_null = TRUE) + if (!leave_as_is(path)) check_string(path, allow_null = TRUE) + if (!leave_as_is(fragment)) check_string(fragment, allow_null = TRUE) - userinfo <- pieces[[2]] - if (!is.null(userinfo)) { - userinfo <- parse_in_half(userinfo, ":") - if (userinfo$right == "") { - userinfo$right <- NULL + if (is_string(query)) { + query <- url_query_parse(query) + } else if (is_named_list(query)) { + for (nm in names(query)) { + check_query_param(query[[nm]], paste0("query$", nm)) } + } else if (!is.null(query) && !leave_as_is(query)) { + stop_input_type(query, "a character vector, named list, or NULL") } - hostname <- pieces[[3]] - port <- pieces[[5]] - - structure( - list( - scheme = scheme, - hostname = hostname, - username = userinfo$left, - password = userinfo$right, - port = port, - path = path, - query = query, - fragment = fragment - ), - class = "httr2_url" + + new <- list( + scheme = scheme, + hostname = hostname, + username = username, + password = password, + port = port, + path = path, + query = query, + fragment = fragment ) + new <- new[!map_lgl(new, leave_as_is)] + url[names(new)] <- new + + if (string_url) { + url_build(url) + } else { + url + } +} + +as_is <- quote(as_is) +leave_as_is <- function(x) identical(x, as_is) + +#' @export +#' @rdname url_modify +#' @param relative_url A relative URL to append to the base URL. +url_modify_relative <- function(url, relative_url) { + string_url <- is_string(url) + if (!string_url) { + url <- url_build(url) + } + + new_url <- url_parse(relative_url, base_url = url) + + if (string_url) { + url_build(new_url) + } else { + new_url + } } -url_modify <- function(url, ..., error_call = caller_env()) { - url <- url_parse(url) - url <- modify_list(url, ..., error_call = error_call) - url_build(url) +#' @export +#' @rdname url_modify +#' @param ... <[`dynamic-dots`][rlang::dyn-dots]> +#' Name-value pairs that define query parameters. Each value must be either +#' an atomic vector or `NULL` (which removes the corresponding parameters). +#' If you want to opt out of escaping, wrap strings in `I()`. +#' @param .multi Controls what happens when a value is a vector: +#' +#' * `"error"`, the default, throws an error. +#' * `"comma"`, separates values with a `,`, e.g. `?x=1,2`. +#' * `"pipe"`, separates values with a `|`, e.g. `?x=1|2`. +#' * `"explode"`, turns each element into its own parameter, e.g. `?x=1&x=2` +#' +#' If none of these options work for your needs, you can instead supply a +#' function that takes a character vector of argument values and returns a +#' a single string. +#' @param .space How should spaces in query params be escaped? The default, +#' "percent", uses standard percent encoding (i.e. `%20`), but you can opt-in +#' to "form" encoding, which uses `+` instead. +url_modify_query <- function( + .url, + ..., + .multi = c("error", "comma", "pipe", "explode"), + .space = c("percent", "form")) { + if (!is_string(.url) && !is_url(.url)) { + stop_input_type(.url, "a string or parsed URL") + } + string_url <- is_string(.url) + if (string_url) { + .url <- url_parse(.url) + } + + new_query <- multi_dots(..., .multi = .multi, .space = .space) + if (length(new_query) > 0) { + .url$query <- modify_list(.url$query, !!!new_query) + } + + if (string_url) { + url_build(.url) + } else { + .url + } } is_url <- function(x) inherits(x, "httr2_url") @@ -109,11 +237,21 @@ print.httr2_url <- function(x, ...) { invisible(x) } +#' Build a string from a URL object +#' +#' This is the inverse of [url_parse()], taking a parsed URL object and +#' turning it back into a string. +#' +#' @param url An URL object created by [url_parse]. +#' @family URL manipulation #' @export -#' @rdname url_parse url_build <- function(url) { + if (!is_url(url)) { + stop_input_type(url, "a parsed URL") + } + if (!is.null(url$query)) { - query <- query_build(url$query) + query <- url_query_build(url$query) } else { query <- NULL } @@ -137,7 +275,7 @@ url_build <- function(url) { authority <- NULL } - if (!is.null(url$path) && !startsWith(url$path, "/")) { + if (is.null(url$path) || !startsWith(url$path, "/")) { url$path <- paste0("/", url$path) } @@ -151,9 +289,23 @@ url_build <- function(url) { ) } -query_parse <- function(x) { - x <- gsub("^\\?", "", x) # strip leading ?, if present - params <- parse_name_equals_value(parse_delim(x, "&")) +#' Parse query parameters and/or build a string +#' +#' `url_query_parse()` parses a query string into a named list; +#' `url_query_build()` builds a query string from a named list. +#' +#' @param query A string, when parsing; a named list when building. +#' @export +#' @examples +#' str(url_query_parse("a=1&b=2")) +#' +#' url_query_build(list(x = 1, y = "z")) +#' url_query_build(list(x = 1, y = 1:2), .multi = "explode") +url_query_parse <- function(query) { + check_string(query) + + query <- gsub("^\\?", "", query) # strip leading ?, if present + params <- parse_name_equals_value(parse_delim(query, "&")) if (length(params) == 0) { return(NULL) @@ -164,12 +316,20 @@ query_parse <- function(x) { out } -query_build <- function(x, error_call = caller_env()) { - elements_build(x, "Query", "&", error_call = error_call) +#' @export +#' @rdname url_query_parse +#' @inheritParams url_modify_query +url_query_build <- function(query, .multi = c("error", "comma", "pipe", "explode")) { + if (!is_named_list(query)) { + stop_input_type(query, "a named list") + } + + query <- multi_dots(!!!query, .multi = .multi, error_arg = "query") + elements_build(query, "Query", "&") } elements_build <- function(x, name, collapse, error_call = caller_env()) { - if (!is_list(x) || (!is_named(x) && length(x) > 0)) { + if (!is_named_list(x)) { cli::cli_abort("{name} must be a named list.", call = error_call) } @@ -187,6 +347,7 @@ elements_build <- function(x, name, collapse, error_call = caller_env()) { format_query_param <- function(x, name, multi = FALSE, + form = FALSE, error_call = caller_env()) { check_query_param(x, name, multi = multi, error_call = error_call) @@ -194,7 +355,11 @@ format_query_param <- function(x, unclass(x) } else { x <- format(x, scientific = FALSE, trim = TRUE, justify = "none") - curl::curl_escape(x) + x <- curl::curl_escape(x) + if (form) { + x <- gsub("%20", "+", x, fixed = TRUE) + } + x } } check_query_param <- function(x, name, multi = FALSE, error_call = caller_env()) { diff --git a/R/utils-multi.R b/R/utils-multi.R index aa16e35..3fd851f 100644 --- a/R/utils-multi.R +++ b/R/utils-multi.R @@ -1,5 +1,6 @@ multi_dots <- function(..., .multi = c("error", "comma", "pipe", "explode"), + .space = c("percent", "form"), error_arg = "...", error_call = caller_env()) { if (is.function(.multi)) { @@ -7,6 +8,8 @@ multi_dots <- function(..., } else { .multi <- arg_match(.multi, error_arg = ".multi", error_call = error_call) } + .space <- arg_match(.space, call = error_call) + form <- .space == "form" dots <- list2(...) if (length(dots) == 0) { @@ -31,20 +34,20 @@ multi_dots <- function(..., n <- lengths(dots) if (any(n > 1)) { if (is.function(.multi)) { - dots[n > 1] <- imap(dots[n > 1], format_query_param, multi = TRUE) + dots[n > 1] <- imap(dots[n > 1], format_query_param, multi = TRUE, form = form) dots[n > 1] <- lapply(dots[n > 1], .multi) dots[n > 1] <- lapply(dots[n > 1], I) } else if (.multi == "comma") { - dots[n > 1] <- imap(dots[n > 1], format_query_param, multi = TRUE) + dots[n > 1] <- imap(dots[n > 1], format_query_param, multi = TRUE, form = form) dots[n > 1] <- lapply(dots[n > 1], paste0, collapse = ",") dots[n > 1] <- lapply(dots[n > 1], I) } else if (.multi == "pipe") { - dots[n > 1] <- imap(dots[n > 1], format_query_param, multi = TRUE) + dots[n > 1] <- imap(dots[n > 1], format_query_param, multi = TRUE, form = form) dots[n > 1] <- lapply(dots[n > 1], paste0, collapse = "|") dots[n > 1] <- lapply(dots[n > 1], I) } else if (.multi == "explode") { dots <- explode(dots) - dots[n > 1] <- imap(dots[n > 1], format_query_param, multi = TRUE) + dots[n > 1] <- imap(dots[n > 1], format_query_param, multi = TRUE, form = form) dots[n > 1] <- lapply(dots[n > 1], I) } else if (.multi == "error") { cli::cli_abort( @@ -58,7 +61,12 @@ multi_dots <- function(..., } # Format other params - dots[n == 1] <- imap(dots[n == 1], format_query_param, error_call = error_call) + dots[n == 1] <- imap( + dots[n == 1], + format_query_param, + form = form, + error_call = error_call + ) dots[n == 1] <- lapply(dots[n == 1], I) dots diff --git a/R/utils.R b/R/utils.R index 91e2516..bce1bed 100644 --- a/R/utils.R +++ b/R/utils.R @@ -8,7 +8,7 @@ bullets_with_header <- function(header, x) { as_simple <- function(x) { if (is.atomic(x) && length(x) == 1) { if (is.character(x)) { - paste0("'", x, "'") + paste0('"', x, '"') } else { format(x) } @@ -18,7 +18,9 @@ bullets_with_header <- function(header, x) { } vals <- map_chr(x, as_simple) - cli::cli_li(paste0("{.field ", names(x), "}: ", vals)) + for (i in seq_along(x)) { + cli::cli_li("{.field {names(x)[[i]]}}: {vals[[i]]}") + } } modify_list <- function(.x, ..., error_call = caller_env()) { @@ -32,6 +34,8 @@ modify_list <- function(.x, ..., error_call = caller_env()) { ) } + + out <- .x[!names(.x) %in% names(dots)] out <- c(out, compact(dots)) @@ -324,3 +328,24 @@ slice <- function(vector, start = 1, end = length(vector) + 1) { vector[start:(end - 1)] } } + +is_named_list <- function(x) { + is_list(x) && (is_named(x) || length(x) == 0) +} + +pretty_json <- function(x) { + parsed <- tryCatch( + jsonlite::parse_json(x), + error = function(e) NULL + ) + if (is.null(parsed)) { + x + } else { + jsonlite::toJSON(parsed, auto_unbox = TRUE, pretty = TRUE) + } +} + +log_stream <- function(..., prefix = "<< ") { + out <- gsub("\n", paste0("\n", prefix), paste0(prefix, ..., collapse = "")) + cli::cat_line(out) +} diff --git a/README.md b/README.md index cb2d1d4..cecbeb5 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,12 @@ coverage](https://codecov.io/gh/r-lib/httr2/branch/main/graph/badge.svg)](https://app.codecov.io/gh/r-lib/httr2?branch=main) -httr2 (pronounced hitter2) is a ground-up rewrite of -[httr](https://httr.r-lib.org) that provides a pipeable API with an -explicit request object that solves more problems felt by packages that -wrap APIs (e.g. built-in rate-limiting, retries, OAuth, secure secrets, -and more). +httr2 (pronounced “hitter2”) is a comprehensive HTTP client that +provides a modern, pipeable API for working with web APIs. It builds on +top of [{curl}](https://jeroen.r-universe.dev/curl) to provide features +like explicit request objects, built-in rate limiting & retry tooling, +comprehensive OAuth support, and secure handling of secrets and +credentials. ## Installation @@ -46,7 +47,7 @@ req |> req_headers("Accept" = "application/json") #> #> GET https://r-project.org #> Headers: -#> • Accept: 'application/json' +#> • Accept: "application/json" #> Body: empty # Add a body, turning it into a POST @@ -69,6 +70,9 @@ req |> req_retry(max_tries = 5) #> Policies: #> • retry_max_tries: 5 #> • retry_on_failure: FALSE +#> • retry_failure_threshold: Inf +#> • retry_failure_timeout: 30 +#> • retry_realm: "r-project.org" # Change the HTTP method req |> req_method("PATCH") @@ -83,9 +87,9 @@ And see exactly what httr2 will send to the server with `req_dry_run()`: req |> req_dry_run() #> GET / HTTP/1.1 #> Host: r-project.org -#> User-Agent: httr2/1.0.3.9000 r-curl/5.2.2 libcurl/8.6.0 +#> User-Agent: httr2/1.0.7.9000 r-curl/6.1.0 libcurl/8.7.1 #> Accept: */* -#> Accept-Encoding: deflate, gzip +#> Accept-Encoding: gzip ``` Use `req_perform()` to perform the request, retrieving a **response**: @@ -97,7 +101,7 @@ resp #> GET https://www.r-project.org/ #> Status: 200 OK #> Content-Type: text/html -#> Body: In memory (6951 bytes) +#> Body: In memory (6774 bytes) ``` The `resp_` functions help you extract various useful components of the @@ -147,10 +151,9 @@ resp |> resp_body_html() ## Acknowledgements -httr2 wouldn’t be possible without -[curl](https://jeroen.cran.dev/curl/), -[openssl](https://github.com/jeroen/openssl/), -[jsonlite](https://jeroen.cran.dev/jsonlite/), and +httr2 wouldn’t be possible without [curl](https://cran.dev/curl/), +[openssl](https://cran.dev/openssl/), +[jsonlite](https://cran.dev/jsonlite/), and [jose](https://github.com/r-lib/jose/), which are all maintained by [Jeroen Ooms](https://github.com/jeroen). A big thanks also go to [Jenny Bryan](https://jennybryan.org) and [Craig diff --git a/inst/doc/httr2.html b/inst/doc/httr2.html index f13d4fb..b6cff07 100644 --- a/inst/doc/httr2.html +++ b/inst/doc/httr2.html @@ -358,7 +358,7 @@

Create a request

req <- request(example_url())
 req
 #> <httr2_request>
-#> GET http://127.0.0.1:50487/
+#> GET http://127.0.0.1:56838/
 #> Body: empty

Here, instead of an external website, we use a test server that’s built-in to httr2 itself. That ensures that this vignette will work @@ -367,8 +367,8 @@

Create a request

dry run:

req |> req_dry_run()
 #> GET / HTTP/1.1
-#> Host: 127.0.0.1:50487
-#> User-Agent: httr2/1.0.7 r-curl/6.0.1 libcurl/8.7.1
+#> Host: 127.0.0.1:56838
+#> User-Agent: httr2/1.1.0 r-curl/6.1.0 libcurl/8.7.1
 #> Accept: */*
 #> Accept-Encoding: gzip

The first line of the request contains three important pieces of @@ -382,7 +382,7 @@

Create a request

  • The path, which is the URL stripped of details that the server already knows, i.e. the protocol (http or https), the host (localhost), and the port -(50487).

    • +(56838).

  • The version of the HTTP protocol. This is unimportant for our purposes because it’s handled at a lower level.

    • @@ -398,8 +398,8 @@

    Create a request

    ) |> req_dry_run() #> GET / HTTP/1.1 -#> Host: 127.0.0.1:50487 -#> User-Agent: httr2/1.0.7 r-curl/6.0.1 libcurl/8.7.1 +#> Host: 127.0.0.1:56838 +#> User-Agent: httr2/1.1.0 r-curl/6.1.0 libcurl/8.7.1 #> Accept-Encoding: gzip #> Name: Hadley #> Shoe-Size: 11 @@ -416,8 +416,8 @@

    Create a request

    req_body_json(list(x = 1, y = "a")) |> req_dry_run() #> POST / HTTP/1.1 -#> Host: 127.0.0.1:50487 -#> User-Agent: httr2/1.0.7 r-curl/6.0.1 libcurl/8.7.1 +#> Host: 127.0.0.1:56838 +#> User-Agent: httr2/1.1.0 r-curl/6.1.0 libcurl/8.7.1 #> Accept: */* #> Accept-Encoding: gzip #> Content-Type: application/json @@ -442,8 +442,8 @@

    Create a request

    req_body_form(x = "1", y = "a") |> req_dry_run() #> POST / HTTP/1.1 -#> Host: 127.0.0.1:50487 -#> User-Agent: httr2/1.0.7 r-curl/6.0.1 libcurl/8.7.1 +#> Host: 127.0.0.1:56838 +#> User-Agent: httr2/1.1.0 r-curl/6.1.0 libcurl/8.7.1 #> Accept: */* #> Accept-Encoding: gzip #> Content-Type: application/x-www-form-urlencoded @@ -457,22 +457,22 @@

    Create a request

    req_body_multipart(x = "1", y = "a") |> req_dry_run() #> POST / HTTP/1.1 -#> Host: 127.0.0.1:50487 -#> User-Agent: httr2/1.0.7 r-curl/6.0.1 libcurl/8.7.1 +#> Host: 127.0.0.1:56838 +#> User-Agent: httr2/1.1.0 r-curl/6.1.0 libcurl/8.7.1 #> Accept: */* #> Accept-Encoding: gzip #> Content-Length: 246 -#> Content-Type: multipart/form-data; boundary=------------------------k9Lr0uMj0d6QwSHcamJX8n +#> Content-Type: multipart/form-data; boundary=------------------------qUCL3dFUsWmQATVBqYbWZQ #> -#> --------------------------k9Lr0uMj0d6QwSHcamJX8n +#> --------------------------qUCL3dFUsWmQATVBqYbWZQ #> Content-Disposition: form-data; name="x" #> #> 1 -#> --------------------------k9Lr0uMj0d6QwSHcamJX8n +#> --------------------------qUCL3dFUsWmQATVBqYbWZQ #> Content-Disposition: form-data; name="y" #> #> a -#> --------------------------k9Lr0uMj0d6QwSHcamJX8n-- +#> --------------------------qUCL3dFUsWmQATVBqYbWZQ--

    If you need to send data encoded in a different form, you can use req_body_raw() to add the data to the body and set the Content-Type header.

    @@ -485,7 +485,7 @@

    Perform a request and fetch the response

    resp <- req |> req_perform() resp #> <httr2_response> -#> GET http://127.0.0.1:50487/json +#> GET http://127.0.0.1:56838/json #> Status: 200 OK #> Content-Type: application/json #> Body: In memory (407 bytes) @@ -493,36 +493,35 @@

    Perform a request and fetch the response

    resp_raw():

    resp |> resp_raw()
 #> HTTP/1.1 200 OK
-#> Connection: close
-#> Date: Tue, 26 Nov 2024 13:45:27 GMT
-#> Content-Type: application/json
-#> Content-Length: 407
-#> ETag: "de760e6d"
-#> 
-#> {
-#>   "firstName": "John",
-#>   "lastName": "Smith",
-#>   "isAlive": true,
-#>   "age": 27,
-#>   "address": {
-#>     "streetAddress": "21 2nd Street",
-#>     "city": "New York",
-#>     "state": "NY",
-#>     "postalCode": "10021-3100"
-#>   },
-#>   "phoneNumbers": [
-#>     {
-#>       "type": "home",
-#>       "number": "212 555-1234"
-#>     },
-#>     {
-#>       "type": "office",
-#>       "number": "646 555-4567"
-#>     }
-#>   ],
-#>   "children": [],
-#>   "spouse": null
-#> }
    +#> Date: Sat, 18 Jan 2025 14:13:15 GMT +#> Content-Type: application/json +#> Content-Length: 407 +#> ETag: "de760e6d" +#> +#> { +#> "firstName": "John", +#> "lastName": "Smith", +#> "isAlive": true, +#> "age": 27, +#> "address": { +#> "streetAddress": "21 2nd Street", +#> "city": "New York", +#> "state": "NY", +#> "postalCode": "10021-3100" +#> }, +#> "phoneNumbers": [ +#> { +#> "type": "home", +#> "number": "212 555-1234" +#> }, +#> { +#> "type": "office", +#> "number": "646 555-4567" +#> } +#> ], +#> "children": [], +#> "spouse": null +#> }

    An HTTP response has a very similar structure to an HTTP request. The first line gives the version of HTTP used, and a status code that’s optionally followed by a short description. Then we have the headers, @@ -541,13 +540,12 @@

    Perform a request and fetch the response

    specific header with resp_header():

    resp |> resp_headers()
 #> <httr2_headers>
-#> Connection: close
-#> Date: Tue, 26 Nov 2024 13:45:27 GMT
-#> Content-Type: application/json
-#> Content-Length: 407
-#> ETag: "de760e6d"
-resp |> resp_header("Content-Length")
-#> [1] "407"
    +#> Date: Sat, 18 Jan 2025 14:13:15 GMT +#> Content-Type: application/json +#> Content-Length: 407 +#> ETag: "de760e6d" +resp |> resp_header("Content-Length") +#> [1] "407"

    Headers are case insensitive:

    resp |> resp_header("ConTEnT-LeNgTH")
 #> [1] "407"
    diff --git a/man/is_online.Rd b/man/is_online.Rd new file mode 100644 index 0000000..8fba512 --- /dev/null +++ b/man/is_online.Rd @@ -0,0 +1,16 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/is-online.R +\name{is_online} +\alias{is_online} +\title{Is your computer currently online?} +\usage{ +is_online() +} +\description{ +This function uses some cheap heuristics to determine if your computer is +currently online. It's a simple wrapper around \code{\link[curl:nslookup]{curl::has_internet()}} +exported from httr2 for convenience. +} +\examples{ +is_online() +} diff --git a/man/multi_req_perform.Rd b/man/multi_req_perform.Rd index a178c02..f47ac90 100644 --- a/man/multi_req_perform.Rd +++ b/man/multi_req_perform.Rd @@ -10,7 +10,8 @@ multi_req_perform(reqs, paths = NULL, pool = NULL, cancel_on_error = FALSE) \item{reqs}{A list of \link{request}s.} \item{paths}{An optional character vector of paths, if you want to download -the request bodies to disk. If supplied, must be the same length as \code{reqs}.} +the response bodies to disk. If supplied, must be the same length as +\code{reqs}.} \item{pool}{Optionally, a curl pool made by \code{\link[curl:multi]{curl::new_pool()}}. Supply this if you want to override the defaults for total concurrent connections diff --git a/man/progress_bars.Rd b/man/progress_bars.Rd index 3acb1d1..83c52c5 100644 --- a/man/progress_bars.Rd +++ b/man/progress_bars.Rd @@ -33,9 +33,6 @@ the same as \code{format}. By default the same as \code{format}. \item \code{name}: progress bar name. This is by default the empty string and it is displayed at the beginning of the progress bar. -\item \code{show_after}: numeric scalar. Only show the progress bar after this -number of seconds. It overrides the \code{cli.progress_show_after} -global option. \item \code{type}: progress bar type. Currently supported types are: \itemize{ \item \code{iterator}: the default, a for loop or a mapping function, diff --git a/man/req_auth_basic.Rd b/man/req_auth_basic.Rd index 1627404..7f30492 100644 --- a/man/req_auth_basic.Rd +++ b/man/req_auth_basic.Rd @@ -11,8 +11,8 @@ req_auth_basic(req, username, password = NULL) \item{username}{User name.} -\item{password}{Password. You avoid entering the password directly when -calling this function as it will be captured by \code{.Rhistory}. Instead, +\item{password}{Password. You should avoid entering the password directly +when calling this function as it will be captured by \code{.Rhistory}. Instead, leave it unset and the default behaviour will prompt you for it interactively.} } diff --git a/man/req_body.Rd b/man/req_body.Rd index cf41628..67224a8 100644 --- a/man/req_body.Rd +++ b/man/req_body.Rd @@ -53,7 +53,8 @@ or an empty list (\code{"list"}).} data in the body. \itemize{ \item For \code{req_body_form()}, the values must be strings (or things easily -coerced to strings); +coerced to strings). Vectors are convertd to strings using the +value of \code{.multi}. \item For \code{req_body_multipart()} the values must be strings or objects produced by \code{\link[curl:multipart]{curl::form_file()}}/\code{\link[curl:multipart]{curl::form_data()}}. \item For \code{req_body_json_modify()}, any simple data made from atomic vectors @@ -63,17 +64,17 @@ and lists. \code{req_body_json()} uses this argument differently; it takes additional arguments passed on to \code{\link[jsonlite:fromJSON]{jsonlite::toJSON()}}.} -\item{.multi}{Controls what happens when an element of \code{...} is a vector -containing multiple values: +\item{.multi}{Controls what happens when a value is a vector: \itemize{ \item \code{"error"}, the default, throws an error. \item \code{"comma"}, separates values with a \verb{,}, e.g. \verb{?x=1,2}. \item \code{"pipe"}, separates values with a \code{|}, e.g. \code{?x=1|2}. -\item \code{"explode"}, turns each element into its own parameter, e.g. \code{?x=1&x=2}. +\item \code{"explode"}, turns each element into its own parameter, e.g. \code{?x=1&x=2} } -If none of these functions work, you can alternatively supply a function -that takes a character vector and returns a string.} +If none of these options work for your needs, you can instead supply a +function that takes a character vector of argument values and returns a +a single string.} } \value{ A modified HTTP \link{request}. diff --git a/man/req_dry_run.Rd b/man/req_dry_run.Rd index 62ce151..50ce30b 100644 --- a/man/req_dry_run.Rd +++ b/man/req_dry_run.Rd @@ -25,6 +25,13 @@ actually sending anything. It requires the httpuv package because it works by sending the real HTTP request to a local webserver, thanks to the magic of \code{\link[curl:curl_echo]{curl::curl_echo()}}. } +\details{ +\subsection{Limitations}{ +\itemize{ +\item The \code{Host} header is not respected. +} +} +} \examples{ # httr2 adds default User-Agent, Accept, and Accept-Encoding headers request("http://example.com") |> req_dry_run() diff --git a/man/req_headers.Rd b/man/req_headers.Rd index c16d693..c78d5cd 100644 --- a/man/req_headers.Rd +++ b/man/req_headers.Rd @@ -2,9 +2,12 @@ % Please edit documentation in R/req-headers.R \name{req_headers} \alias{req_headers} +\alias{req_headers_redacted} \title{Modify request headers} \usage{ req_headers(.req, ..., .redact = NULL) + +req_headers_redacted(.req, ...) } \arguments{ \item{.req}{A \link{request}.} @@ -25,6 +28,10 @@ A modified HTTP \link{request}. } \description{ \code{req_headers()} allows you to set the value of any header. + +\code{req_headers_redacted()} is a variation that adds "redacted" headers, which +httr2 avoids printing on the console. This is good practice for +authentication headers to avoid accidentally leaking them in log files. } \examples{ req <- request("http://example.com") @@ -59,11 +66,14 @@ req |> # If you have headers in a list, use !!! headers <- list(HeaderOne = "one", HeaderTwo = "two") req |> - req_headers(!!!headers, HeaderThree = "three") |> - req_dry_run() - -# Use `.redact` to hide a header in the output -req |> - req_headers(Secret = "this-is-private", Public = "but-this-is-not", .redact = "Secret") |> + req_headers(!!!headers, HeaderThree = "three") |> req_dry_run() + +# Use `req_headers_redacted()`` to hide a header in the output +req_secret <- req |> + req_headers_redacted(Secret = "this-is-private") |> + req_headers(Public = "but-this-is-not") + +req_secret +req_secret |> req_dry_run() } diff --git a/man/req_oauth_auth_code.Rd b/man/req_oauth_auth_code.Rd index f00fec1..6ceae16 100644 --- a/man/req_oauth_auth_code.Rd +++ b/man/req_oauth_auth_code.Rd @@ -101,29 +101,32 @@ by \href{https://datatracker.ietf.org/doc/html/rfc6749#section-4.1}{Section 4.1 This flow is the most commonly used OAuth flow where the user opens a page in their browser, approves the access, and then returns to R. When possible, it redirects the browser back to a temporary local webserver -to capture the authorization code. When this is not possible (e.g. when +to capture the authorization code. When this is not possible (e.g., when running on a hosted platform like RStudio Server), provide a custom \code{redirect_uri} and httr2 will prompt the user to enter the code manually. Learn more about the overall OAuth authentication flow in -\url{https://httr2.r-lib.org/articles/oauth.html}. +\url{https://httr2.r-lib.org/articles/oauth.html}, and more about the motivations +behind this flow in +\url{https://stack-auth.com/blog/oauth-from-first-principles}. } \section{Security considerations}{ The authorization code flow is used for both web applications and native applications (which are equivalent to R packages). \href{https://datatracker.ietf.org/doc/html/rfc8252}{RFC 8252} spells out important considerations for native apps. Most importantly there's no way for native apps to keep secrets from their users. This means that the -server should either not require a \code{client_secret} (i.e. a public client -not an confidential client) or ensure that possession of the \code{client_secret} -doesn't bestow any meaningful rights. - -Only modern APIs from the bigger players (Azure, Google, etc) explicitly -native apps. However, in most cases, even for older APIs, possessing the -\code{client_secret} gives you no ability to do anything harmful, so our -general principle is that it's fine to include it in an R package, as long -as it's mildly obfuscated to protect it from credential scraping. There's -no incentive to steal your client credentials if it takes less time to -create a new client than find your client secret. +server should either not require a \code{client_secret} (i.e. it should be a +public client and not a confidential client) or ensure that possession of +the \code{client_secret} doesn't grant any significant privileges. + +Only modern APIs from major providers (like Azure and Google) explicitly +support native apps. However, in most cases, even for older APIs, possessing +the \code{client_secret} provides limited ability to perform harmful actions. +Therefore, our general principle is that it's acceptable to include it in an +R package, as long as it's mildly obfuscated to protect against credential +scraping attacks (which aim to acquire large numbers of client secrets by +scanning public sites like GitHub). The goal is to ensure that obtaining your +client credentials is more work than just creating a new client. } \examples{ diff --git a/man/req_oauth_password.Rd b/man/req_oauth_password.Rd index 0fe91d1..713f623 100644 --- a/man/req_oauth_password.Rd +++ b/man/req_oauth_password.Rd @@ -31,8 +31,8 @@ oauth_flow_password( \item{username}{User name.} -\item{password}{Password. You avoid entering the password directly when -calling this function as it will be captured by \code{.Rhistory}. Instead, +\item{password}{Password. You should avoid entering the password directly +when calling this function as it will be captured by \code{.Rhistory}. Instead, leave it unset and the default behaviour will prompt you for it interactively.} diff --git a/man/req_perform_connection.Rd b/man/req_perform_connection.Rd index a6126f3..cc9961b 100644 --- a/man/req_perform_connection.Rd +++ b/man/req_perform_connection.Rd @@ -4,7 +4,7 @@ \alias{req_perform_connection} \title{Perform a request and return a streaming connection} \usage{ -req_perform_connection(req, blocking = TRUE) +req_perform_connection(req, blocking = TRUE, verbosity = NULL) } \arguments{ \item{req}{A httr2 \link{request} object.} @@ -12,10 +12,21 @@ req_perform_connection(req, blocking = TRUE) \item{blocking}{When retrieving data, should the connection block and wait for the desired information or immediately return what it has (possibly nothing)?} + +\item{verbosity}{How much information to print? This is a wrapper +around \code{\link[=req_verbose]{req_verbose()}} that uses an integer to control verbosity: +\itemize{ +\item \code{0}: no output +\item \code{1}: show headers +\item \code{2}: show headers and bodies as they're streamed +\item \code{3}: show headers, bodies, curl status messages, and stream buffer +management } -\description{ -\ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#experimental}{\figure{lifecycle-experimental.svg}{options: alt='[Experimental]'}}}{\strong{[Experimental]}} +Use \code{\link[=with_verbosity]{with_verbosity()}} to control the verbosity of requests that +you can't affect directly.} +} +\description{ Use \code{req_perform_connection()} to perform a request if you want to stream the response body. A response returned by \code{req_perform_connection()} includes a connection as the body. You can then use \code{\link[=resp_stream_raw]{resp_stream_raw()}}, @@ -40,4 +51,11 @@ length(resp_stream_raw(resp, kb = 16)) # Always close the response when you're done close(resp) + +# You can loop until complete with resp_stream_is_complete() +resp <- req_perform_connection(req) +while (!resp_stream_is_complete(resp)) { + print(length(resp_stream_raw(resp, kb = 12))) +} +close(resp) } diff --git a/man/req_perform_iterative.Rd b/man/req_perform_iterative.Rd index 8c8ce2f..4b86e53 100644 --- a/man/req_perform_iterative.Rd +++ b/man/req_perform_iterative.Rd @@ -35,9 +35,11 @@ perform all requests until \code{next_req()} returns \code{NULL}.} far, as well as an error object for the failed request. }} -\item{progress}{Display a progress bar? Use \code{TRUE} to turn on a basic -progress bar, use a string to give it a name, or see \link{progress_bars} to -customise it in other ways.} +\item{progress}{Display a progress bar for the status of all requests? Use +\code{TRUE} to turn on a basic progress bar, use a string to give it a name, +or see \link{progress_bars} to customize it in other ways. Not compatible with +\code{\link[=req_progress]{req_progress()}}, as httr2 can only display a single progress bar at a +time.} } \value{ A list, at most length \code{max_reqs}, containing \link{response}s and possibly one diff --git a/man/req_perform_parallel.Rd b/man/req_perform_parallel.Rd index e6dfb5e..f9c9439 100644 --- a/man/req_perform_parallel.Rd +++ b/man/req_perform_parallel.Rd @@ -16,7 +16,8 @@ req_perform_parallel( \item{reqs}{A list of \link{request}s.} \item{paths}{An optional character vector of paths, if you want to download -the request bodies to disk. If supplied, must be the same length as \code{reqs}.} +the response bodies to disk. If supplied, must be the same length as +\code{reqs}.} \item{pool}{Optionally, a curl pool made by \code{\link[curl:multi]{curl::new_pool()}}. Supply this if you want to override the defaults for total concurrent connections @@ -30,9 +31,11 @@ received so far, as well as an error object for the failed request. \item \code{continue}: continue iterating, recording errors in the result. }} -\item{progress}{Display a progress bar? Use \code{TRUE} to turn on a basic -progress bar, use a string to give it a name, or see \link{progress_bars} to -customise it in other ways.} +\item{progress}{Display a progress bar for the status of all requests? Use +\code{TRUE} to turn on a basic progress bar, use a string to give it a name, +or see \link{progress_bars} to customize it in other ways. Not compatible with +\code{\link[=req_progress]{req_progress()}}, as httr2 can only display a single progress bar at a +time.} } \value{ A list, the same length as \code{reqs}, containing \link{response}s and possibly diff --git a/man/req_perform_promise.Rd b/man/req_perform_promise.Rd index 0573a6d..8f0ca12 100644 --- a/man/req_perform_promise.Rd +++ b/man/req_perform_promise.Rd @@ -4,7 +4,7 @@ \alias{req_perform_promise} \title{Perform request asynchronously using the promises package} \usage{ -req_perform_promise(req, path = NULL, pool = NULL) +req_perform_promise(req, path = NULL, pool = NULL, verbosity = NULL) } \arguments{ \item{req}{A httr2 \link{request} object.} @@ -15,6 +15,18 @@ for large responses since it avoids storing the response in memory.} \item{pool}{Optionally, a curl pool made by \code{\link[curl:multi]{curl::new_pool()}}. Supply this if you want to override the defaults for total concurrent connections (100) or concurrent connections per host (6).} + +\item{verbosity}{How much information to print? This is a wrapper +around \code{\link[=req_verbose]{req_verbose()}} that uses an integer to control verbosity: +\itemize{ +\item \code{0}: no output +\item \code{1}: show headers +\item \code{2}: show headers and bodies +\item \code{3}: show headers, bodies, and curl status messages. +} + +Use \code{\link[=with_verbosity]{with_verbosity()}} to control the verbosity of requests that +you can't affect directly.} } \value{ a \code{\link[promises:promise]{promises::promise()}} object which resolves to a \link{response} if diff --git a/man/req_perform_sequential.Rd b/man/req_perform_sequential.Rd index c5d1334..1089db6 100644 --- a/man/req_perform_sequential.Rd +++ b/man/req_perform_sequential.Rd @@ -15,7 +15,8 @@ req_perform_sequential( \item{reqs}{A list of \link{request}s.} \item{paths}{An optional character vector of paths, if you want to download -the request bodies to disk. If supplied, must be the same length as \code{reqs}.} +the response bodies to disk. If supplied, must be the same length as +\code{reqs}.} \item{on_error}{What should happen if one of the requests fails? \itemize{ @@ -25,9 +26,11 @@ received so far, as well as an error object for the failed request. \item \code{continue}: continue iterating, recording errors in the result. }} -\item{progress}{Display a progress bar? Use \code{TRUE} to turn on a basic -progress bar, use a string to give it a name, or see \link{progress_bars} to -customise it in other ways.} +\item{progress}{Display a progress bar for the status of all requests? Use +\code{TRUE} to turn on a basic progress bar, use a string to give it a name, +or see \link{progress_bars} to customize it in other ways. Not compatible with +\code{\link[=req_progress]{req_progress()}}, as httr2 can only display a single progress bar at a +time.} } \value{ A list, the same length as \code{reqs}, containing \link{response}s and possibly diff --git a/man/req_perform_stream.Rd b/man/req_perform_stream.Rd index 29ee829..f00274c 100644 --- a/man/req_perform_stream.Rd +++ b/man/req_perform_stream.Rd @@ -35,6 +35,12 @@ successful (since the \code{callback} function will have handled it). The body will contain the HTTP response body if the request was unsuccessful. } \description{ +\ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#superseded}{\figure{lifecycle-superseded.svg}{options: alt='[Superseded]'}}}{\strong{[Superseded]}} + +We now recommend \code{\link[=req_perform_connection]{req_perform_connection()}} since it has a considerably more +flexible interface. Unless I hear compelling reasons otherwise, I'm likely +to deprecate \code{req_perform_stream()} in a future release. + After preparing a request, call \code{req_perform_stream()} to perform the request and handle the result with a streaming callback. This is useful for streaming HTTP APIs where potentially the stream never ends. diff --git a/man/req_retry.Rd b/man/req_retry.Rd index 9c3d504..c04a47d 100644 --- a/man/req_retry.Rd +++ b/man/req_retry.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/req-retries.R \name{req_retry} \alias{req_retry} -\title{Control when a request will retry, and how long it will wait between tries} +\title{Automatically retry a request on failure} \usage{ req_retry( req, @@ -11,22 +11,24 @@ req_retry( retry_on_failure = FALSE, is_transient = NULL, backoff = NULL, - after = NULL + after = NULL, + failure_threshold = Inf, + failure_timeout = 30, + failure_realm = NULL ) } \arguments{ \item{req}{A httr2 \link{request} object.} -\item{max_tries, max_seconds}{Cap the maximum number of attempts with -\code{max_tries} or the total elapsed time from the first request with -\code{max_seconds}. If neither option is supplied (the default), \code{\link[=req_perform]{req_perform()}} -will not retry. +\item{max_tries, max_seconds}{Cap the maximum number of attempts +(\code{max_tries}), the total elapsed time from the first request +(\code{max_seconds}), or both. -\code{max_tries} is the total number of attempts make, so this should always -be greater than one.`} +\code{max_tries} is the total number of attempts made, so this should always +be greater than one.} \item{retry_on_failure}{Treat low-level failures as if they are -transient errors, and can be retried.} +transient errors that can be retried.} \item{is_transient}{A predicate function that takes a single argument (the response) and returns \code{TRUE} or \code{FALSE} specifying whether or not @@ -36,44 +38,52 @@ the response represents a transient error.} attempts so far) and returns the number of seconds to wait.} \item{after}{A function that takes a single argument (the response) and -returns either a number of seconds to wait or \code{NULL}, which indicates -that a precise wait time is not available that the \code{backoff} strategy -should be used instead..} +returns either a number of seconds to wait or \code{NA}. \code{NA} indicates +that a precise wait time is not available and that the \code{backoff} strategy +should be used instead.} + +\item{failure_threshold, failure_timeout, failure_realm}{Set \code{failure_threshold} to activate "circuit breaking" where if a request +continues to fail after \code{failure_threshold} times, cause the request to +error until a timeout of \code{failure_timeout} seconds has elapsed. This +timeout will persist across all requests with the same \code{failure_realm} +(which defaults to the hostname of the request) and is intended to detect +failing servers without needing to wait each time.} } \value{ A modified HTTP \link{request}. } \description{ -\code{req_retry()} alters \code{\link[=req_perform]{req_perform()}} so that it will automatically retry -in the case of failure. To activate it, you must specify either the total -number of requests to make with \code{max_tries} or the total amount of time -to spend with \code{max_seconds}. Then \code{req_perform()} will retry if the error is -"transient", i.e. it's an HTTP error that can be resolved by waiting. By -default, 429 and 503 statuses are treated as transient, but if the API you -are wrapping has other transient status codes (or conveys transient-ness -with some other property of the response), you can override the default -with \code{is_transient}. +\code{req_retry()} allows \code{\link[=req_perform]{req_perform()}} to automatically retry failing +requests. It's particularly important for APIs with rate limiting, but can +also be useful when dealing with flaky servers. -Additionally, if you set \code{retry_on_failure = TRUE}, the request will retry -if either the HTTP request or HTTP response doesn't complete successfully +By default, \code{req_perform()} will retry if the response is a 429 +("too many requests", often used for rate limiting) or 503 +("service unavailable"). If the API you are wrapping has other transient +status codes (or conveys transience with some other property of the +response), you can override the default with \code{is_transient}. And +if you set \code{retry_on_failure = TRUE}, the request will retry +if either the HTTP request or HTTP response doesn't complete successfully, leading to an error from curl, the lower-level library that httr2 uses to -perform HTTP request. This occurs, for example, if your wifi is down. +perform HTTP requests. This occurs, for example, if your Wi-Fi is down. +\subsection{Delay}{ It's a bad idea to immediately retry a request, so \code{req_perform()} will wait a little before trying again: \itemize{ \item If the response contains the \code{Retry-After} header, httr2 will wait the amount of time it specifies. If the API you are wrapping conveys this -information with a different header (or other property of the response) -you can override the default behaviour with \code{retry_after}. +information with a different header (or other property of the response), +you can override the default behavior with \code{retry_after}. \item Otherwise, httr2 will use "truncated exponential backoff with full -jitter", i.e. it will wait a random amount of time between one second and -\code{2 ^ tries} seconds, capped to at most 60 seconds. In other words, it +jitter", i.e., it will wait a random amount of time between one second and +\code{2 ^ tries} seconds, capped at a maximum of 60 seconds. In other words, it waits \code{runif(1, 1, 2)} seconds after the first failure, \code{runif(1, 1, 4)} after the second, \code{runif(1, 1, 8)} after the third, and so on. If you'd prefer a different strategy, you can override the default with \code{backoff}. } } +} \examples{ # google APIs assume that a 500 is also a transient error request("http://google.com") |> @@ -81,7 +91,7 @@ request("http://google.com") |> # use a constant 10s delay after every failure request("http://example.com") |> - req_retry(backoff = ~10) + req_retry(backoff = \(resp) 10) # When rate-limited, GitHub's API returns a 403 with # `X-RateLimit-Remaining: 0` and an Unix time stored in the diff --git a/man/req_url.Rd b/man/req_url.Rd index 87bdc28..1e9ebb6 100644 --- a/man/req_url.Rd +++ b/man/req_url.Rd @@ -2,6 +2,7 @@ % Please edit documentation in R/req-url.R \name{req_url} \alias{req_url} +\alias{req_url_relative} \alias{req_url_query} \alias{req_url_path} \alias{req_url_path_append} @@ -9,7 +10,14 @@ \usage{ req_url(req, url) -req_url_query(.req, ..., .multi = c("error", "comma", "pipe", "explode")) +req_url_relative(req, url) + +req_url_query( + .req, + ..., + .multi = c("error", "comma", "pipe", "explode"), + .space = c("percent", "form") +) req_url_path(req, ...) @@ -18,7 +26,8 @@ req_url_path_append(req, ...) \arguments{ \item{req, .req}{A httr2 \link{request} object.} -\item{url}{New URL; completely replaces existing.} +\item{url}{A new URL; either an absolute URL for \code{req_url()} or a +relative URL for \code{req_url_relative()}.} \item{...}{For \code{req_url_query()}: <\code{\link[rlang:dyn-dots]{dynamic-dots}}> Name-value pairs that define query parameters. Each value must be either @@ -28,31 +37,43 @@ If you want to opt out of escaping, wrap strings in \code{I()}. For \code{req_url_path()} and \code{req_url_path_append()}: A sequence of path components that will be combined with \code{/}.} -\item{.multi}{Controls what happens when an element of \code{...} is a vector -containing multiple values: +\item{.multi}{Controls what happens when a value is a vector: \itemize{ \item \code{"error"}, the default, throws an error. \item \code{"comma"}, separates values with a \verb{,}, e.g. \verb{?x=1,2}. \item \code{"pipe"}, separates values with a \code{|}, e.g. \code{?x=1|2}. -\item \code{"explode"}, turns each element into its own parameter, e.g. \code{?x=1&x=2}. +\item \code{"explode"}, turns each element into its own parameter, e.g. \code{?x=1&x=2} } -If none of these functions work, you can alternatively supply a function -that takes a character vector and returns a string.} +If none of these options work for your needs, you can instead supply a +function that takes a character vector of argument values and returns a +a single string.} + +\item{.space}{How should spaces in query params be escaped? The default, +"percent", uses standard percent encoding (i.e. \verb{\%20}), but you can opt-in +to "form" encoding, which uses \code{+} instead.} } \value{ A modified HTTP \link{request}. } \description{ \itemize{ -\item \code{req_url()} replaces the entire url -\item \code{req_url_query()} modifies the components of the query -\item \code{req_url_path()} modifies the path -\item \code{req_url_path_append()} adds to the path +\item \code{req_url()} replaces the entire URL. +\item \code{req_url_relative()} navigates to a relative URL. +\item \code{req_url_query()} modifies individual query components. +\item \code{req_url_path()} modifies just the path. +\item \code{req_url_path_append()} adds to the path. } } \examples{ +# Change complete url req <- request("http://example.com") +req |> req_url("http://google.com") + +# Use a relative url +req <- request("http://example.com/a/b/c") +req |> req_url_relative("..") +req |> req_url_relative("/d/e/f") # Change url components req |> @@ -61,9 +82,11 @@ req |> req_url_path_append("search.html") |> req_url_query(q = "the cool ice") -# Change complete url -req |> - req_url("http://google.com") +# Modify individual query parameters +req <- request("http://example.com?a=1&b=2") +req |> req_url_query(a = 10) +req |> req_url_query(a = NULL) +req |> req_url_query(c = 3) # Use .multi to control what happens with vector parameters: req |> req_url_query(id = 100:105, .multi = "comma") @@ -74,3 +97,10 @@ params <- list(a = "1", b = "2") req |> req_url_query(!!!params, c = "3") } +\seealso{ +\itemize{ +\item To modify a URL without creating a request, see \code{\link[=url_modify]{url_modify()}} and +friends. +\item To use a template like \code{GET /user/{user}}, see \code{\link[=req_template]{req_template()}}. +} +} diff --git a/man/resp_request.Rd b/man/resp_request.Rd new file mode 100644 index 0000000..ff9960b --- /dev/null +++ b/man/resp_request.Rd @@ -0,0 +1,20 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/resp-request.R +\name{resp_request} +\alias{resp_request} +\title{Find the request responsible for a response} +\usage{ +resp_request(resp) +} +\arguments{ +\item{resp}{A httr2 \link{response} object, created by \code{\link[=req_perform]{req_perform()}}.} +} +\description{ +To make debugging easier, httr2 includes the request that was used to +generate every response. You can use this function to access it. +} +\examples{ +req <- request(example_url()) +resp <- req_perform(req) +resp_request(resp) +} diff --git a/man/resp_stream_raw.Rd b/man/resp_stream_raw.Rd index e386a88..c6a43b6 100644 --- a/man/resp_stream_raw.Rd +++ b/man/resp_stream_raw.Rd @@ -6,6 +6,7 @@ \alias{resp_stream_sse} \alias{resp_stream_aws} \alias{close.httr2_response} +\alias{resp_stream_is_complete} \title{Read a streaming body a chunk at a time} \usage{ resp_stream_raw(resp, kb = 32) @@ -17,6 +18,8 @@ resp_stream_sse(resp, max_size = Inf) resp_stream_aws(resp, max_size = Inf) \method{close}{httr2_response}(con, ...) + +resp_stream_is_complete(resp) } \arguments{ \item{resp, con}{A streaming \link{response} created by \code{\link[=req_perform_connection]{req_perform_connection()}}.} @@ -48,7 +51,6 @@ the end of the stream has been reached or, if in nonblocking mode, that no event is currently available. } \description{ -\ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#experimental}{\figure{lifecycle-experimental.svg}{options: alt='[Experimental]'}}}{\strong{[Experimental]}} \itemize{ \item \code{resp_stream_raw()} retrieves bytes (\code{raw} vectors). \item \code{resp_stream_lines()} retrieves lines of text (\code{character} vectors). @@ -56,4 +58,25 @@ no event is currently available. \item \code{resp_stream_aws()} retrieves a single event from an AWS stream (i.e. mime type `application/vnd.amazon.eventstream``). } + +Use \code{resp_stream_is_complete()} to determine if there is further data +waiting on the stream. +} +\examples{ +req <- request(example_url()) |> + req_template("GET /stream/:n", n = 5) + +con <- req |> req_perform_connection() +while (!resp_stream_is_complete(con)) { + lines <- con |> resp_stream_lines(2) + cat(length(lines), " lines received\n", sep = "") +} +close(con) + +# You can also see what's happening by setting verbosity +con <- req |> req_perform_connection(verbosity = 2) +while (!resp_stream_is_complete(con)) { + lines <- con |> resp_stream_lines(2) +} +close(con) } diff --git a/man/resps_successes.Rd b/man/resps_successes.Rd index 5744b0f..aca4539 100644 --- a/man/resps_successes.Rd +++ b/man/resps_successes.Rd @@ -19,7 +19,11 @@ resps_data(resps, resp_data) \item{resps}{A list of responses (possibly including errors).} \item{resp_data}{A function that takes a response (\code{resp}) and -returns the data found inside that response as a vector or data frame.} +returns the data found inside that response as a vector or data frame. + +NB: If you're using \code{\link[=resp_body_raw]{resp_body_raw()}}, you're likely to want to wrap its +output in \code{list()} to avoid combining all the bodies into a single raw +vector, e.g. \verb{resps |> resps_data(\\(resp) list(resp_body_raw(resp)))}.} } \description{ These function provide a basic toolkit for operating with lists of @@ -47,8 +51,12 @@ resps <- req_perform_parallel(reqs, on_error = "continue") resps |> resps_successes() # collect all their data -resps |> resps_successes() |> resps_data(\(resp) resp_body_json(resp)) +resps |> + resps_successes() |> + resps_data(\(resp) resp_body_json(resp)) # find requests corresponding to failure responses -resps |> resps_failures() |> resps_requests() +resps |> + resps_failures() |> + resps_requests() } diff --git a/man/url_build.Rd b/man/url_build.Rd new file mode 100644 index 0000000..c209950 --- /dev/null +++ b/man/url_build.Rd @@ -0,0 +1,21 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/url.R +\name{url_build} +\alias{url_build} +\title{Build a string from a URL object} +\usage{ +url_build(url) +} +\arguments{ +\item{url}{An URL object created by \link{url_parse}.} +} +\description{ +This is the inverse of \code{\link[=url_parse]{url_parse()}}, taking a parsed URL object and +turning it back into a string. +} +\seealso{ +Other URL manipulation: +\code{\link{url_modify}()}, +\code{\link{url_parse}()} +} +\concept{URL manipulation} diff --git a/man/url_modify.Rd b/man/url_modify.Rd new file mode 100644 index 0000000..1eaedfd --- /dev/null +++ b/man/url_modify.Rd @@ -0,0 +1,106 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/url.R +\name{url_modify} +\alias{url_modify} +\alias{url_modify_relative} +\alias{url_modify_query} +\title{Modify a URL} +\usage{ +url_modify( + url, + scheme = as_is, + hostname = as_is, + username = as_is, + password = as_is, + port = as_is, + path = as_is, + query = as_is, + fragment = as_is +) + +url_modify_relative(url, relative_url) + +url_modify_query( + .url, + ..., + .multi = c("error", "comma", "pipe", "explode"), + .space = c("percent", "form") +) +} +\arguments{ +\item{url, .url}{A string or \link[=url_parse]{parsed URL}.} + +\item{scheme}{The scheme, typically either \code{http} or \code{https}.} + +\item{hostname}{The hostname, e.g., \code{www.google.com} or \code{posit.co}.} + +\item{username, password}{Username and password to embed in the URL. +Not generally recommended but needed for some legacy applications.} + +\item{port}{An integer port number.} + +\item{path}{The path, e.g., \verb{/search}. Paths must start with \code{/}, so this +will be automatically added if omitted.} + +\item{query}{Either a query string or a named list of query components.} + +\item{fragment}{The fragment, e.g., \verb{#section-1}.} + +\item{relative_url}{A relative URL to append to the base URL.} + +\item{...}{<\code{\link[rlang:dyn-dots]{dynamic-dots}}> +Name-value pairs that define query parameters. Each value must be either +an atomic vector or \code{NULL} (which removes the corresponding parameters). +If you want to opt out of escaping, wrap strings in \code{I()}.} + +\item{.multi}{Controls what happens when a value is a vector: +\itemize{ +\item \code{"error"}, the default, throws an error. +\item \code{"comma"}, separates values with a \verb{,}, e.g. \verb{?x=1,2}. +\item \code{"pipe"}, separates values with a \code{|}, e.g. \code{?x=1|2}. +\item \code{"explode"}, turns each element into its own parameter, e.g. \code{?x=1&x=2} +} + +If none of these options work for your needs, you can instead supply a +function that takes a character vector of argument values and returns a +a single string.} + +\item{.space}{How should spaces in query params be escaped? The default, +"percent", uses standard percent encoding (i.e. \verb{\%20}), but you can opt-in +to "form" encoding, which uses \code{+} instead.} +} +\value{ +An object of the same type as \code{url}. +} +\description{ +Use \code{url_modify()} to modify any component of the URL, +\code{url_modify_relative()} to modify with a relative URL, +or \code{url_modify_query()} to modify individual query parameters. + +For \code{url_modify()}, components that aren't specified in the +function call will be left as is; components set to \code{NULL} will be removed, +and all other values will be updated. Note that removing \code{scheme} or +\code{hostname} will create a relative URL. +} +\examples{ +url_modify("http://hadley.nz", path = "about") +url_modify("http://hadley.nz", scheme = "https") +url_modify("http://hadley.nz/abc", path = "/cde") +url_modify("http://hadley.nz/abc", path = "") +url_modify("http://hadley.nz?a=1", query = "b=2") +url_modify("http://hadley.nz?a=1", query = list(c = 3)) + +url_modify_query("http://hadley.nz?a=1&b=2", c = 3) +url_modify_query("http://hadley.nz?a=1&b=2", b = NULL) +url_modify_query("http://hadley.nz?a=1&b=2", a = 100) + +url_modify_relative("http://hadley.nz/a/b/c.html", "/d.html") +url_modify_relative("http://hadley.nz/a/b/c.html", "d.html") +url_modify_relative("http://hadley.nz/a/b/c.html", "../d.html") +} +\seealso{ +Other URL manipulation: +\code{\link{url_build}()}, +\code{\link{url_parse}()} +} +\concept{URL manipulation} diff --git a/man/url_parse.Rd b/man/url_parse.Rd index 15a8d0a..9a881c0 100644 --- a/man/url_parse.Rd +++ b/man/url_parse.Rd @@ -2,29 +2,24 @@ % Please edit documentation in R/url.R \name{url_parse} \alias{url_parse} -\alias{url_build} -\title{Parse and build URLs} +\title{Parse a URL into its component pieces} \usage{ -url_parse(url) - -url_build(url) +url_parse(url, base_url = NULL) } \arguments{ -\item{url}{For \code{url_parse()} a string to parse into a URL; -for \code{url_build()} a URL to turn back into a string.} +\item{url}{A string containing the URL to parse.} + +\item{base_url}{Use this as a parent, if \code{url} is a relative URL.} } \value{ -\itemize{ -\item \code{url_build()} returns a string. -\item \code{url_parse()} returns a URL: a S3 list with class \code{httr2_url} -and elements \code{scheme}, \code{hostname}, \code{port}, \code{path}, \code{fragment}, \code{query}, -\code{username}, \code{password}. -} +An S3 object of class \code{httr2_url} with the following components: +\code{scheme}, \code{hostname}, \code{username}, \code{password}, \code{port}, \code{path}, \code{query}, and +\code{fragment}. } \description{ -\code{url_parse()} parses a URL into its component pieces; \code{url_build()} does -the reverse, converting a list of pieces into a string URL. See \href{https://datatracker.ietf.org/doc/html/rfc3986}{RFC 3986} -for the details of the parsing algorithm. +\code{url_parse()} parses a URL into its component parts, powered by +\code{\link[curl:curl_parse_url]{curl::curl_parse_url()}}. The parsing algorithm follows the specifications +detailed in \href{https://datatracker.ietf.org/doc/html/rfc3986}{RFC 3986}. } \examples{ url_parse("http://google.com/") @@ -32,9 +27,13 @@ url_parse("http://google.com:80/") url_parse("http://google.com:80/?a=1&b=2") url_parse("http://username@google.com:80/path;test?a=1&b=2#40") -url <- url_parse("http://google.com/") -url$port <- 80 -url$hostname <- "example.com" -url$query <- list(a = 1, b = 2, c = 3) -url_build(url) +# You can parse a relative URL if you also provide a base url +url_parse("foo", "http://google.com/bar/") +url_parse("..", "http://google.com/bar/") +} +\seealso{ +Other URL manipulation: +\code{\link{url_build}()}, +\code{\link{url_modify}()} } +\concept{URL manipulation} diff --git a/man/url_query_parse.Rd b/man/url_query_parse.Rd new file mode 100644 index 0000000..14e248e --- /dev/null +++ b/man/url_query_parse.Rd @@ -0,0 +1,36 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/url.R +\name{url_query_parse} +\alias{url_query_parse} +\alias{url_query_build} +\title{Parse query parameters and/or build a string} +\usage{ +url_query_parse(query) + +url_query_build(query, .multi = c("error", "comma", "pipe", "explode")) +} +\arguments{ +\item{query}{A string, when parsing; a named list when building.} + +\item{.multi}{Controls what happens when a value is a vector: +\itemize{ +\item \code{"error"}, the default, throws an error. +\item \code{"comma"}, separates values with a \verb{,}, e.g. \verb{?x=1,2}. +\item \code{"pipe"}, separates values with a \code{|}, e.g. \code{?x=1|2}. +\item \code{"explode"}, turns each element into its own parameter, e.g. \code{?x=1&x=2} +} + +If none of these options work for your needs, you can instead supply a +function that takes a character vector of argument values and returns a +a single string.} +} +\description{ +\code{url_query_parse()} parses a query string into a named list; +\code{url_query_build()} builds a query string from a named list. +} +\examples{ +str(url_query_parse("a=1&b=2")) + +url_query_build(list(x = 1, y = "z")) +url_query_build(list(x = 1, y = 1:2), .multi = "explode") +} diff --git a/tests/testthat/_snaps/curl.md b/tests/testthat/_snaps/curl.md index 3d58afc..fd8f649 100644 --- a/tests/testthat/_snaps/curl.md +++ b/tests/testthat/_snaps/curl.md @@ -40,18 +40,18 @@ Code curl_translate("curl http://x.com") Output - request("http://x.com") |> + request("http://x.com/") |> req_perform() Code curl_translate("curl http://x.com -X DELETE") Output - request("http://x.com") |> + request("http://x.com/") |> req_method("DELETE") |> req_perform() Code curl_translate("curl http://x.com -H A:1") Output - request("http://x.com") |> + request("http://x.com/") |> req_headers( A = "1", ) |> @@ -59,7 +59,7 @@ Code curl_translate("curl http://x.com -H 'A B:1'") Output - request("http://x.com") |> + request("http://x.com/") |> req_headers( `A B` = "1", ) |> @@ -67,13 +67,13 @@ Code curl_translate("curl http://x.com -u u:p") Output - request("http://x.com") |> + request("http://x.com/") |> req_auth_basic("u", "p") |> req_perform() Code curl_translate("curl http://x.com --verbose") Output - request("http://x.com") |> + request("http://x.com/") |> req_perform(verbosity = 1) # can translate query @@ -81,7 +81,7 @@ Code curl_translate("curl http://x.com?string=abcde&b=2") Output - request("http://x.com") |> + request("http://x.com/") |> req_url_query( string = "abcde", b = "2", @@ -93,17 +93,61 @@ Code curl_translate("curl http://example.com --data abcdef") Output - request("http://example.com") |> + request("http://example.com/") |> req_body_raw("abcdef", "application/x-www-form-urlencoded") |> req_perform() Code curl_translate( "curl http://example.com --data abcdef -H Content-Type:text/plain") Output - request("http://example.com") |> + request("http://example.com/") |> req_body_raw("abcdef", "text/plain") |> req_perform() +# can translate ocokies + + Code + curl_translate("curl 'http://test' -H 'Cookie: x=1; y=2;z=3'") + Output + request("http://test/") |> + req_cookies_set( + x = "1", + y = "2", + z = "3", + ) |> + req_perform() + +# can translate json + + Code + curl_translate( + "curl http://example.com --data-raw '{\"a\": 1, \"b\": \"text\"}' -H Content-Type:application/json") + Output + request("http://example.com/") |> + req_body_json( + data = list(a = 1L, b = "text"), + ) |> + req_perform() + Code + curl_translate("curl http://example.com --json '{\"a\": 1, \"b\": \"text\"}'") + Output + request("http://example.com/") |> + req_body_json( + data = list(a = 1L, b = "text"), + ) |> + req_perform() + +# content type stays in header if no data + + Code + curl_translate("curl http://example.com -H Content-Type:text/plain") + Output + request("http://example.com/") |> + req_headers( + `Content-Type` = "text/plain", + ) |> + req_perform() + # can read from clipboard Code @@ -111,7 +155,7 @@ Message v Copying to clipboard: Output - request("http://example.com") |> + request("http://example.com/") |> req_headers( A = "1", B = "2", @@ -120,17 +164,22 @@ Code clipr::read_clip() Output - [1] "request(\"http://example.com\") |> " " req_headers(" - [3] " A = \"1\"," " B = \"2\"," - [5] " ) |> " " req_perform()" + [1] "request(\"http://example.com/\") |> " + [2] " req_headers(" + [3] " A = \"1\"," + [4] " B = \"2\"," + [5] " ) |> " + [6] " req_perform()" # encode_string2() produces simple strings Code curl_translate(cmd) Output - request("http://example.com") |> + request("http://example.com/") |> req_method("PATCH") |> - req_body_raw('{"data":{"x":1,"y":"a","nested":{"z":[1,2,3]}}}', "application/json") |> + req_body_json( + data = list(data = list(x = 1L, y = "a", nested = list(z = list(1L, 2L, 3L)))), + ) |> req_perform() diff --git a/tests/testthat/_snaps/req-mock.md b/tests/testthat/_snaps/req-mock.md index 6d0d477..3b77c31 100644 --- a/tests/testthat/_snaps/req-mock.md +++ b/tests/testthat/_snaps/req-mock.md @@ -4,13 +4,13 @@ local_mock(~ response(404)) Condition Warning: - `local_mock()` was deprecated in httr2 1.0.0. + `local_mock()` was deprecated in httr2 1.1.0. i Please use `local_mocked_responses()` instead. Code . <- with_mock(NULL, ~ response(404)) Condition - Warning: - `with_mock()` was deprecated in httr2 1.0.0. + Error: + ! `with_mock()` was deprecated in httr2 1.1.0 and is now defunct. i Please use `with_mocked_responses()` instead. # validates inputs diff --git a/tests/testthat/_snaps/req-promise.md b/tests/testthat/_snaps/req-promise.md index d42917e..efcdd75 100644 --- a/tests/testthat/_snaps/req-promise.md +++ b/tests/testthat/_snaps/req-promise.md @@ -1,3 +1,44 @@ +# checks its inputs + + Code + req_perform_promise(1) + Condition + Error in `req_perform_promise()`: + ! `req` must be an HTTP request object, not the number 1. + Code + req_perform_promise(req, path = 1) + Condition + Error in `req_perform_promise()`: + ! `path` must be a single string or `NULL`, not the number 1. + Code + req_perform_promise(req, pool = "INVALID") + Condition + Error in `req_perform_promise()`: + ! `pool` must be a {curl} pool or `NULL`, not the string "INVALID". + Code + req_perform_promise(req, verbosity = "INVALID") + Condition + Error in `req_perform_promise()`: + ! `verbosity` must 0, 1, 2, or 3. + +# correctly prepares request + + Code + . <- extract_promise(req_perform_promise(req, verbosity = 1)) + Output + -> GET /get HTTP/1.1 + -> Host: + -> User-Agent: + -> Accept: */* + -> Accept-Encoding: + -> + <- HTTP/1.1 200 OK + <- Date: + <- Content-Type: application/json + <- Content-Length: + <- ETag: + <- + # req_perform_promise uses the default loop Code diff --git a/tests/testthat/_snaps/req-retries.md b/tests/testthat/_snaps/req-retries.md index a44fc28..a1d8ea1 100644 --- a/tests/testthat/_snaps/req-retries.md +++ b/tests/testthat/_snaps/req-retries.md @@ -1,3 +1,10 @@ +# has useful default (with message) + + Code + req <- req_retry(req) + Message + Setting `max_tries = 2`. + # useful message if `after` wrong Code @@ -9,17 +16,17 @@ # validates its inputs Code - req_retry(req, max_tries = 1) + req_retry(req, max_tries = 0) Condition Error in `req_retry()`: - ! `max_tries` must be a whole number larger than or equal to 2 or `NULL`, not the number 1. + ! `max_tries` must be a whole number larger than or equal to 1 or `NULL`, not the number 0. Code - req_retry(req, max_seconds = "x") + req_retry(req, max_tries = 2, max_seconds = "x") Condition Error in `req_retry()`: ! `max_seconds` must be a whole number or `NULL`, not the string "x". Code - req_retry(req, retry_on_failure = "x") + req_retry(req, max_tries = 2, retry_on_failure = "x") Condition Error in `req_retry()`: ! `retry_on_failure` must be `TRUE` or `FALSE`, not the string "x". diff --git a/tests/testthat/_snaps/req-url.md b/tests/testthat/_snaps/req-url.md index aeb5062..b67f376 100644 --- a/tests/testthat/_snaps/req-url.md +++ b/tests/testthat/_snaps/req-url.md @@ -1,9 +1,17 @@ +# can control space handling + + Code + req_url_query(req, a = " ", .space = "bar") + Condition + Error in `multi_dots()`: + ! `.space` must be one of "percent" or "form", not "bar". + # can handle multi query params Code req_url_query_multi("error") Condition - Error in `req_url_query()`: + Error in `url_modify_query()`: ! All vector elements of `...` must be length 1. i Use `.multi` to choose a strategy for handling vectors. @@ -12,22 +20,22 @@ Code req %>% req_url_query(1) Condition - Error in `req_url_query()`: + Error in `url_modify_query()`: ! All components of `...` must be named. Code req %>% req_url_query(a = I(1)) Condition - Error in `req_url_query()`: + Error in `url_modify_query()`: ! Escaped query value `a` must be a single string, not the number 1. Code req %>% req_url_query(a = 1:2) Condition - Error in `req_url_query()`: + Error in `url_modify_query()`: ! All vector elements of `...` must be length 1. i Use `.multi` to choose a strategy for handling vectors. Code req %>% req_url_query(a = mean) Condition - Error in `req_url_query()`: + Error in `url_modify_query()`: ! All elements of `...` must be either an atomic vector or NULL. diff --git a/tests/testthat/_snaps/req.md b/tests/testthat/_snaps/req.md index 08bad20..e69c648 100644 --- a/tests/testthat/_snaps/req.md +++ b/tests/testthat/_snaps/req.md @@ -20,6 +20,18 @@ POST https://example.com Body: multipart encoded data +# printing headers works with {} + + Code + req_headers(request("http://test"), x = "{z}", `{z}` = "x") + Message + + GET http://test + Headers: + * x: "{z}" + * {z}: "x" + Body: empty + # individually prints repeated headers Code @@ -28,9 +40,9 @@ GET https://example.com Headers: - * A: '1' - * A: '2' - * A: '3' + * A: "1" + * A: "2" + * A: "3" Body: empty # check_request() gives useful error diff --git a/tests/testthat/_snaps/resp-stream.md b/tests/testthat/_snaps/resp-stream.md index 9b3cdf2..11dfe94 100644 --- a/tests/testthat/_snaps/resp-stream.md +++ b/tests/testthat/_snaps/resp-stream.md @@ -1,3 +1,11 @@ +# can determine if incomplete data is complete + + Code + expect_equal(resp_stream_sse(con), NULL) + Condition + Warning: + Premature end of input; ignoring final partial chunk + # can't read from a closed connection Code @@ -6,3 +14,43 @@ Error in `resp_stream_raw()`: ! `resp` has already been closed. +# verbosity = 2 streams request bodies + + Code + stream_all(req, resp_stream_lines, 1) + Output + << line 1 + + << line 2 + + Code + stream_all(req, resp_stream_raw, 5 / 1024) + Output + << Streamed 5 bytes + + << Streamed 5 bytes + + << Streamed 4 bytes + + +# verbosity = 3 shows buffer info + + Code + while (!resp_stream_is_complete(con)) { + resp_stream_lines(con, 1) + } + Output + * Buffer to parse: + * Received chunk: 6c 69 6e 65 20 31 0a 6c 69 6e 65 20 32 0a + * Combined buffer: 6c 69 6e 65 20 31 0a 6c 69 6e 65 20 32 0a + * Buffer to parse: 6c 69 6e 65 20 31 0a 6c 69 6e 65 20 32 0a + * Matched data: 6c 69 6e 65 20 31 0a + * Remaining buffer: 6c 69 6e 65 20 32 0a + << line 1 + + * Buffer to parse: 6c 69 6e 65 20 32 0a + * Matched data: 6c 69 6e 65 20 32 0a + * Remaining buffer: + << line 2 + + diff --git a/tests/testthat/_snaps/url.md b/tests/testthat/_snaps/url.md index a80e0ec..79b0658 100644 --- a/tests/testthat/_snaps/url.md +++ b/tests/testthat/_snaps/url.md @@ -24,18 +24,107 @@ Error in `url_build()`: ! Cannot set url `password` without `username`. +# url_build validates its input + + Code + url_build("abc") + Condition + Error in `url_build()`: + ! `url` must be a parsed URL, not the string "abc". + +# url_modify checks its inputs + + Code + url_modify(1) + Condition + Error in `url_modify()`: + ! `url` must be a string or parsed URL, not the number 1. + Code + url_modify(url, scheme = 1) + Condition + Error in `url_modify()`: + ! `scheme` must be a single string or `NULL`, not the number 1. + Code + url_modify(url, hostname = 1) + Condition + Error in `url_modify()`: + ! `hostname` must be a single string or `NULL`, not the number 1. + Code + url_modify(url, port = "x") + Condition + Error in `url_modify()`: + ! `port` must be a whole number or `NULL`, not the string "x". + Code + url_modify(url, username = 1) + Condition + Error in `url_modify()`: + ! `username` must be a single string or `NULL`, not the number 1. + Code + url_modify(url, password = 1) + Condition + Error in `url_modify()`: + ! `password` must be a single string or `NULL`, not the number 1. + Code + url_modify(url, path = 1) + Condition + Error in `url_modify()`: + ! `path` must be a single string or `NULL`, not the number 1. + Code + url_modify(url, fragment = 1) + Condition + Error in `url_modify()`: + ! `fragment` must be a single string or `NULL`, not the number 1. + +# checks various query formats + + Code + url_modify(url, query = 1) + Condition + Error in `url_modify()`: + ! `query` must be a character vector, named list, or NULL, not the number 1. + Code + url_modify(url, query = list(1)) + Condition + Error in `url_modify()`: + ! `query` must be a character vector, named list, or NULL, not a list. + Code + url_modify(url, query = list(x = 1:2)) + Condition + Error in `url_modify()`: + ! Query value `query$x` must be a length-1 atomic vector, not an integer vector. + # validates inputs Code - query_build(1:3) + url_modify_query(1) Condition - Error: - ! Query must be a named list. + Error in `url_modify_query()`: + ! `.url` must be a string or parsed URL, not the number 1. Code - query_build(list(x = 1:2, y = 1:3)) + url_modify_query(url, 1) Condition - Error: - ! Query value `x` must be a length-1 atomic vector, not an integer vector. + Error in `url_modify_query()`: + ! All components of `...` must be named. + Code + url_modify_query(url, x = 1:2) + Condition + Error in `url_modify_query()`: + ! All vector elements of `...` must be length 1. + i Use `.multi` to choose a strategy for handling vectors. + +--- + + Code + url_query_build(1:3) + Condition + Error in `url_query_build()`: + ! `query` must be a named list, not an integer vector. + Code + url_query_build(list(x = 1:2, y = 1:3)) + Condition + Error in `url_query_build()`: + ! All vector elements of `query` must be length 1. + i Use `.multi` to choose a strategy for handling vectors. # can't opt out of escaping non strings diff --git a/tests/testthat/helper-promise.R b/tests/testthat/helper-promise.R new file mode 100644 index 0000000..f4beb8a --- /dev/null +++ b/tests/testthat/helper-promise.R @@ -0,0 +1,26 @@ +# promises package test helper +extract_promise <- function(promise, timeout = 30) { + promise_value <- NULL + error <- NULL + promises::then( + promise, + onFulfilled = function(value) promise_value <<- value, + onRejected = function(reason) { + error <<- reason + } + ) + + start <- Sys.time() + while (!later::loop_empty()) { + if (difftime(Sys.time(), start, units = "secs") > timeout) { + stop("Waited too long") + } + later::run_now() + Sys.sleep(0.01) + } + + if (!is.null(error)) { + cnd_signal(error) + } else + promise_value +} diff --git a/tests/testthat/helper.R b/tests/testthat/helper.R index 0eef785..128daaf 100644 --- a/tests/testthat/helper.R +++ b/tests/testthat/helper.R @@ -1,3 +1,3 @@ testthat::set_state_inspector(function() { - getAllConnections() + list(connections = getAllConnections()) }) diff --git a/tests/testthat/test-curl.R b/tests/testthat/test-curl.R index 225983a..a9cff6a 100644 --- a/tests/testthat/test-curl.R +++ b/tests/testthat/test-curl.R @@ -51,7 +51,11 @@ test_that("can handle line breaks", { test_that("headers are parsed", { expect_equal( curl_normalize("curl http://x.com -H 'A: 1'")$headers, - as_headers("A: 1") + new_headers(list(A = "1")) + ) + expect_equal( + curl_normalize("curl http://x.com -H 'B:'")$headers, + new_headers(list(B = "")) ) }) @@ -138,6 +142,31 @@ test_that("can translate data", { }) }) +test_that("can translate ocokies", { + skip_if(getRversion() < "4.1") + + expect_snapshot({ + curl_translate("curl 'http://test' -H 'Cookie: x=1; y=2;z=3'") + }) +}) + +test_that("can translate json", { + skip_if(getRversion() < "4.1") + + expect_snapshot({ + curl_translate(r"--{curl http://example.com --data-raw '{"a": 1, "b": "text"}' -H Content-Type:application/json}--") + curl_translate(r"--{curl http://example.com --json '{"a": 1, "b": "text"}'}--") + }) +}) + +test_that("content type stays in header if no data", { + skip_if(getRversion() < "4.1") + + expect_snapshot( + curl_translate("curl http://example.com -H Content-Type:text/plain") + ) +}) + test_that("can evaluate simple calls", { request_test() # hack to start server diff --git a/tests/testthat/test-multi-req.R b/tests/testthat/test-multi-req.R index 9b637ec..1650e02 100644 --- a/tests/testthat/test-multi-req.R +++ b/tests/testthat/test-multi-req.R @@ -9,13 +9,9 @@ test_that("correctly prepares request", { }) test_that("requests happen in parallel", { - # GHA MacOS builder seems to be very slow - skip_if( - isTRUE(as.logical(Sys.getenv("CI", "false"))) && - Sys.info()[["sysname"]] == "Darwin" - ) - + # test works best if webfakes has ample threads and keepalive reqs <- list2( + request_test("/delay/:secs", secs = 0), request_test("/delay/:secs", secs = 0.25), request_test("/delay/:secs", secs = 0.25), request_test("/delay/:secs", secs = 0.25), diff --git a/tests/testthat/test-oauth-flow-auth-code.R b/tests/testthat/test-oauth-flow-auth-code.R index 72cd8e6..9d0da0b 100644 --- a/tests/testthat/test-oauth-flow-auth-code.R +++ b/tests/testthat/test-oauth-flow-auth-code.R @@ -75,12 +75,12 @@ test_that("old args are deprecated", { expect_snapshot( redirect <- normalize_redirect_uri("http://localhost", port = 1234) ) - expect_equal(redirect$uri, "http://localhost:1234") + expect_equal(redirect$uri, "http://localhost:1234/") expect_snapshot( redirect <- normalize_redirect_uri("http://x.com", host_name = "y.com") ) - expect_equal(redirect$uri, "http://y.com") + expect_equal(redirect$uri, "http://y.com/") expect_snapshot( redirect <- normalize_redirect_uri("http://x.com", host_ip = "y.com") diff --git a/tests/testthat/test-oauth-flow.R b/tests/testthat/test-oauth-flow.R index 0bce31e..a3222f5 100644 --- a/tests/testthat/test-oauth-flow.R +++ b/tests/testthat/test-oauth-flow.R @@ -21,6 +21,24 @@ test_that("userful errors if response isn't parseable", { }) }) +test_that("can inspect the original response if response isn't parseable", { + resp1 <- response(headers = list(`content-type` = "text/plain")) + resp2 <- response_json(body = list()) + + tryCatch( + oauth_flow_parse(resp1, "test"), + httr2_oauth_parse = function(cnd) { + expect_equal(cnd$resp, resp1) + } + ) + tryCatch( + oauth_flow_parse(resp2, "test"), + httr2_oauth_parse = function(cnd) { + expect_equal(cnd$resp, resp2) + } + ) +}) + test_that("returns body if known good structure", { resp <- response_json(body = list(access_token = "10")) expect_equal(oauth_flow_parse(resp, "test"), list(access_token = "10")) diff --git a/tests/testthat/test-req-auth-aws.R b/tests/testthat/test-req-auth-aws.R index e024f12..b0e7542 100644 --- a/tests/testthat/test-req-auth-aws.R +++ b/tests/testthat/test-req-auth-aws.R @@ -34,17 +34,15 @@ test_that("signing agrees with glacier example", { aws_access_key_id = "AKIAIOSFODNN7EXAMPLE", aws_secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" ) - signature_pieces <- strsplit(paste0("Authorization=", signature$Authorization), ",")[[1]] - known <- list( - Authorization = "AWS4-HMAC-SHA256", - Credential = "AKIAIOSFODNN7EXAMPLE/20120525/us-east-1/glacier/aws4_request", - SignedHeaders = "host;x-amz-date;x-amz-glacier-version", - Signature = "3ce5b2f2fffac9262b4da9256f8d086b4aaf42eba5f111c21681a65a127b7c2a" + expected <- paste0( + "AWS4-HMAC-SHA256 ", + "Credential=AKIAIOSFODNN7EXAMPLE/20120525/us-east-1/glacier/aws4_request,", + "SignedHeaders=host;x-amz-date;x-amz-glacier-version,", + "Signature=3ce5b2f2fffac9262b4da9256f8d086b4aaf42eba5f111c21681a65a127b7c2a" ) - known_signature <- paste0(names(known), "=", known) - expect_equal(signature_pieces, known_signature) + expect_equal(signature$Authorization, expected) }) test_that("validates its inputs", { diff --git a/tests/testthat/test-req-headers.R b/tests/testthat/test-req-headers.R index 036a679..1b7f388 100644 --- a/tests/testthat/test-req-headers.R +++ b/tests/testthat/test-req-headers.R @@ -30,6 +30,8 @@ test_that("can control which headers to redact", { expect_redact(req_headers(req, a = 1L, b = 2L, .redact = c("a", "b")), c("a", "b")) expect_redact(req_headers(req, a = 1L, b = 2L, .redact = "a"), "a") + expect_redact(req_headers_redacted(req, a = 1L, b = 2L), c("a", "b")) + expect_snapshot(error = TRUE, { req_headers(req, a = 1L, b = 2L, .redact = 1L) }) diff --git a/tests/testthat/test-req-mock.R b/tests/testthat/test-req-mock.R index 457f788..453caec 100644 --- a/tests/testthat/test-req-mock.R +++ b/tests/testthat/test-req-mock.R @@ -16,7 +16,7 @@ test_that("can generate errors with mocking", { }) test_that("local_mock and with_mock are deprecated", { - expect_snapshot({ + expect_snapshot(error = TRUE, { local_mock(~ response(404)) . <- with_mock(NULL, ~ response(404)) }) diff --git a/tests/testthat/test-req-perform.R b/tests/testthat/test-req-perform.R index 0dbafcc..11a750a 100644 --- a/tests/testthat/test-req-perform.R +++ b/tests/testthat/test-req-perform.R @@ -55,11 +55,11 @@ test_that("persistent HTTP errors only get single attempt", { }) test_that("don't retry curl errors by default", { - req <- request("") %>% req_retry(max_tries = 2) + req <- request("") %>% req_retry(max_tries = 2, failure_realm = "x") expect_error(req_perform(req), class = "httr2_failure") # But can opt-in to it - req <- request("") %>% req_retry(max_tries = 2, retry_on_failure = TRUE) + req <- request("") %>% req_retry(max_tries = 2, retry_on_failure = TRUE, failure_realm = "x") cnd <- catch_cnd(req_perform(req), "httr2_retry") expect_equal(cnd$tries, 1) }) @@ -225,3 +225,8 @@ test_that("authorization headers are redacted", { req_dry_run() }) }) + +test_that("doen't add space to urls (#567)", { + req <- request("https://example.com/test:1:2") + expect_output(req_dry_run(req), "test:1:2") +}) diff --git a/tests/testthat/test-req-promise.R b/tests/testthat/test-req-promise.R index 841d9c9..a869329 100644 --- a/tests/testthat/test-req-promise.R +++ b/tests/testthat/test-req-promise.R @@ -1,29 +1,13 @@ -# promises package test helper -extract_promise <- function(promise, timeout = 30) { - promise_value <- NULL - error <- NULL - promises::then( - promise, - onFulfilled = function(value) promise_value <<- value, - onRejected = function(reason) { - error <<- reason - } - ) - - start <- Sys.time() - while (!later::loop_empty()) { - if (difftime(Sys.time(), start, units = "secs") > timeout) { - stop("Waited too long") - } - later::run_now() - Sys.sleep(0.01) - } - - if (!is.null(error)) { - cnd_signal(error) - } else - promise_value -} +test_that("checks its inputs", { + req <- request_test("/status/:status", status = 200) + + expect_snapshot(error = TRUE, { + req_perform_promise(1) + req_perform_promise(req, path = 1) + req_perform_promise(req, pool = "INVALID") + req_perform_promise(req, verbosity = "INVALID") + }) +}) test_that("returns a promise that resolves", { p1 <- req_perform_promise(request_test("/delay/:secs", secs = 0.25)) @@ -42,6 +26,16 @@ test_that("correctly prepares request", { expect_no_error(extract_promise(prom)) }) +test_that("correctly prepares request", { + req <- request_test("/get") + expect_snapshot( + . <- extract_promise(req_perform_promise(req, verbosity = 1)), + transform = function(x) { + gsub("(Date|Host|User-Agent|ETag|Content-Length|Accept-Encoding): .*", "\\1: ", x) + } + ) +}) + test_that("can promise to download files", { req <- request_test("/json") path <- withr::local_tempfile() @@ -81,12 +75,6 @@ test_that("both curl and HTTP errors in promises are rejected", { ), class = "httr2_failure" ) - expect_error( - extract_promise( - req_perform_promise(request_test("/status/:status", status = 200), pool = "INVALID") - ), - 'inherits\\(pool, "curl_multi"\\) is not TRUE' - ) }) test_that("req_perform_promise doesn't leave behind poller", { diff --git a/tests/testthat/test-req-retries.R b/tests/testthat/test-req-retries.R index c59b3dc..d2e7d5d 100644 --- a/tests/testthat/test-req-retries.R +++ b/tests/testthat/test-req-retries.R @@ -1,3 +1,10 @@ +test_that("has useful default (with message)", { + req <- request_test() + expect_snapshot(req <- req_retry(req)) + expect_equal(retry_max_tries(req), 2) + expect_equal(retry_max_seconds(req), Inf) +}) + test_that("can set define maximum retries", { req <- request_test() expect_equal(retry_max_tries(req), 1) @@ -70,9 +77,9 @@ test_that("validates its inputs", { req <- new_request("http://example.com") expect_snapshot(error = TRUE, { - req_retry(req, max_tries = 1) - req_retry(req, max_seconds = "x") - req_retry(req, retry_on_failure = "x") + req_retry(req, max_tries = 0) + req_retry(req, max_tries = 2, max_seconds = "x") + req_retry(req, max_tries = 2, retry_on_failure = "x") }) }) @@ -85,3 +92,33 @@ test_that("is_number_or_na implemented correctly", { expect_equal(is_number_or_na(numeric()), FALSE) expect_equal(is_number_or_na("x"), FALSE) }) + + +# circuit breaker -------------------------------------------------------- + +test_that("triggered after specified requests", { + req <- request_test("/status/:status", status = 429) %>% + req_retry( + after = function(resp) 0, + max_tries = 10, + failure_threshold = 1 + ) + + # First attempt performs, retries, then errors + req_perform(req) %>% + expect_condition(class = "httr2_perform") %>% + expect_condition(class = "httr2_retry") %>% + expect_error(class = "httr2_breaker") + + # Second attempt errors without performing + req_perform(req) %>% + expect_no_condition(class = "httr2_perform") %>% + expect_error(class = "httr2_breaker") + + # Attempt on same realm errors without trying at all + req2 <- request_test("/status/:status", status = 200) %>% + req_retry() + req_perform(req) %>% + expect_no_condition(class = "httr2_perform") %>% + expect_error(class = "httr2_breaker") +}) diff --git a/tests/testthat/test-req-url.R b/tests/testthat/test-req-url.R index aa55bcd..759c33a 100644 --- a/tests/testthat/test-req-url.R +++ b/tests/testthat/test-req-url.R @@ -52,6 +52,17 @@ test_that("can set query params", { expect_equal(req_url_query(req, !!!list(a = 1, a = 2))$url, "http://example.com/?a=1&a=2") }) +test_that("can control space handling", { + req <- request("http://example.com/") + expect_equal(req_url_query(req, a = " ")$url, "http://example.com/?a=%20") + expect_equal(req_url_query(req, a = " ", .space = "form")$url, "http://example.com/?a=+") + + expect_snapshot( + req_url_query(req, a = " ", .space = "bar"), + error = TRUE + ) +}) + test_that("can handle multi query params", { req <- request("http://example.com/") @@ -96,6 +107,12 @@ test_that("can opt-out of query escaping", { expect_equal(req_url_query(req, a = I(","))$url, "http://example.com/?a=,") }) +test_that("can construct relative urls", { + req <- request("http://example.com/a/b/c.html") + expect_equal(req_url_relative(req, ".")$url, "http://example.com/a/b/") + expect_equal(req_url_relative(req, "..")$url, "http://example.com/a/") + expect_equal(req_url_relative(req, "/d/e/f")$url, "http://example.com/d/e/f") +}) # explode ----------------------------------------------------------------- test_that("explode handles expected inputs", { diff --git a/tests/testthat/test-req.R b/tests/testthat/test-req.R index a445930..600d6b1 100644 --- a/tests/testthat/test-req.R +++ b/tests/testthat/test-req.R @@ -7,6 +7,10 @@ test_that("req has basic print method", { }) }) +test_that("printing headers works with {}", { + expect_snapshot(req_headers(request("http://test"), x = "{z}", `{z}` = "x")) +}) + test_that("individually prints repeated headers", { expect_snapshot(request("https://example.com") %>% req_headers(A = 1:3)) }) @@ -18,7 +22,7 @@ test_that("print method obfuscates Authorization header unless requested", { expect_false(any(grepl("SECRET", output, fixed = TRUE))) output <- testthat::capture_messages(print(req, redact_headers = FALSE)) - expect_true(any(grepl("Authorization: 'Basic", output, fixed = TRUE))) + expect_true(any(grepl("Authorization: \"Basic", output, fixed = TRUE))) expect_false(any(grepl("REDACTED", output, fixed = TRUE))) }) diff --git a/tests/testthat/test-resp-headers.R b/tests/testthat/test-resp-headers.R index 52c9c42..cfc3070 100644 --- a/tests/testthat/test-resp-headers.R +++ b/tests/testthat/test-resp-headers.R @@ -58,3 +58,12 @@ test_that("can extract specified link url", { expect_equal(resp_link_url(resp, "first"), NULL) expect_equal(resp_link_url(response(), "first"), NULL) }) + +test_that("can extract from multiple link headers", { + resp <- response(headers = c( + 'Link: ; rel="next"', + 'Link: ; rel="last"' + )) + expect_equal(resp_link_url(resp, "next"), "https://example.com/1") + expect_equal(resp_link_url(resp, "last"), "https://example.com/2") +}) diff --git a/tests/testthat/test-resp-request.R b/tests/testthat/test-resp-request.R new file mode 100644 index 0000000..e1f2644 --- /dev/null +++ b/tests/testthat/test-resp-request.R @@ -0,0 +1,5 @@ +test_that("can extract request", { + req <- request_test() + resp <- req_perform(req) + expect_equal(resp_request(resp), req) +}) diff --git a/tests/testthat/test-resp-stream.R b/tests/testthat/test-resp-stream.R index 718b824..a476de3 100644 --- a/tests/testthat/test-resp-stream.R +++ b/tests/testthat/test-resp-stream.R @@ -1,4 +1,3 @@ - test_that("can stream bytes from a connection", { resp <- request_test("/stream-bytes/2048") %>% req_perform_connection() withr::defer(close(resp)) @@ -16,6 +15,39 @@ test_that("can stream bytes from a connection", { expect_length(out, 0) }) +test_that("can determine if a stream is complete (blocking)", { + resp <- request_test("/stream-bytes/2048") %>% req_perform_connection() + withr::defer(close(resp)) + + expect_false(resp_stream_is_complete(resp)) + expect_length(resp_stream_raw(resp, kb = 2), 2048) + expect_length(resp_stream_raw(resp, kb = 1), 0) + expect_true(resp_stream_is_complete(resp)) +}) + +test_that("can determine if a stream is complete (non-blocking)", { + resp <- request_test("/stream-bytes/2048") %>% req_perform_connection(blocking = FALSE) + withr::defer(close(resp)) + + expect_false(resp_stream_is_complete(resp)) + expect_length(resp_stream_raw(resp, kb = 2), 2048) + expect_length(resp_stream_raw(resp, kb = 1), 0) + expect_true(resp_stream_is_complete(resp)) +}) + +test_that("can determine if incomplete data is complete", { + req <- local_app_request(function(req, res) { + res$send_chunk("data: 1\n\n") + res$send_chunk("data: ") + }) + + con <- req %>% req_perform_connection(blocking = TRUE) + expect_equal(resp_stream_sse(con, 10), list(type = "message", data = "1", id = character())) + expect_snapshot(expect_equal(resp_stream_sse(con), NULL)) + expect_true(resp_stream_is_complete(con)) + close(con) +}) + test_that("can't read from a closed connection", { resp <- request_test("/stream-bytes/1024") %>% req_perform_connection() close(resp) @@ -42,7 +74,7 @@ test_that("can join lines across multiple reads", { expect_equal(out, character()) expect_equal(resp1$cache$push_back, charToRaw("This is a ")) - while(length(out) == 0) { + while (length(out) == 0) { Sys.sleep(0.1) out <- resp_stream_lines(resp1) } @@ -147,7 +179,7 @@ test_that("streams the specified number of lines", { test_that("can feed sse events one at a time", { req <- local_app_request(function(req, res) { - for(i in 1:3) { + for (i in 1:3) { res$send_chunk(sprintf("data: %s\n\n", i)) } }) @@ -185,7 +217,7 @@ test_that("can join sse events across multiple reads", { expect_equal(out, NULL) expect_equal(resp1$cache$push_back, charToRaw("data: 1\n")) - while(is.null(out)) { + while (is.null(out)) { Sys.sleep(0.1) out <- resp_stream_sse(resp1) } @@ -213,7 +245,7 @@ test_that("sse always interprets data as UTF-8", { withr::defer(close(resp1)) out <- NULL - while(is.null(out)) { + while (is.null(out)) { Sys.sleep(0.1) out <- resp_stream_sse(resp1) } @@ -236,7 +268,7 @@ test_that("streaming size limits enforced", { resp1 <- req_perform_connection(req, blocking = FALSE) withr::defer(close(resp1)) expect_error( - while(is.null(out)) { + while (is.null(out)) { Sys.sleep(0.1) out <- resp_stream_sse(resp1, max_size = 999) } @@ -255,6 +287,47 @@ test_that("streaming size limits enforced", { ) }) +test_that("verbosity = 2 streams request bodies", { + req <- local_app_request(function(req, res) { + res$send_chunk("line 1\n") + res$send_chunk("line 2\n") + }) + + stream_all <- function(req, fun, ...) { + con <- req_perform_connection(req, blocking = TRUE, verbosity = 2) + on.exit(close(con)) + while (!resp_stream_is_complete(con)) { + fun(con, ...) + } + } + expect_snapshot( + { + stream_all(req, resp_stream_lines, 1) + stream_all(req, resp_stream_raw, 5 / 1024) + }, + transform = function(lines) lines[!grepl("^(<-|->)", lines)] + ) +}) + +test_that("verbosity = 3 shows buffer info", { + req <- local_app_request(function(req, res) { + res$send_chunk("line 1\n") + res$send_chunk("line 2\n") + }) + + con <- req_perform_connection(req, blocking = TRUE, verbosity = 3) + on.exit(close(con)) + expect_snapshot( + { + while (!resp_stream_is_complete(con)) { + resp_stream_lines(con, 1) + } + }, + transform = function(lines) lines[!grepl("^(<-|->)", lines)] + ) +}) + + test_that("has a working find_event_boundary", { boundary_test <- function(x, matched, remaining) { buffer <- charToRaw(x) @@ -266,7 +339,7 @@ test_that("has a working find_event_boundary", { } expect_identical( result, - list(matched=charToRaw(matched), remaining = charToRaw(remaining)) + list(matched = charToRaw(matched), remaining = charToRaw(remaining)) ) } diff --git a/tests/testthat/test-url.R b/tests/testthat/test-url.R index 9dc3de0..153309d 100644 --- a/tests/testthat/test-url.R +++ b/tests/testthat/test-url.R @@ -1,22 +1,11 @@ test_that("can parse special cases", { - url <- url_parse("//google.com") - expect_equal(url$scheme, NULL) - expect_equal(url$hostname, "google.com") - url <- url_parse("file:///tmp") expect_equal(url$scheme, "file") expect_equal(url$path, "/tmp") - - url <- url_parse("/") - expect_equal(url$scheme, NULL) - expect_equal(url$path, "/") }) test_that("can round trip urls", { urls <- list( - "/", - "//google.com", - "file:///", "http://google.com/", "http://google.com/path", "http://google.com/path?a=1&b=2", @@ -31,47 +20,191 @@ test_that("can round trip urls", { expect_equal(map(urls, ~ url_build(url_parse(.x))), urls) }) +test_that("can parse relative urls", { + base <- "http://example.com/a/b/c/" + expect_equal(url_parse("d", base)$path, "/a/b/c/d") + expect_equal(url_parse("..", base)$path, "/a/b/") + + expect_equal(url_parse("//archive.org", base)$scheme, "http") +}) + test_that("can print all url details", { expect_snapshot( url_parse("http://user:pass@example.com:80/path?a=1&b=2&c={1{2}3}#frag") ) }) -test_that("ensures path always starts with /", { - expect_equal( - url_modify("https://example.com/abc", path = "def"), - "https://example.com/def" - ) -}) - test_that("password also requires username", { url <- url_parse("http://username:pwd@example.com") url$username <- NULL expect_snapshot(url_build(url), error = TRUE) +}) +test_that("url_build validates its input", { + expect_snapshot(url_build("abc"), error = TRUE) +}) + +test_that("decodes query params but not paths", { + url <- url_parse("http://example.com/a%20b?q=a%20b") + expect_equal(url$path, "/a%20b") + expect_equal(url$query$q, "a b") +}) + +# modify url ------------------------------------------------------------- + +test_that("url_modify checks its inputs", { + url <- "http://example.com" + + expect_snapshot(error = TRUE, { + url_modify(1) + url_modify(url, scheme = 1) + url_modify(url, hostname = 1) + url_modify(url, port = "x") + url_modify(url, username = 1) + url_modify(url, password = 1) + url_modify(url, path = 1) + url_modify(url, fragment = 1) + }) +}) + +test_that("no arguments is idempotent", { + string <- "http://example.com/" + url <- url_parse(string) + + expect_equal(url_modify(string), string) + expect_equal(url_modify(url), url) +}) + +test_that("can round-trip escaped components", { + url <- "https://example.com/a%20b" + expect_equal(url_modify(url), url) + + url <- "https://example.com/?q=a%20b" + expect_equal(url_modify(url), url) +}) + +test_that("can accept query as a string or list", { + url <- "http://test/" + + expect_equal(url_modify(url, query = "a=1&b=2"), "http://test/?a=1&b=2") + expect_equal(url_modify(url, query = list(a = 1, b = 2)), "http://test/?a=1&b=2") + + expect_equal(url_modify(url, query = ""), "http://test/") + expect_equal(url_modify(url, query = list()), "http://test/") +}) + +test_that("automatically escapes query components", { + expect_equal( + url_modify("https://example.com", query = list(q = "a b")), + "https://example.com/?q=a%20b" + ) +}) + +test_that("checks various query formats", { + url <- "http://example.com" + + expect_snapshot(error = TRUE, { + url_modify(url, query = 1) + url_modify(url, query = list(1)) + url_modify(url, query = list(x = 1:2)) + }) }) +test_that("path always starts with /", { + expect_equal(url_modify("https://x.com/abc", path = "def"), "https://x.com/def") + expect_equal(url_modify("https://x.com/abc", path = ""), "https://x.com/") + expect_equal(url_modify("https://x.com/abc", path = NULL), "https://x.com/") +}) + +# relative url ------------------------------------------------------------ + +test_that("can set relative urls", { + base <- "http://example.com/a/b/c/" + expect_equal(url_modify_relative(base, "d"), "http://example.com/a/b/c/d") + expect_equal(url_modify_relative(base, ".."), "http://example.com/a/b/") + expect_equal(url_modify_relative(base, "//archive.org"), "http://archive.org/") +}) + +test_that("is idempotent", { + string <- "http://example.com/" + url <- url_parse(string) + + expect_equal(url_modify_relative(string, "."), string) + expect_equal(url_modify_relative(url, "."), url) +}) + +# modify query ------------------------------------------------------------- + +test_that("can add, modify, and delete query components", { + expect_equal( + url_modify_query("http://test/path", new = "value"), + "http://test/path?new=value" + ) + expect_equal( + url_modify_query("http://test/path", new = "one", new = "two"), + "http://test/path?new=one&new=two" + ) + expect_equal( + url_modify_query("http://test/path?a=old&b=old", a = "new"), + "http://test/path?b=old&a=new" + ) + expect_equal( + url_modify_query("http://test/path?remove=me&keep=this", remove = NULL), + "http://test/path?keep=this" + ) +}) + +test_that("can control space formatting", { + expect_equal( + url_modify_query("http://test/path", new = "a b"), + "http://test/path?new=a%20b" + ) + expect_equal( + url_modify_query("http://test/path", new = "a b", .space = "form"), + "http://test/path?new=a+b" + ) +}) + +test_that("is idempotent", { + string <- "http://example.com/" + url <- url_parse(string) + + expect_equal(url_modify_query(string), string) + expect_equal(url_modify_query(url), url) +}) + +test_that("validates inputs", { + url <- "http://example.com/" + + expect_snapshot(error = TRUE, { + url_modify_query(1) + url_modify_query(url, 1) + url_modify_query(url, x = 1:2) + }) +}) + + # query ------------------------------------------------------------------- test_that("missing query values become empty strings", { - expect_equal(query_parse("?q="), list(q = "")) - expect_equal(query_parse("?q"), list(q = "")) - expect_equal(query_parse("?a&q"), list(a = "", q = "")) + expect_equal(url_query_parse("?q="), list(q = "")) + expect_equal(url_query_parse("?q"), list(q = "")) + expect_equal(url_query_parse("?a&q"), list(a = "", q = "")) }) test_that("handles equals in values", { - expect_equal(query_parse("?x==&y=="), list(x = "=", y = "=")) - }) + expect_equal(url_query_parse("?x==&y=="), list(x = "=", y = "=")) +}) test_that("empty queries become NULL", { - expect_equal(query_parse("?"), NULL) - expect_equal(query_parse(""), NULL) + expect_equal(url_query_parse("?"), NULL) + expect_equal(url_query_parse(""), NULL) }) test_that("validates inputs", { expect_snapshot(error = TRUE, { - query_build(1:3) - query_build(list(x = 1:2, y = 1:3)) + url_query_build(1:3) + url_query_build(list(x = 1:2, y = 1:3)) }) })