From 2ced9e4d1910ba2182b85a6168429c0332ca25b3 Mon Sep 17 00:00:00 2001 From: Bruce Weirdan Date: Tue, 6 Aug 2024 22:28:38 +0200 Subject: [PATCH] Add multipart/form-data uploads (#13532) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes nushell/nushell#11046 # Description This adds support for `multipart/form-data` (RFC 7578) uploads to nushell. Binary data is uploaded as files (`application/octet-stream`), everything else is uploaded as plain text. ```console $ http post https://echo.free.beeceptor.com --content-type multipart/form-data {cargo: (open -r Cargo.toml | into binary ), description: "It's some TOML"} | upsert ip "" ╭───────────────────┬─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │ method │ POST │ │ protocol │ https │ │ host │ echo.free.beeceptor.com │ │ path │ / │ │ ip │ │ │ │ ╭─────────────────┬────────────────────────────────────────────────────────────────────╮ │ │ headers │ │ Host │ echo.free.beeceptor.com │ │ │ │ │ User-Agent │ nushell │ │ │ │ │ Content-Length │ 9453 │ │ │ │ │ Accept │ */* │ │ │ │ │ Accept-Encoding │ gzip │ │ │ │ │ Content-Type │ multipart/form-data; boundary=a15f6a14-5768-4a6a-b3a4-686a112d9e27 │ │ │ │ ╰─────────────────┴────────────────────────────────────────────────────────────────────╯ │ │ parsedQueryParams │ {record 0 fields} │ │ │ ╭─────────────────┬───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │ │ parsedBody │ │ │ ╭─────────────┬────────────────╮ │ │ │ │ │ textFields │ │ description │ It's some TOML │ │ │ │ │ │ │ ╰─────────────┴────────────────╯ │ │ │ │ │ │ ╭───┬───────┬──────────┬──────────────────────────┬───────────────────────────┬───────────────────────────────────────────┬────────────────╮ │ │ │ │ │ files │ │ # │ name │ fileName │ Content-Type │ Content-Transfer-Encoding │ Content-Disposition │ Content-Length │ │ │ │ │ │ │ ├───┼───────┼──────────┼──────────────────────────┼───────────────────────────┼───────────────────────────────────────────┼────────────────┤ │ │ │ │ │ │ │ 0 │ cargo │ cargo │ application/octet-stream │ binary │ form-data; name="cargo"; filename="cargo" │ 9101 │ │ │ │ │ │ │ ╰───┴───────┴──────────┴──────────────────────────┴───────────────────────────┴───────────────────────────────────────────┴────────────────╯ │ │ │ │ ╰─────────────────┴───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │ ╰───────────────────┴─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` # User-Facing Changes `http post --content-type multipart/form-data` now accepts a record which is uploaded as `multipart/form-data`. Binary data is uploaded as files (`application/octet-stream`), everything else is uploaded as plain text. Previously `http post --content-type multipart/form-data` rejected records, so there's no BC break. # Tests + Formatting Added. # After Submitting - [ ] update docs to showcase new functionality --- Cargo.lock | 16 +++++++ Cargo.toml | 2 + crates/nu-command/Cargo.toml | 3 +- crates/nu-command/src/network/http/client.rs | 46 +++++++++++++++++++ crates/nu-command/src/network/http/post.rs | 5 ++ .../tests/commands/network/http/post.rs | 33 ++++++++++++- 6 files changed, 103 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 902210ffbcdf..ddf5c566f24e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2748,6 +2748,20 @@ dependencies = [ "tokio", ] +[[package]] +name = "multipart-rs" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ea34e5c0fa65ba84707cfaf5bf43d500f1c5a4c6c36327bf5541c5bcd17e98" +dependencies = [ + "bytes", + "futures-core", + "futures-util", + "memchr", + "mime", + "uuid", +] + [[package]] name = "multiversion" version = "0.7.4" @@ -2877,6 +2891,7 @@ dependencies = [ "log", "miette", "mimalloc", + "multipart-rs", "nix", "nu-cli", "nu-cmd-base", @@ -3064,6 +3079,7 @@ dependencies = [ "mime", "mime_guess", "mockito", + "multipart-rs", "native-tls", "nix", "notify-debouncer-full", diff --git a/Cargo.toml b/Cargo.toml index ce62fa745e33..225834de70ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -113,6 +113,7 @@ miette = "7.2" mime = "0.3" mime_guess = "2.0" mockito = { version = "1.4", default-features = false } +multipart-rs = "0.1.11" native-tls = "0.2" nix = { version = "0.28", default-features = false } notify-debouncer-full = { version = "0.3", default-features = false } @@ -207,6 +208,7 @@ dirs = { workspace = true } log = { workspace = true } miette = { workspace = true, features = ["fancy-no-backtrace", "fancy"] } mimalloc = { version = "0.1.42", default-features = false, optional = true } +multipart-rs = { workspace = true } serde_json = { workspace = true } simplelog = "0.12" time = "0.3" diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index d1defd432779..2459d74ad903 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -60,6 +60,7 @@ lscolors = { workspace = true, default-features = false, features = ["nu-ansi-te md5 = { workspace = true } mime = { workspace = true } mime_guess = { workspace = true } +multipart-rs = { workspace = true } native-tls = { workspace = true } notify-debouncer-full = { workspace = true, default-features = false } num-format = { workspace = true } @@ -146,4 +147,4 @@ quickcheck = { workspace = true } quickcheck_macros = { workspace = true } rstest = { workspace = true, default-features = false } pretty_assertions = { workspace = true } -tempfile = { workspace = true } \ No newline at end of file +tempfile = { workspace = true } diff --git a/crates/nu-command/src/network/http/client.rs b/crates/nu-command/src/network/http/client.rs index e348bedc2575..7bb63a8fbd4a 100644 --- a/crates/nu-command/src/network/http/client.rs +++ b/crates/nu-command/src/network/http/client.rs @@ -4,10 +4,12 @@ use base64::{ engine::{general_purpose::PAD, GeneralPurpose}, Engine, }; +use multipart_rs::MultipartWriter; use nu_engine::command_prelude::*; use nu_protocol::{ByteStream, Signals}; use std::{ collections::HashMap, + io::Cursor, path::PathBuf, str::FromStr, sync::mpsc::{self, RecvTimeoutError}, @@ -20,6 +22,7 @@ use url::Url; pub enum BodyType { Json, Form, + Multipart, Unknown, } @@ -210,6 +213,7 @@ pub fn send_request( let (body_type, req) = match content_type { Some(it) if it == "application/json" => (BodyType::Json, request), Some(it) if it == "application/x-www-form-urlencoded" => (BodyType::Form, request), + Some(it) if it == "multipart/form-data" => (BodyType::Multipart, request), Some(it) => { let r = request.clone().set("Content-Type", &it); (BodyType::Unknown, r) @@ -265,6 +269,48 @@ pub fn send_request( }; send_cancellable_request(&request_url, Box::new(request_fn), span, signals) } + // multipart form upload + Value::Record { val, .. } if body_type == BodyType::Multipart => { + let mut builder = MultipartWriter::new(); + + let err = |e| { + ShellErrorOrRequestError::ShellError(ShellError::IOError { + msg: format!("failed to build multipart data: {}", e), + }) + }; + + for (col, val) in val.into_owned() { + if let Value::Binary { val, .. } = val { + let headers = [ + "Content-Type: application/octet-stream".to_string(), + "Content-Transfer-Encoding: binary".to_string(), + format!( + "Content-Disposition: form-data; name=\"{}\"; filename=\"{}\"", + col, col + ), + format!("Content-Length: {}", val.len()), + ]; + builder + .add(&mut Cursor::new(val), &headers.join("\n")) + .map_err(err)?; + } else { + let headers = + format!(r#"Content-Disposition: form-data; name="{}""#, col); + builder + .add(val.coerce_into_string()?.as_bytes(), &headers) + .map_err(err)?; + } + } + builder.finish(); + + let (boundary, data) = (builder.boundary, builder.data); + let content_type = format!("multipart/form-data; boundary={}", boundary); + + let request_fn = + move || req.set("Content-Type", &content_type).send_bytes(&data); + + send_cancellable_request(&request_url, Box::new(request_fn), span, signals) + } Value::List { vals, .. } if body_type == BodyType::Form => { if vals.len() % 2 != 0 { return Err(ShellErrorOrRequestError::ShellError(ShellError::IOError { diff --git a/crates/nu-command/src/network/http/post.rs b/crates/nu-command/src/network/http/post.rs index ea4ec093f38d..a593c5489c80 100644 --- a/crates/nu-command/src/network/http/post.rs +++ b/crates/nu-command/src/network/http/post.rs @@ -127,6 +127,11 @@ impl Command for SubCommand { example: "open foo.json | http post https://www.example.com", result: None, }, + Example { + description: "Upload a file to example.com", + example: "http post --content-type multipart/form-data https://www.example.com { audio: (open -r file.mp3) }", + result: None, + }, ] } } diff --git a/crates/nu-command/tests/commands/network/http/post.rs b/crates/nu-command/tests/commands/network/http/post.rs index 02aa8df23f00..11db87168ec1 100644 --- a/crates/nu-command/tests/commands/network/http/post.rs +++ b/crates/nu-command/tests/commands/network/http/post.rs @@ -1,4 +1,4 @@ -use mockito::Server; +use mockito::{Matcher, Server, ServerOpts}; use nu_test_support::{nu, pipeline}; #[test] @@ -197,3 +197,34 @@ fn http_post_redirect_mode_error() { "Redirect encountered when redirect handling mode was 'error' (301 Moved Permanently)" )); } +#[test] +fn http_post_multipart_is_success() { + let mut server = Server::new_with_opts(ServerOpts { + assert_on_drop: true, + ..Default::default() + }); + let _mock = server + .mock("POST", "/") + .match_header( + "content-type", + Matcher::Regex("multipart/form-data; boundary=.*".to_string()), + ) + .match_body(Matcher::AllOf(vec![ + Matcher::Regex(r#"(?m)^Content-Disposition: form-data; name="foo""#.to_string()), + Matcher::Regex(r#"(?m)^Content-Type: application/octet-stream"#.to_string()), + Matcher::Regex(r#"(?m)^Content-Length: 3"#.to_string()), + Matcher::Regex(r#"(?m)^bar"#.to_string()), + ])) + .with_status(200) + .create(); + + let actual = nu!(pipeline( + format!( + "http post --content-type multipart/form-data {url} {{foo: ('bar' | into binary) }}", + url = server.url() + ) + .as_str() + )); + + assert!(actual.out.is_empty()) +}