Add --preserve-encoding option to keep response body encoding

src/cli.rs

@@ -159,6 +159,13 @@ Defaults to \"format\" if the NO_COLOR env is set and to \"none\" if stdout is n
  #[clap(short = 'd', long)]
  pub download: bool,
 
+ /// During download, keep the raw encoding of the body. Intended for use with download or
+ /// to debug an encoded response body.
+ ///
+ /// For example, set Accept-Encoding: gzip and use --preserve-encoding to skip decompression.
+ #[clap(long)]
+ pub preserve_encoding: bool,
+
  /// Resume an interrupted download. Requires --download and --output.
  #[clap(
  short = 'c',

src/download.rs

@@ -160,6 +160,7 @@ const UNCOLORED_SPINNER_TEMPLATE: &str = "{spinner} {bytes} {bytes_per_sec} {wid
 
 pub fn download_file(
  mut response: Response,
+ preserve_encoding: bool,
  file_name: Option<PathBuf>,
  // If we fall back on taking the filename from the URL it has to be the
  // original URL, before redirects. That's less surprising and matches
@@ -244,9 +245,13 @@ pub fn download_file(
  pb.reset_eta();
  }
 
+ let compression_type = if !preserve_encoding {
+ get_compression_type(response.headers())
+ } else {
+ None
+ };
  match pb {
  Some(ref pb) => {
- let compression_type = get_compression_type(response.headers());
  copy_largebuf(
  &mut decompress(&mut pb.wrap_read(response), compression_type),
  &mut buffer,
@@ -267,7 +272,6 @@ pub fn download_file(
  }
  }
  None => {
- let compression_type = get_compression_type(response.headers());
  copy_largebuf(
  &mut decompress(&mut response, compression_type),
  &mut buffer,

src/main.rs

@@ -354,11 +354,25 @@ fn run(args: Cli) -> Result<i32> {
  let mut request = {
  let mut request_builder = client
  .request(method, url.clone())
- .header(
+ .header(USER_AGENT, get_user_agent());
+
+ if args.download {
+ if let Some(encoding) = headers.get(ACCEPT_ENCODING) {
+ if args.resume && encoding != HeaderValue::from_static("identity") {
+ return Err(anyhow!(
+ "Cannot use --continue with --download, when the encoding is not 'identity'"
+ ));
+ }
+ } else {
+ request_builder =
+ request_builder.header(ACCEPT_ENCODING, HeaderValue::from_static("identity"));
+ }
+ } else {
+ request_builder = request_builder.header(
  ACCEPT_ENCODING,
  HeaderValue::from_static("gzip, deflate, br"),
- )
-  .header(USER_AGENT, get_user_agent());
+ );
+ }
 
  if matches!(
  args.http_version,
@@ -465,12 +479,6 @@ fn run(args: Cli) -> Result<i32> {
  request
  };
 
- if args.download {
- request
- .headers_mut()
- .insert(ACCEPT_ENCODING, HeaderValue::from_static("identity"));
- }
-
  let buffer = Buffer::new(
  args.download,
  args.output.as_deref(),
@@ -501,6 +509,7 @@ fn run(args: Cli) -> Result<i32> {
  printer.print_request_body(&mut request)?;
  }
 
+ let preserve_encoding = args.preserve_encoding;
  if !args.offline {
  let mut response = {
  let history_print = args.history_print.unwrap_or(print);
@@ -515,6 +524,7 @@ fn run(args: Cli) -> Result<i32> {
  prev_response,
  response_charset,
  response_mime,
+ preserve_encoding,
  )?;
  printer.print_separator()?;
  }
@@ -556,6 +566,7 @@ fn run(args: Cli) -> Result<i32> {
  if exit_code == 0 {
  download_file(
  response,
+ args.preserve_encoding,
  args.output,
  &url,
  resume,
@@ -564,7 +575,12 @@ fn run(args: Cli) -> Result<i32> {
  )?;
  }
  } else if print.response_body {
- printer.print_response_body(&mut response, response_charset, response_mime)?;
+ printer.print_response_body(
+ &mut response,
+ response_charset,
+ response_mime,
+ args.preserve_encoding,
+ )?;
  }
  }
 

src/printer.rs

@@ -428,15 +428,33 @@ impl Printer {
  response: &mut Response,
  encoding: Option<&'static Encoding>,
  mime: Option<&str>,
+ preserve_encoding: bool,
  ) -> anyhow::Result<()> {
  let url = response.url().clone();
  let content_type =
  mime.map_or_else(|| get_content_type(response.headers()), ContentType::from);
  let encoding = encoding.or_else(|| get_charset(response));
- let compression_type = get_compression_type(response.headers());
+ let (may_decode, compression_type) = if !preserve_encoding {
+ let compression_type = get_compression_type(response.headers());
+ (true, compression_type)
+ } else {
+ (false, None)
+ };
  let mut body = decompress(response, compression_type);
 
- if !self.buffer.is_terminal() {
+ if !may_decode {
+ // The user explicitly asked for preserving the encoding. We don't
+ // decode the body, even if it's text, so we can't write it to the terminal.
+ if self.buffer.is_terminal() {
+ self.buffer.print(BINARY_SUPPRESSOR)?;
+ } else if self.stream {
+ copy_largebuf(&mut body, &mut self.buffer, true)?;
+ } else {
+ let mut buf = Vec::new();
+ body.read_to_end(&mut buf)?;
+ self.buffer.print(&buf)?;
+ }
+ } else if !self.buffer.is_terminal() {
  if (self.color || self.indent_json) && content_type.is_text() {
  // The user explicitly asked for formatting even though this is
  // going into a file, and the response is at least supposed to be

tests/cli.rs

@@ -413,7 +413,7 @@ fn download() {
 }
 
 #[test]
-fn accept_encoding_not_modifiable_in_download_mode() {
+fn accept_encoding_identity_default_in_download_mode() {
  let server = server::http(|req| async move {
  assert_eq!(req.headers()["accept-encoding"], "identity");
  hyper::Response::builder()
@@ -424,11 +424,55 @@ fn accept_encoding_not_modifiable_in_download_mode() {
  let dir = tempdir().unwrap();
  get_command()
  .current_dir(&dir)
- .args([&server.base_url(), "--download", "accept-encoding:gzip"])
+ .args([&server.base_url(), "--download"])
  .assert()
  .success();
 }
 
+#[test]
+fn accept_encoding_identity_in_resumable_download_mode() {
+ let server = server::http(|req| async move {
+ assert_eq!(req.headers()["accept-encoding"], "identity");
+ hyper::Response::builder()
+ .body(r#"{"ids":[1,2,3]}"#.into())
+ .unwrap()
+ });
+
+ let dir = tempdir().unwrap();
+ get_command()
+ .current_dir(&dir)
+ .args([
+ &server.base_url(),
+ "--download",
+ "--output",
+ "foo",
+ "--continue",
+ "accept-encoding:identity",
+ ])
+ .assert()
+ .success();
+}
+
+#[test]
+fn accept_encoding_not_modifiable_in_resumable_download_mode() {
+ let dir = tempdir().unwrap();
+ get_command()
+ .current_dir(&dir)
+ .args([
+ ":",
+ "--download",
+ "--output",
+ "foo",
+ "--continue",
+ "accept-encoding:gzip",
+ ])
+ .assert()
+ .failure()
+ .stderr(indoc! {r#"
+ xh: error: Cannot use --continue with --download, when the encoding is not 'identity'
+ "#});
+}
+
 #[test]
 fn download_generated_filename() {
  let dir = tempdir().unwrap();