Fix handling of binary input (#191)

Gracefully handle binary input with from_utf8_lossy. Fixes #189
rcoh · May 19, 2023 · d38fba9 · d38fba9
1 parent 27c2c38
commit d38fba9
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 2 deletions.
diff --git a/src/lib.rs b/src/lib.rs
@@ -283,8 +283,8 @@ pub mod pipeline {
  if ct == 0 {
  break;
  }
- let data = std::str::from_utf8(&line[..ct]).unwrap();
- if self.filter.matches(data)
+ let data = String::from_utf8_lossy(&line[..ct]);
+ if self.filter.matches(data.as_ref())
  && !Pipeline::proc_preagg(Record::new(data), &mut preaggs, &tx)
  {
  break;

diff --git a/test_files/binary_data.bin b/test_files/binary_data.bin
@@ -0,0 +1,3 @@
+line of regular text k=v2
+�K���@�V���+P��Q�Þ�0����S��oU�!+��X-!����޴A�� �r��[�OC��j_kΕ����V����5; �x1rYuV���
+line of regular text k=v
diff --git a/tests/integration.rs b/tests/integration.rs
@@ -146,6 +146,18 @@ None 1\n",
  );
  }
 
+ #[test]
+ fn binary_input() {
+ run()
+ .args([
+ "* | parse 'k=*' as k",
+ "--file",
+ "test_files/binary_data.bin",
+ ])
+ .assert()
+ .stdout("[k=v2]\n[k=v]\n");
+ }
+
  #[test]
  fn filter_wildcard() {
  run()