use JSON for concat field (#4937)

use JSON for concat fields instead of concatenating text fields.

With quickwit-oss/tantivy#2383 we support now
non-object values on the root in the JSON field.
As a nice side-effect, this will make regular JSON fields more powerful.
Instead only for nested types, JSON is now also useable for flat mixed-type fields.

Closes: #4924

quickwit/quickwit-doc-mapper/src/default_doc_mapper/default_mapper.rs

@@ -730,7 +730,6 @@ mod tests {
 
  use super::DefaultDocMapper;
  use crate::default_doc_mapper::field_mapping_entry::DEFAULT_TOKENIZER_NAME;
- use crate::default_doc_mapper::mapping_tree::value_to_pretokenized;
  use crate::{
  DefaultDocMapperBuilder, DocMapper, DocParsingError, DOCUMENT_SIZE_FIELD_NAME,
  DYNAMIC_FIELD_NAME, FIELD_PRESENCE_FIELD_NAME, SOURCE_FIELD_NAME,
@@ -1815,7 +1814,7 @@ mod tests {
  }"#,
  "concat",
  r#"{"some_int": 25}"#,
- vec![value_to_pretokenized(25).into()],
+ vec![25_u64.into()],
  );
  test_doc_from_json_test_aux(
  r#"{
@@ -1830,7 +1829,7 @@ mod tests {
  }"#,
  "concat",
  r#"{"some_int": 25}"#,
- vec![value_to_pretokenized(25).into()],
+ vec![25_u64.into()],
  );
  }
 
@@ -1853,7 +1852,7 @@ mod tests {
  }"#,
  "concat",
  r#"{"some_bool": false}"#,
- vec![value_to_pretokenized(false).into()],
+ vec![false.into()],
  );
  test_doc_from_json_test_aux(
  r#"{
@@ -1868,7 +1867,7 @@ mod tests {
  }"#,
  "concat",
  r#"{"some_bool": true}"#,
- vec![value_to_pretokenized(true).into()],
+ vec![true.into()],
  );
  }
 

quickwit/quickwit-doc-mapper/src/default_doc_mapper/field_mapping_entry.rs

@@ -702,9 +702,9 @@ impl Default for QuickwitConcatenateOptions {
  }
 }
 
-impl From<QuickwitConcatenateOptions> for TextOptions {
+impl From<QuickwitConcatenateOptions> for JsonObjectOptions {
  fn from(quickwit_text_options: QuickwitConcatenateOptions) -> Self {
- let mut text_options = TextOptions::default();
+ let mut text_options = JsonObjectOptions::default();
  let text_field_indexing = TextFieldIndexing::default()
  .set_index_option(quickwit_text_options.indexing_options.record)
  .set_fieldnorms(quickwit_text_options.indexing_options.fieldnorms)

quickwit/quickwit-doc-mapper/src/default_doc_mapper/mapping_tree.rs

@@ -29,7 +29,6 @@ use tantivy::schema::{
  BytesOptions, DateOptions, Field, IntoIpv6Addr, IpAddrOptions, JsonObjectOptions,
  NumericOptions, OwnedValue as TantivyValue, SchemaBuilder, TextOptions,
 };
-use tantivy::tokenizer::{PreTokenizedString, Token};
 use tantivy::TantivyDocument as Document;
 
 use super::date_time_type::QuickwitDateTimeOptions;
@@ -54,20 +53,6 @@ pub enum LeafType {
  Text(QuickwitTextOptions),
 }
 
-pub(crate) fn value_to_pretokenized<T: ToString>(val: T) -> PreTokenizedString {
- let text = val.to_string();
- PreTokenizedString {
- text: text.clone(),
- tokens: vec![Token {
- offset_from: 0,
- offset_to: 1,
- position: 0,
- text,
- position_length: 1,
- }],
- }
-}
-
 enum MapOrArrayIter {
  Array(std::vec::IntoIter<JsonValue>),
  Map(serde_json::map::IntoIter),
@@ -161,12 +146,12 @@ pub(crate) fn map_primitive_json_to_tantivy(value: JsonValue) -> Option<TantivyV
  match value {
  JsonValue::Array(_) | JsonValue::Object(_) | JsonValue::Null => None,
  JsonValue::String(text) => Some(TantivyValue::Str(text)),
- JsonValue::Bool(val) => Some(value_to_pretokenized(val).into()),
+ JsonValue::Bool(val) => Some((val).into()),
  JsonValue::Number(number) => {
  if let Some(val) = u64::from_json_number(&number) {
- Some(value_to_pretokenized(val).into())
+ Some((val).into())
  } else {
- i64::from_json_number(&number).map(|val| value_to_pretokenized(val).into())
+ i64::from_json_number(&number).map(|val| (val).into())
  }
  }
  }
@@ -219,7 +204,7 @@ impl LeafType {
  }
  }
 
- fn tantivy_string_value_from_json(
+ fn tantivy_value_from_json(
  &self,
  json_val: JsonValue,
  ) -> Result<impl Iterator<Item = TantivyValue>, String> {
@@ -233,16 +218,16 @@ impl LeafType {
  }
  LeafType::I64(numeric_options) => {
  let val = i64::from_json_to_self(json_val, numeric_options.coerce)?;
- Ok(OneOrIter::one(value_to_pretokenized(val).into()))
+ Ok(OneOrIter::one((val).into()))
  }
  LeafType::U64(numeric_options) => {
  let val = u64::from_json_to_self(json_val, numeric_options.coerce)?;
- Ok(OneOrIter::one(value_to_pretokenized(val).into()))
+ Ok(OneOrIter::one((val).into()))
  }
  LeafType::F64(_) => Err("unsuported concat type: f64".to_string()),
  LeafType::Bool(_) => {
  if let JsonValue::Bool(val) = json_val {
- Ok(OneOrIter::one(value_to_pretokenized(val).into()))
+ Ok(OneOrIter::one((val).into()))
  } else {
  Err(format!("expected boolean, got `{json_val}`"))
  }
@@ -313,7 +298,7 @@ impl MappingLeaf {
  if !self.concatenate.is_empty() {
  let concat_values = self
  .typ
- .tantivy_string_value_from_json(el_json_val.clone())
+ .tantivy_value_from_json(el_json_val.clone())
  .map_err(|err_msg| DocParsingError::ValueError(path.join("."), err_msg))?;
  for concat_value in concat_values {
  for field in &self.concatenate {
@@ -333,7 +318,7 @@ impl MappingLeaf {
  if !self.concatenate.is_empty() {
  let concat_values = self
  .typ
- .tantivy_string_value_from_json(json_val.clone())
+ .tantivy_value_from_json(json_val.clone())
  .map_err(|err_msg| DocParsingError::ValueError(path.join("."), err_msg))?;
  for concat_value in concat_values {
  for field in &self.concatenate {
@@ -982,8 +967,8 @@ fn build_mapping_tree_from_entries<'a>(
  if mapping_node.branches.contains_key(name) {
  bail!("duplicated field definition `{}`", name);
  }
- let text_options: TextOptions = options.clone().into();
- let field = schema.add_text_field(name, text_options);
+ let text_options: JsonObjectOptions = options.clone().into();
+ let field = schema.add_json_field(name, text_options);
  for sub_field in &options.concatenate_fields {
  for matched_field in
  mapping_node

quickwit/quickwit-query/src/query_ast/utils.rs

@@ -54,13 +54,7 @@ pub fn find_field_or_hit_dynamic<'a>(
  };
  let field_entry = schema.get_field_entry(field);
  let typ = field_entry.field_type().value_type();
- if path.is_empty() {
- if typ == Type::Json {
- return Err(InvalidQuery::JsonFieldRootNotSearchable {
- full_path: full_path.to_string(),
- });
- }
- } else if typ != Type::Json {
+ if !path.is_empty() && typ != Type::Json {
  return Err(InvalidQuery::FieldDoesNotExist {
  full_path: full_path.to_string(),
  });