fix: fix mmap failed when string field all value is empty (#31418)

pr: #31406 Signed-off-by: luzhang <[email protected]> Co-authored-by: luzhang <[email protected]>
milvus-io · Mar 20, 2024 · 6856ba1 · 6856ba1
1 parent 7f201e8
commit 6856ba1
Showing 1 changed file with 32 additions and 8 deletions.
diff --git a/internal/core/src/mmap/Column.h b/internal/core/src/mmap/Column.h
@@ -41,17 +41,23 @@
 
 namespace milvus {
 
+/*
+* If string field's value all empty, need a string padding to avoid 
+* mmap failing because size_ is zero which causing invalid arguement
+* array has the same problem
+* TODO: remove it when support NULL value
+*/
+constexpr size_t STRING_PADDING = 1;
+constexpr size_t ARRAY_PADDING = 1;
+
 class ColumnBase {
  public:
  // memory mode ctor
  ColumnBase(size_t reserve, const FieldMeta& field_meta)
  : type_size_(datatype_is_sparse_vector(field_meta.get_data_type())
  ? 1
  : field_meta.get_sizeof()) {
- // simdjson requires a padding following the json data
- padding_ = field_meta.get_data_type() == DataType::JSON
- ? simdjson::SIMDJSON_PADDING
- : 0;
+ SetPaddingSize(field_meta.get_data_type());
 
  if (datatype_is_variable(field_meta.get_data_type())) {
  return;
@@ -78,9 +84,7 @@ class ColumnBase {
  ? 1
  : field_meta.get_sizeof()),
  num_rows_(size / type_size_) {
- padding_ = field_meta.get_data_type() == DataType::JSON
- ? simdjson::SIMDJSON_PADDING
- : 0;
+ SetPaddingSize(field_meta.get_data_type());
 
  size_ = size;
  cap_size_ = size;
@@ -105,7 +109,7 @@ class ColumnBase {
  num_rows_(size / datatype_sizeof(data_type, dim)),
  size_(size),
  cap_size_(size) {
- padding_ = data_type == DataType::JSON ? simdjson::SIMDJSON_PADDING : 0;
+ SetPaddingSize(data_type);
 
  data_ = static_cast<char*>(mmap(nullptr,
  cap_size_ + padding_,
@@ -194,6 +198,26 @@ class ColumnBase {
  num_rows_++;
  }
 
+ void
+ SetPaddingSize(const DataType& type) {
+ switch (type) {
+ case DataType::JSON:
+ // simdjson requires a padding following the json data
+ padding_ = simdjson::SIMDJSON_PADDING;
+ break;
+ case DataType::VARCHAR:
+ case DataType::STRING:
+ padding_ = STRING_PADDING;
+ break;
+ case DataType::ARRAY:
+ padding_ = ARRAY_PADDING;
+ break;
+ default:
+ padding_ = 0;
+ break;
+ }
+ }
+
  protected:
  // only for memory mode, not mmap
  void