Merge stable/v1.25 into main

adapters/handlers/grpc/v1/parse_search_request.go

@@ -368,12 +368,6 @@ func extractTargetVectors(req *pb.SearchRequest, class *models.Class) (*[]string
  var targetVectors *[]string
  if hs := req.HybridSearch; hs != nil {
  targetVectors = &hs.TargetVectors
- if hs.NearText != nil {
- targetVectors = &hs.NearText.TargetVectors
- }
- if hs.NearVector != nil {
- targetVectors = &hs.NearVector.TargetVectors
- }
  }
  if na := req.NearAudio; na != nil {
  targetVectors = &na.TargetVectors

adapters/handlers/grpc/v1/parse_search_request_test.go

@@ -204,11 +204,11 @@ func TestGRPCRequest(t *testing.T) {
  Alpha: 1.0,
  Query: "nearvecquery",
  NearVector: &pb.NearVector{
- VectorBytes: byteops.Float32ToByteVector([]float32{1, 2, 3}),
- TargetVectors: []string{"custom"},
- Certainty: &one,
- Distance: &one,
+ VectorBytes: byteops.Float32ToByteVector([]float32{1, 2, 3}),
+ Certainty: &one,
+ Distance: &one,
  },
+ TargetVectors: []string{"custom"},
  },
  },
  out: dto.GetParams{
@@ -221,12 +221,12 @@ func TestGRPCRequest(t *testing.T) {
  Query: "nearvecquery",
  FusionAlgorithm: 1,
  NearVectorParams: &searchparams.NearVector{
- Vector: []float32{1, 2, 3},
- TargetVectors: []string{"custom"},
- Certainty: 1.0,
- Distance: 1.0,
- WithDistance: true,
+ Vector: []float32{1, 2, 3},
+ Certainty: 1.0,
+ Distance: 1.0,
+ WithDistance: true,
  },
+ TargetVectors: []string{"custom"},
  },
  },
  error: false,

adapters/handlers/grpc/v1/tenants.go

@@ -27,7 +27,7 @@ func (s *Service) tenantsGet(ctx context.Context, principal *models.Principal, r
  var err error
  var tenants []*models.Tenant
  if req.Params == nil {
- tenants, err = s.schemaManager.GetConsistentTenants(ctx, principal, req.Collection, req.IsConsistent, []string{})
+ tenants, err = s.schemaManager.GetConsistentTenants(ctx, principal, req.Collection, true, []string{})
  if err != nil {
  return nil, err
  }
@@ -38,7 +38,7 @@ func (s *Service) tenantsGet(ctx context.Context, principal *models.Principal, r
  if len(requestedNames) == 0 {
  return nil, fmt.Errorf("must specify at least one tenant name")
  }
- tenants, err = s.schemaManager.GetConsistentTenants(ctx, principal, req.Collection, req.IsConsistent, requestedNames)
+ tenants, err = s.schemaManager.GetConsistentTenants(ctx, principal, req.Collection, true, requestedNames)
  if err != nil {
  return nil, err
  }

adapters/handlers/rest/clusterapi/indices.go

@@ -1105,7 +1105,7 @@ func (i *indices) postShard() http.Handler {
 func (i *indices) putShardReinit() http.Handler {
  return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  args := i.regexpShardReinit.FindStringSubmatch(r.URL.Path)
- fmt.Println(args)
+
  if len(args) != 3 {
  http.Error(w, "invalid URI", http.StatusBadRequest)
  return

adapters/handlers/rest/configure_api.go

@@ -193,6 +193,8 @@ func MakeAppState(ctx context.Context, options *swag.CommandLineOptionsGroup) *s
  MemtablesMaxSizeMB: appState.ServerConfig.Config.Persistence.MemtablesMaxSizeMB,
  MemtablesMinActiveSeconds: appState.ServerConfig.Config.Persistence.MemtablesMinActiveDurationSeconds,
  MemtablesMaxActiveSeconds: appState.ServerConfig.Config.Persistence.MemtablesMaxActiveDurationSeconds,
+ MaxSegmentSize: appState.ServerConfig.Config.Persistence.LSMMaxSegmentSize,
+ HNSWMaxLogSize: appState.ServerConfig.Config.Persistence.HNSWMaxLogSize,
  RootPath: appState.ServerConfig.Config.Persistence.DataPath,
  QueryLimit: appState.ServerConfig.Config.QueryDefaults.Limit,
  QueryMaximumResults: appState.ServerConfig.Config.QueryMaximumResults,
@@ -251,8 +253,6 @@ func MakeAppState(ctx context.Context, options *swag.CommandLineOptionsGroup) *s
  remoteIndexClient, appState.Logger, appState.ServerConfig.Config.Persistence.DataPath)
  appState.Scaler = scaler
 
- /// TODO-RAFT START
- //
  server2port, err := parseNode2Port(appState)
  if len(server2port) == 0 || err != nil {
  appState.Logger.
@@ -268,31 +268,31 @@ func MakeAppState(ctx context.Context, options *swag.CommandLineOptionsGroup) *s
  dataPath := appState.ServerConfig.Config.Persistence.DataPath
 
  rConfig := rStore.Config{
- WorkDir: filepath.Join(dataPath, "raft"),
- NodeID: nodeName,
- Host: addrs[0],
- RaftPort: appState.ServerConfig.Config.Raft.Port,
- RPCPort: appState.ServerConfig.Config.Raft.InternalRPCPort,
- RaftRPCMessageMaxSize: appState.ServerConfig.Config.Raft.RPCMessageMaxSize,
- ServerName2PortMap: server2port,
- BootstrapTimeout: appState.ServerConfig.Config.Raft.BootstrapTimeout,
- BootstrapExpect: appState.ServerConfig.Config.Raft.BootstrapExpect,
- HeartbeatTimeout: appState.ServerConfig.Config.Raft.HeartbeatTimeout,
- RecoveryTimeout: appState.ServerConfig.Config.Raft.RecoveryTimeout,
- ElectionTimeout: appState.ServerConfig.Config.Raft.ElectionTimeout,
- SnapshotInterval: appState.ServerConfig.Config.Raft.SnapshotInterval,
- SnapshotThreshold: appState.ServerConfig.Config.Raft.SnapshotThreshold,
- UpdateWaitTimeout: time.Second * 10, // TODO-RAFT read from the flag
- MetadataOnlyVoters: appState.ServerConfig.Config.Raft.MetadataOnlyVoters,
- DB: nil,
- Parser: schema.NewParser(appState.Cluster, vectorIndex.ParseAndValidateConfig, migrator),
- AddrResolver: appState.Cluster,
- Logger: appState.Logger,
- LogLevel: logLevel(),
- LogJSONFormat: !logTextFormat(),
- IsLocalHost: appState.ServerConfig.Config.Cluster.Localhost,
- LoadLegacySchema: schemaRepo.LoadLegacySchema,
- SaveLegacySchema: schemaRepo.SaveLegacySchema,
+ WorkDir:  filepath.Join(dataPath, config.DefaultRaftDir),
+ NodeID:  nodeName,
+ Host:  addrs[0],
+ RaftPort:  appState.ServerConfig.Config.Raft.Port,
+ RPCPort:  appState.ServerConfig.Config.Raft.InternalRPCPort,
+ RaftRPCMessageMaxSize:  appState.ServerConfig.Config.Raft.RPCMessageMaxSize,
+ ServerName2PortMap:  server2port,
+ BootstrapTimeout:  appState.ServerConfig.Config.Raft.BootstrapTimeout,
+ BootstrapExpect:  appState.ServerConfig.Config.Raft.BootstrapExpect,
+ HeartbeatTimeout:  appState.ServerConfig.Config.Raft.HeartbeatTimeout,
+ RecoveryTimeout:  appState.ServerConfig.Config.Raft.RecoveryTimeout,
+ ElectionTimeout:  appState.ServerConfig.Config.Raft.ElectionTimeout,
+ SnapshotInterval:  appState.ServerConfig.Config.Raft.SnapshotInterval,
+ SnapshotThreshold:  appState.ServerConfig.Config.Raft.SnapshotThreshold,
+ ConsistencyWaitTimeout: appState.ServerConfig.Config.Raft.ConsistencyWaitTimeout,
+ MetadataOnlyVoters:  appState.ServerConfig.Config.Raft.MetadataOnlyVoters,
+ DB:  nil,
+ Parser:  schema.NewParser(appState.Cluster, vectorIndex.ParseAndValidateConfig, migrator),
+ AddrResolver:  appState.Cluster,
+ Logger:  appState.Logger,
+ LogLevel:  logLevel(),
+ LogJSONFormat:  !logTextFormat(),
+ IsLocalHost:  appState.ServerConfig.Config.Cluster.Localhost,
+ LoadLegacySchema:  schemaRepo.LoadLegacySchema,
+ SaveLegacySchema:  schemaRepo.SaveLegacySchema,
  }
  for _, name := range appState.ServerConfig.Config.Raft.Join[:rConfig.BootstrapExpect] {
  if strings.Contains(name, rConfig.NodeID) {

adapters/repos/db/aggregations_fixtures_for_test.go

@@ -36,6 +36,25 @@ var productClass = &models.Class{
  },
 }
 
+func boolRef(b bool) *bool {
+ return &b
+}
+
+var notIndexedClass = &models.Class{
+ Class: "NotIndexedClass",
+ VectorIndexConfig: enthnsw.NewDefaultUserConfig(),
+ InvertedIndexConfig: invertedConfig(),
+ Properties: []*models.Property{
+ {
+ Name: "name",
+ DataType: schema.DataTypeText.PropString(),
+ Tokenization: models.PropertyTokenizationWhitespace,
+ IndexFilterable: boolRef(false),
+ IndexInverted: boolRef(false),
+ },
+ },
+}
+
 var companyClass = &models.Class{
  Class: "AggregationsTestCompany",
  VectorIndexConfig: enthnsw.NewDefaultUserConfig(),

adapters/repos/db/aggregations_integration_test.go

@@ -129,6 +129,7 @@ func prepareCompanyTestSchemaAndData(repo *DB,
  Objects: &models.Schema{
  Classes: []*models.Class{
  productClass,
+ notIndexedClass,
  companyClass,
  arrayTypesClass,
  customerClass,
@@ -147,6 +148,8 @@ func prepareCompanyTestSchemaAndData(repo *DB,
  migrator.AddClass(context.Background(), arrayTypesClass, schemaGetter.shardState))
  require.Nil(t,
  migrator.AddClass(context.Background(), customerClass, schemaGetter.shardState))
+ require.Nil(t,
+ migrator.AddClass(context.Background(), notIndexedClass, schemaGetter.shardState))
  })
 
  schemaGetter.schema = schema
@@ -165,6 +168,20 @@ func prepareCompanyTestSchemaAndData(repo *DB,
  }
  })
 
+ t.Run("import products into notIndexed class", func(t *testing.T) {
+ for i, schema := range products {
+ t.Run(fmt.Sprintf("importing product %d", i), func(t *testing.T) {
+ fixture := models.Object{
+ Class: notIndexedClass.Class,
+ ID: productsIds[i],
+ Properties: schema,
+ }
+ require.Nil(t,
+ repo.PutObject(context.Background(), &fixture, []float32{0.1, 0.2, 0.01, 0.2}, nil, nil, 0))
+ })
+ }
+ })
+
  t.Run("import companies", func(t *testing.T) {
  for j := 0; j < importFactor; j++ {
  for i, schema := range companies {

adapters/repos/db/aggregator/filtered.go

@@ -124,6 +124,9 @@ func (fa *filteredAggregator) bm25Objects(ctx context.Context, kw *searchparams.
  return nil, nil, fmt.Errorf("bm25 objects: could not find class %s in schema", fa.params.ClassName)
  }
  cfg := inverted.ConfigFromModel(class.InvertedIndexConfig)
+
+ kw.ChooseSearchableProperties(class)
+
  objs, scores, err := inverted.NewBM25Searcher(cfg.BM25, fa.store, fa.getSchema.ReadOnlyClass,
  propertyspecific.Indices{}, fa.classSearcher,
  fa.GetPropertyLengthTracker(), fa.logger, fa.shardVersion,

adapters/repos/db/aggregator/hybrid.go

@@ -48,6 +48,8 @@ func (a *Aggregator) bm25Objects(ctx context.Context, kw *searchparams.KeywordRa
  }
  cfg := inverted.ConfigFromModel(class.InvertedIndexConfig)
 
+ kw.ChooseSearchableProperties(class)
+
  objs, dists, err := inverted.NewBM25Searcher(cfg.BM25, a.store, a.getSchema.ReadOnlyClass,
  propertyspecific.Indices{}, a.classSearcher,
  a.GetPropertyLengthTracker(), a.logger, a.shardVersion,

adapters/repos/db/bm25f_test.go

@@ -114,6 +114,14 @@ func SetupClass(t require.TestingT, repo *DB, schemaGetter *fakeSchemaGetter, lo
  IndexFilterable: &vFalse,
  IndexSearchable: &vTrue,
  },
+ // Test that bm25f handles this property being unsearchable
+ {
+ Name: "notSearchable",
+ DataType: schema.DataTypeTextArray.PropString(),
+ Tokenization: models.PropertyTokenizationWhitespace,
+ IndexFilterable: &vFalse,
+ IndexSearchable: &vFalse,
+ },
  },
  }
 

adapters/repos/db/helper_for_test.go

@@ -26,11 +26,11 @@ import (
  "github.com/weaviate/weaviate/adapters/repos/db/indexcheckpoint"
  "github.com/weaviate/weaviate/adapters/repos/db/inverted"
  "github.com/weaviate/weaviate/adapters/repos/db/inverted/stopwords"
- "github.com/weaviate/weaviate/entities/locks"
  "github.com/weaviate/weaviate/entities/models"
  "github.com/weaviate/weaviate/entities/schema"
  schemaConfig "github.com/weaviate/weaviate/entities/schema/config"
  "github.com/weaviate/weaviate/entities/storobj"
+ esync "github.com/weaviate/weaviate/entities/sync"
  enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw"
  "github.com/weaviate/weaviate/usecases/memwatch"
 )
@@ -258,6 +258,7 @@ func testShardWithSettings(t *testing.T, ctx context.Context, class *models.Clas
  RootPath: tmpDir,
  ClassName: schema.ClassName(class.Class),
  QueryMaximumResults: maxResults,
+ ReplicationFactor: NewAtomicInt64(1),
  },
  invertedIndexConfig: iic,
  vectorIndexUserConfig: vic,
@@ -267,8 +268,8 @@ func testShardWithSettings(t *testing.T, ctx context.Context, class *models.Clas
  stopwords: sd,
  indexCheckpoints: checkpts,
  allocChecker: memwatch.NewDummyMonitor(),
- shardCreateLocks: locks.NewNamedLocks(),
- shardInUseLocks: locks.NewNamedRWLocks(),
+ shardCreateLocks: esync.NewKeyLocker(),
+ shardInUseLocks: esync.NewKeyRWLocker(),
  }
  idx.closingCtx, idx.closingCancel = context.WithCancel(context.Background())
  idx.initCycleCallbacksNoop()
@@ -294,7 +295,7 @@ func testObject(className string) *storobj.Object {
  }
 }
 
-func createRandomObjects(r *rand.Rand, className string, numObj int) []*storobj.Object {
+func createRandomObjects(r *rand.Rand, className string, numObj int, vectorDim int) []*storobj.Object {
  obj := make([]*storobj.Object, numObj)
 
  for i := 0; i < numObj; i++ {
@@ -304,7 +305,11 @@ func createRandomObjects(r *rand.Rand, className string, numObj int) []*storobj.
  ID: strfmt.UUID(uuid.NewString()),
  Class: className,
  },
- Vector: []float32{r.Float32(), r.Float32(), r.Float32(), r.Float32()},
+ Vector: make([]float32, vectorDim),
+ }
+
+ for d := 0; d < vectorDim; d++ {
+ obj[i].Vector[d] = r.Float32()
  }
  }
  return obj