Remove unsearchable properties

.github/workflows/linter.yml

@@ -16,7 +16,7 @@ jobs:
  go-version: '1.21'
  - uses: actions/checkout@v4
  - name: golangci-lint
- uses: golangci/golangci-lint-action@v5
+ uses: golangci/golangci-lint-action@v6
  with:
  # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
  version: v1.54

.github/workflows/pull_requests.yaml

@@ -125,7 +125,13 @@ jobs:
  go-version: '1.21'
  cache: true
  - name: Acceptance tests (modules)
- run: ./test/run.sh ${{ matrix.test }}
+ uses: nick-fields/retry@v2
+ with:
+ # 15 Minute is a large enough timeout for most of our tests
+ timeout_minutes: 15
+ max_attempts: 3
+ command: ./test/run.sh ${{ matrix.test }}
+ on_retry_command: ./test/run.sh --cleanup
  Modules-Acceptance-Tests-large:
  name: modules-acceptance-tests-large
  runs-on: ubuntu-latest-4-cores
@@ -141,7 +147,13 @@ jobs:
  go-version: '1.21'
  cache: true
  - name: Acceptance tests Large (modules)
- run: ./test/run.sh ${{ matrix.test }}
+ uses: nick-fields/retry@v2
+ with:
+ # 15 Minute is a large enough timeout for most of our tests
+ timeout_minutes: 15
+ max_attempts: 3
+ command: ./test/run.sh ${{ matrix.test }}
+ on_retry_command: ./test/run.sh --cleanup
  Modules-Acceptance-Tests-gcp:
  name: modules-acceptance-tests-gcp
  runs-on: ubuntu-latest
@@ -183,7 +195,7 @@ jobs:
  if : ${{ env.run_pipeline == 'true' || startsWith(github.ref, 'refs/tags') }}
  run: PALM_APIKEY=$(gcloud auth print-access-token) ./test/run.sh ${{ matrix.test }}
  Acceptance-Tests:
- name: acceptance-tests 
+ name: acceptance-tests
  runs-on: ubuntu-latest
  strategy:
  fail-fast: false
@@ -197,10 +209,16 @@ jobs:
  go-version: '1.21'
  cache: true
  - name: Acceptance tests
+ uses: nick-fields/retry@v2
  env:
  WCS_DUMMY_CI_PW: ${{ secrets.WCS_DUMMY_CI_PW }}
- WCS_DUMMY_CI_PW_2: ${{ secrets.WCS_DUMMY_CI_PW_2 }} 
- run: ./test/run.sh ${{ matrix.test }}
+ WCS_DUMMY_CI_PW_2: ${{ secrets.WCS_DUMMY_CI_PW_2 }}
+ with:
+ # 15 Minute is a large enough timeout for most of our tests
+ timeout_minutes: 15
+ max_attempts: 3
+ command: ./test/run.sh ${{ matrix.test }}
+ on_retry_command: ./test/run.sh --cleanup
  Codecov:
  needs: [Unit-Tests, Integration-Tests]
  name: codecov

adapters/handlers/rest/configure_api.go

@@ -1063,6 +1063,10 @@ func reasonableHttpClient(authConfig cluster.AuthConfig) *http.Client {
 }
 
 func setupGoProfiling(config config.Config, logger logrus.FieldLogger) {
+ if config.Profiling.Disabled {
+ return
+ }
+
  enterrors.GoWrapper(func() {
  portNumber := config.Profiling.Port
  if portNumber == 0 {

adapters/repos/db/aggregations_fixtures_for_test.go

@@ -36,6 +36,25 @@ var productClass = &models.Class{
  },
 }
 
+func boolRef(b bool) *bool {
+ return &b
+}
+
+var notIndexedClass = &models.Class{
+ Class: "NotIndexedClass",
+ VectorIndexConfig: enthnsw.NewDefaultUserConfig(),
+ InvertedIndexConfig: invertedConfig(),
+ Properties: []*models.Property{
+ {
+ Name: "name",
+ DataType: schema.DataTypeText.PropString(),
+ Tokenization: models.PropertyTokenizationWhitespace,
+ IndexFilterable: boolRef(false),
+ IndexInverted: boolRef(false),
+ },
+ },
+}
+
 var companyClass = &models.Class{
  Class: "AggregationsTestCompany",
  VectorIndexConfig: enthnsw.NewDefaultUserConfig(),

adapters/repos/db/aggregations_integration_test.go

@@ -129,6 +129,7 @@ func prepareCompanyTestSchemaAndData(repo *DB,
  Objects: &models.Schema{
  Classes: []*models.Class{
  productClass,
+ notIndexedClass,
  companyClass,
  arrayTypesClass,
  customerClass,
@@ -147,6 +148,8 @@ func prepareCompanyTestSchemaAndData(repo *DB,
  migrator.AddClass(context.Background(), arrayTypesClass, schemaGetter.shardState))
  require.Nil(t,
  migrator.AddClass(context.Background(), customerClass, schemaGetter.shardState))
+ require.Nil(t,
+ migrator.AddClass(context.Background(), notIndexedClass, schemaGetter.shardState))
  })
 
  schemaGetter.schema = schema
@@ -165,6 +168,20 @@ func prepareCompanyTestSchemaAndData(repo *DB,
  }
  })
 
+ t.Run("import products into notIndexed class", func(t *testing.T) {
+ for i, schema := range products {
+ t.Run(fmt.Sprintf("importing product %d", i), func(t *testing.T) {
+ fixture := models.Object{
+ Class: notIndexedClass.Class,
+ ID: productsIds[i],
+ Properties: schema,
+ }
+ require.Nil(t,
+ repo.PutObject(context.Background(), &fixture, []float32{0.1, 0.2, 0.01, 0.2}, nil, nil, 0))
+ })
+ }
+ })
+
  t.Run("import companies", func(t *testing.T) {
  for j := 0; j < importFactor; j++ {
  for i, schema := range companies {

adapters/repos/db/aggregator/filtered.go

@@ -124,6 +124,9 @@ func (fa *filteredAggregator) bm25Objects(ctx context.Context, kw *searchparams.
  return nil, nil, fmt.Errorf("bm25 objects: could not find class %s in schema", fa.params.ClassName)
  }
  cfg := inverted.ConfigFromModel(class.InvertedIndexConfig)
+
+ // kw.ChooseSearchableProperties(class)
+
  objs, scores, err := inverted.NewBM25Searcher(cfg.BM25, fa.store, fa.getSchema.ReadOnlyClass,
  propertyspecific.Indices{}, fa.classSearcher,
  fa.GetPropertyLengthTracker(), fa.logger, fa.shardVersion,

adapters/repos/db/aggregator/hybrid.go

@@ -48,6 +48,8 @@ func (a *Aggregator) bm25Objects(ctx context.Context, kw *searchparams.KeywordRa
  }
  cfg := inverted.ConfigFromModel(class.InvertedIndexConfig)
 
+ // kw.ChooseSearchableProperties(class)
+
  objs, dists, err := inverted.NewBM25Searcher(cfg.BM25, a.store, a.getSchema.ReadOnlyClass,
  propertyspecific.Indices{}, a.classSearcher,
  a.GetPropertyLengthTracker(), a.logger, a.shardVersion,

adapters/repos/db/bm25f_test.go

@@ -114,6 +114,14 @@ func SetupClass(t require.TestingT, repo *DB, schemaGetter *fakeSchemaGetter, lo
  IndexFilterable: &vFalse,
  IndexSearchable: &vTrue,
  },
+ // Test that bm25f handles this property being unsearchable
+ {
+ Name: "notSearchable",
+ DataType: schema.DataTypeTextArray.PropString(),
+ Tokenization: models.PropertyTokenizationWhitespace,
+ IndexFilterable: &vFalse,
+ IndexSearchable: &vFalse,
+ },
  },
  }
 

adapters/repos/db/inverted/bm25_searcher.go

@@ -79,6 +79,7 @@ func (b *BM25Searcher) BM25F(ctx context.Context, filterDocIds helpers.AllowList
  return nil, nil, inverted.NewMissingSearchableIndexError(property)
  }
  }
+
  class := b.getClass(className.String())
  if class == nil {
  return nil, nil, fmt.Errorf("could not find class %s in schema", className)

entities/searchparams/retrieval.go

@@ -11,6 +11,14 @@
 
 package searchparams
 
+import (
+ "fmt"
+ "strings"
+
+ "github.com/weaviate/weaviate/entities/models"
+ "github.com/weaviate/weaviate/entities/schema"
+)
+
 type NearVector struct {
  Vector []float32 `json:"vector"`
  Certainty float64 `json:"certainty"`
@@ -26,6 +34,61 @@ type KeywordRanking struct {
  AdditionalExplanations bool `json:"additionalExplanations"`
 }
 
+// Indicates whether property should be indexed
+// Index holds document ids with property of/containing particular value
+// and number of its occurrences in that property
+// (index created using bucket of StrategyMapCollection)
+func HasSearchableIndex(prop *models.Property) bool {
+ switch dt, _ := schema.AsPrimitive(prop.DataType); dt {
+ case schema.DataTypeText, schema.DataTypeTextArray:
+ // by default property has searchable index only for text/text[] props
+ if prop.IndexSearchable == nil {
+ return true
+ }
+ return *prop.IndexSearchable
+ default:
+ return false
+ }
+}
+
+func PropertyHasSearchableIndex(class *models.Class, tentativePropertyName string) bool {
+ if class == nil {
+ return false
+ }
+
+ propertyName := strings.Split(tentativePropertyName, "^")[0]
+ p, err := schema.GetPropertyByName(class, propertyName)
+ if err != nil {
+ return false
+ }
+ return HasSearchableIndex(p)
+}
+
+// GetPropertyByName returns the class by its name
+func GetPropertyByName(c *models.Class, propName string) (*models.Property, error) {
+ for _, prop := range c.Properties {
+ // Check if the name of the property is the given name, that's the property we need
+ if prop.Name == strings.Split(propName, ".")[0] {
+ return prop, nil
+ }
+ }
+ return nil, fmt.Errorf("Property %v not found %v", propName, c.Class)
+}
+
+func (k *KeywordRanking) ChooseSearchableProperties(class *models.Class) {
+ var validProperties []string
+ for _, prop := range k.Properties {
+ property, err := GetPropertyByName(class, prop)
+ if err != nil {
+ continue
+ }
+ if HasSearchableIndex(property) {
+ validProperties = append(validProperties, prop)
+ }
+ }
+ k.Properties = validProperties
+}
+
 type WeightedSearchResult struct {
  SearchParams interface{} `json:"searchParams"`
  Weight float64 `json:"weight"`

test/acceptance/replication/crud_test.go

@@ -252,7 +252,7 @@ func immediateReplicaCRUD(t *testing.T) {
  })
 
  t.Run("OnNode-1", func(t *testing.T) {
- _, err := getObjectFromNode(t, compose.ContainerURI(1), "Article", articleIDs[0], "node2")
+ _, err := getObjectFromNode(t, compose.ContainerURI(1), "Article", articleIDs[0], "node1")
  assert.Equal(t, &objects.ObjectsClassGetNotFound{}, err)
  })
  t.Run("OnNode-2", func(t *testing.T) {
@@ -297,7 +297,7 @@ func eventualReplicaCRUD(t *testing.T) {
  defer cancel()
 
  compose, err := docker.New().
- WithWeaviateCluster().
+ With3NodeCluster().
  WithText2VecContextionary().
  Start(ctx)
  require.Nil(t, err)
@@ -340,31 +340,28 @@ func eventualReplicaCRUD(t *testing.T) {
  createObjects(t, compose.GetWeaviate().URI(), batch)
  })
 
- t.Run("configure classes to replicate to node 2", func(t *testing.T) {
+ t.Run("configure classes to replicate to node 3", func(t *testing.T) {
  ac := helper.GetClass(t, "Article")
  ac.ReplicationConfig = &models.ReplicationConfig{
- Factor: 2,
+ Factor: 3,
  }
  helper.UpdateClass(t, ac)
 
  pc := helper.GetClass(t, "Paragraph")
  pc.ReplicationConfig = &models.ReplicationConfig{
- Factor: 2,
+ Factor: 3,
  }
  helper.UpdateClass(t, pc)
  })
 
- t.Run("StopNode-1", func(t *testing.T) {
- stopNodeAt(ctx, t, compose, 1)
+ t.Run("StopNode-2", func(t *testing.T) {
+ stopNodeAt(ctx, t, compose, 2)
  })
 
- t.Run("assert all previous data replicated to node 2", func(t *testing.T) {
- // TODO-RAFT : we need to avoid any sleeps, come back and remove it
- // sleep 2 sec to make sure data not affected by EC issue
- time.Sleep(2 * time.Second)
- resp := gqlGet(t, compose.GetWeaviateNode2().URI(), "Article", replica.One)
+ t.Run("assert all previous data replicated to node 3", func(t *testing.T) {
+ resp := gqlGet(t, compose.GetWeaviateNode3().URI(), "Article", replica.One)
  assert.Len(t, resp, len(articleIDs))
- resp = gqlGet(t, compose.GetWeaviateNode2().URI(), "Paragraph", replica.One)
+ resp = gqlGet(t, compose.GetWeaviateNode3().URI(), "Paragraph", replica.One)
  assert.Len(t, resp, len(paragraphIDs))
  })
 
@@ -401,27 +398,26 @@ func eventualReplicaCRUD(t *testing.T) {
  })
 
  t.Run("RestartNode-2", func(t *testing.T) {
- err = compose.Start(ctx, compose.GetWeaviateNode2().Name())
- require.Nil(t, err)
+ startNodeAt(ctx, t, compose, 2)
  })
  })
 
  t.Run("DeleteObject", func(t *testing.T) {
- t.Run("OnNode-1", func(t *testing.T) {
- deleteObject(t, compose.GetWeaviate().URI(), "Article", articleIDs[0])
+ t.Run("OnNode-2", func(t *testing.T) {
+ deleteObject(t, compose.GetWeaviateNode2().URI(), "Article", articleIDs[0])
  })
 
- t.Run("StopNode-1", func(t *testing.T) {
- stopNodeAt(ctx, t, compose, 1)
+ t.Run("StopNode-2", func(t *testing.T) {
+ stopNodeAt(ctx, t, compose, 2)
  })
 
- t.Run("OnNode-2", func(t *testing.T) {
- _, err := getObjectFromNode(t, compose.GetWeaviateNode2().URI(), "Article", articleIDs[0], "node2")
+ t.Run("OnNode-1", func(t *testing.T) {
+ _, err := getObjectFromNode(t, compose.GetWeaviate().URI(), "Article", articleIDs[0], "node1")
  assert.Equal(t, &objects.ObjectsClassGetNotFound{}, err)
  })
 
- t.Run("RestartNode-1", func(t *testing.T) {
- restartNode1(ctx, t, compose)
+ t.Run("RestartNode-2", func(t *testing.T) {
+ startNodeAt(ctx, t, compose, 2)
  })
  })
 
@@ -441,15 +437,14 @@ func eventualReplicaCRUD(t *testing.T) {
  })
 
  t.Run("RestartNode-2", func(t *testing.T) {
- err = compose.Start(ctx, compose.GetWeaviateNode2().Name())
- require.Nil(t, err)
+ startNodeAt(ctx, t, compose, 2)
  })
  })
  })
 }
 
 func restartNode1(ctx context.Context, t *testing.T, compose *docker.DockerCompose) {
- // since node1 is the gossip "leader", node 2 must be stopped and restarted
+ // since node1 is the gossip "leader", node 2 and 3 must be stopped and restarted
  // after node1 to re-facilitate internode communication
  eg := errgroup.Group{}
  eg.Go(func() error {

test/acceptance/replication/scale_test.go

@@ -34,7 +34,7 @@ func multiShardScaleOut(t *testing.T) {
  defer cancel()
 
  compose, err := docker.New().
- WithWeaviateCluster().
+ With3NodeCluster().
  WithText2VecContextionary().
  Start(ctx)
  require.Nil(t, err)
@@ -134,9 +134,6 @@ func multiShardScaleOut(t *testing.T) {
 
  t.Run("kill a node and check contents of remaining node", func(t *testing.T) {
  stopNodeAt(ctx, t, compose, 2)
- // TODO-RAFT : we need to avoid any sleeps, come back and remove it
- // sleep 2 sec to make sure data not affected by EC issue
- time.Sleep(2 * time.Second)
  p := gqlGet(t, compose.GetWeaviate().URI(), paragraphClass.Class, replica.One)
  assert.Len(t, p, 10)
  a := gqlGet(t, compose.GetWeaviate().URI(), articleClass.Class, replica.One)