Add possibility to invalidate cached indices with indexOnly constraint

Related #3155 #3263 #3264

Signed-off-by: Oleksandr Porunov <[email protected]>

janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java

@@ -8242,6 +8242,68 @@ public void testIndexStoreForceInvalidationFromDBCache() throws InterruptedExcep
  graph2.close();
  }
 
+ @Test
+ public void testIndexWithIndexOnlyConstraintForceInvalidationFromDBCache() throws InterruptedException, ExecutionException {
+ if (features.hasLocking() || !features.isDistributed()) {
+ return;
+ }
+
+ String indexPropName = "indexedProp";
+ String vertexLabelName = "vertexLabelForIndexOnlyConstraint";
+ String indexName = "indexWithIndexOnlyConstraint";
+ PropertyKey indexedProp = mgmt.makePropertyKey(indexPropName).dataType(Integer.class).cardinality(Cardinality.SINGLE).make();
+ VertexLabel vertexLabel = mgmt.makeVertexLabel(vertexLabelName).make();
+ mgmt.buildIndex(indexName, Vertex.class).addKey(indexedProp).indexOnly(vertexLabel).buildCompositeIndex();
+ finishSchema();
+ ManagementSystem.awaitGraphIndexStatus(graph, indexName).call();
+ mgmt.updateIndex(mgmt.getGraphIndex(indexName), SchemaAction.REINDEX).get();
+ finishSchema();
+
+ StandardJanusGraph graph1 = openInstanceWithDBCacheEnabled("testIndexWithIndexOnlyConstraintForceInvalidationFromDBCache1");
+ StandardJanusGraph graph2 = openInstanceWithDBCacheEnabled("testIndexWithIndexOnlyConstraintForceInvalidationFromDBCache2");
+
+ JanusGraphVertex v1 = graph1.addVertex(vertexLabelName);
+ v1.property(indexPropName, 1);
+
+ graph1.tx().commit();
+
+ // Cache data
+ JanusGraphTransaction tx2 = graph2.newTransaction();
+ assertEquals(1L, tx2.traversal().V().hasLabel(vertexLabelName).has(indexPropName, 1).count().next());
+ tx2.commit();
+
+ // Remove vertex
+ JanusGraphTransaction tx1 = graph1.newTransaction();
+ tx1.traversal().V(v1.id()).drop().iterate();
+ tx1.commit();
+
+ // Check that cached indexed vertex in graph2 was not refreshed
+ tx2 = graph2.newTransaction();
+ assertEquals(1L, tx2.traversal().V().hasLabel(vertexLabelName).has(indexPropName, 1).count().next());
+
+ // Try to invalidate data without vertex label with index constraints filter enabled
+ invalidateUpdatedVertexProperty(graph2, v1.longId(), indexPropName, 1, -1, true);
+ tx2.rollback();
+
+ tx2 = graph2.newTransaction();
+ // Check that invalidation didn't work
+ assertEquals(1L, tx2.traversal().V().hasLabel(vertexLabelName).has(indexPropName, 1).count().next());
+ tx2.rollback();
+
+ tx2 = graph2.newTransaction();
+ // Invalidate data without vertex label with index constraints filter disabled
+ invalidateUpdatedVertexProperty(graph2, v1.longId(), indexPropName, 1, -1, false);
+ tx2.commit();
+
+ tx2 = graph2.newTransaction();
+ // Check that invalidation worked
+ assertEquals(0L, tx2.traversal().V().hasLabel(vertexLabelName).has(indexPropName, 1).count().next());
+ tx2.rollback();
+
+ graph1.close();
+ graph2.close();
+ }
+
  @Test
  public void testFullDBCacheInvalidation() throws InterruptedException, ExecutionException {
  if (features.hasLocking() || !features.isDistributed()) {
@@ -8339,6 +8401,10 @@ public void testFullDBCacheInvalidation() throws InterruptedException, Execution
  }
 
  private void invalidateUpdatedVertexProperty(StandardJanusGraph graph, long vertexIdUpdated, String propertyNameUpdated, Object previousPropertyValue, Object newPropertyValue){
+ invalidateUpdatedVertexProperty(graph, vertexIdUpdated, propertyNameUpdated, previousPropertyValue, newPropertyValue, true);
+ }
+
+ private void invalidateUpdatedVertexProperty(StandardJanusGraph graph, long vertexIdUpdated, String propertyNameUpdated, Object previousPropertyValue, Object newPropertyValue, boolean withIndexConstraintsFilter){
  JanusGraphTransaction tx = graph.newTransaction();
  JanusGraphManagement graphMgmt = graph.openManagement();
  PropertyKey propertyKey = graphMgmt.getPropertyKey(propertyNameUpdated);
@@ -8347,7 +8413,13 @@ private void invalidateUpdatedVertexProperty(StandardJanusGraph graph, long vert
  StandardVertexProperty propertyNewVal = new StandardVertexProperty(propertyKey.longId(), propertyKey, cacheVertex, newPropertyValue, ElementLifeCycle.New);
  IndexSerializer indexSerializer = graph.getIndexSerializer();
 
- Collection<IndexUpdate> indexUpdates = indexSerializer.getIndexUpdates(cacheVertex, Arrays.asList(propertyPreviousVal, propertyNewVal));
+ Collection<IndexUpdate> indexUpdates;
+ if(withIndexConstraintsFilter){
+ indexUpdates = indexSerializer.getIndexUpdates(cacheVertex, Arrays.asList(propertyPreviousVal, propertyNewVal));
+ } else {
+ indexUpdates = indexSerializer.getIndexUpdatesNoConstraints(cacheVertex, Arrays.asList(propertyPreviousVal, propertyNewVal));
+ }
+
  CacheInvalidationService invalidationService = graph.getDBCacheInvalidationService();
 
  for(IndexUpdate indexUpdate : indexUpdates){

janusgraph-core/src/main/java/org/janusgraph/graphdb/database/IndexSerializer.java

@@ -51,6 +51,7 @@
 import org.janusgraph.graphdb.database.index.IndexRecords;
 import org.janusgraph.graphdb.database.index.IndexUpdate;
 import org.janusgraph.graphdb.database.serialize.Serializer;
+import org.janusgraph.graphdb.database.util.IndexAppliesToFunction;
 import org.janusgraph.graphdb.database.util.IndexRecordUtil;
 import org.janusgraph.graphdb.internal.ElementCategory;
 import org.janusgraph.graphdb.internal.InternalRelation;
@@ -88,6 +89,8 @@
 import javax.annotation.Nullable;
 
 import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.INDEX_NAME_MAPPING;
+import static org.janusgraph.graphdb.database.util.IndexRecordUtil.FULL_INDEX_APPLIES_TO_FILTER;
+import static org.janusgraph.graphdb.database.util.IndexRecordUtil.INDEX_APPLIES_TO_NO_CONSTRAINTS_FILTER;
 import static org.janusgraph.graphdb.database.util.IndexRecordUtil.bytebuffer2RelationId;
 import static org.janusgraph.graphdb.database.util.IndexRecordUtil.getCompositeIndexUpdate;
 import static org.janusgraph.graphdb.database.util.IndexRecordUtil.element2String;
@@ -180,14 +183,30 @@ public IndexInfoRetriever getIndexInfoRetriever(StandardJanusGraphTx tx) {
  ################################################### */
 
  public Collection<IndexUpdate> getIndexUpdates(InternalRelation relation) {
+ return getIndexUpdates(relation, FULL_INDEX_APPLIES_TO_FILTER);
+ }
+
+ public Collection<IndexUpdate> getIndexUpdates(InternalVertex vertex, Collection<InternalRelation> updatedProperties) {
+ return getIndexUpdates(vertex, updatedProperties, FULL_INDEX_APPLIES_TO_FILTER);
+ }
+
+ public Collection<IndexUpdate> getIndexUpdatesNoConstraints(InternalRelation relation) {
+ return getIndexUpdates(relation, INDEX_APPLIES_TO_NO_CONSTRAINTS_FILTER);
+ }
+
+ public Collection<IndexUpdate> getIndexUpdatesNoConstraints(InternalVertex vertex, Collection<InternalRelation> updatedProperties) {
+ return getIndexUpdates(vertex, updatedProperties, INDEX_APPLIES_TO_NO_CONSTRAINTS_FILTER);
+ }
+
+ public Collection<IndexUpdate> getIndexUpdates(InternalRelation relation, IndexAppliesToFunction indexFilter) {
  assert relation.isNew() || relation.isRemoved();
  final Set<IndexUpdate> updates = new HashSet<>();
  final IndexMutationType updateType = getUpdateType(relation);
  final int ttl = updateType==IndexMutationType.ADD?StandardJanusGraph.getTTL(relation):0;
  for (final PropertyKey type : relation.getPropertyKeysDirect()) {
  if (type == null) continue;
  for (final IndexType index : ((InternalRelationType) type).getKeyIndexes()) {
- if (!indexAppliesTo(index,relation)) continue;
+ if (!indexFilter.indexAppliesTo(index,relation)) continue;
  IndexUpdate update;
  if (index instanceof CompositeIndexType) {
  final CompositeIndexType iIndex= (CompositeIndexType) index;
@@ -206,7 +225,7 @@ public Collection<IndexUpdate> getIndexUpdates(InternalRelation relation) {
  return updates;
  }
 
- public Collection<IndexUpdate> getIndexUpdates(InternalVertex vertex, Collection<InternalRelation> updatedProperties) {
+ public Collection<IndexUpdate> getIndexUpdates(InternalVertex vertex, Collection<InternalRelation> updatedProperties, IndexAppliesToFunction indexFilter) {
  if (updatedProperties.isEmpty()) return Collections.emptyList();
  final Set<IndexUpdate> updates = new HashSet<>();
 
@@ -216,7 +235,7 @@ public Collection<IndexUpdate> getIndexUpdates(InternalVertex vertex, Collection
  assert rel.isNew() || rel.isRemoved(); assert rel.getVertex(0).equals(vertex);
  final IndexMutationType updateType = getUpdateType(rel);
  for (final IndexType index : ((InternalRelationType)p.propertyKey()).getKeyIndexes()) {
- if (!indexAppliesTo(index,vertex)) continue;
+ if (!indexFilter.indexAppliesTo(index,vertex)) continue;
  if (index.isCompositeIndex()) { //Gather composite indexes
  final CompositeIndexType cIndex = (CompositeIndexType)index;
  final IndexRecords updateRecords = indexMatches(vertex,cIndex,updateType==IndexMutationType.DELETE,p.propertyKey(),new IndexRecordEntry(p));

janusgraph-core/src/main/java/org/janusgraph/graphdb/database/util/IndexAppliesToFunction.java

@@ -0,0 +1,22 @@
+// Copyright 2022 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.graphdb.database.util;
+
+import org.janusgraph.core.JanusGraphElement;
+import org.janusgraph.graphdb.types.IndexType;
+
+public interface IndexAppliesToFunction {
+ boolean indexAppliesTo(IndexType index, JanusGraphElement element);
+}

janusgraph-core/src/main/java/org/janusgraph/graphdb/database/util/IndexRecordUtil.java

@@ -61,6 +61,9 @@
 
 public class IndexRecordUtil {
 
+ public static final IndexAppliesToFunction FULL_INDEX_APPLIES_TO_FILTER = IndexRecordUtil::indexAppliesTo;
+ public static final IndexAppliesToFunction INDEX_APPLIES_TO_NO_CONSTRAINTS_FILTER = IndexRecordUtil::indexAppliesToWithoutConstraints;
+
  private static final int DEFAULT_OBJECT_BYTELEN = 30;
  private static final byte FIRST_INDEX_COLUMN_BYTE = 0;
 
@@ -134,10 +137,17 @@ public static IndexMutationType getUpdateType(InternalRelation relation) {
  }
 
  public static boolean indexAppliesTo(IndexType index, JanusGraphElement element) {
+ return indexAppliesToWithoutConstraints(index, element) && indexMatchesConstraints(index, element);
+ }
+
+ public static boolean indexAppliesToWithoutConstraints(IndexType index, JanusGraphElement element) {
  return index.getElement().isInstance(element) &&
- (!(index instanceof CompositeIndexType) || ((CompositeIndexType)index).getStatus()!= SchemaStatus.DISABLED) &&
- (!index.hasSchemaTypeConstraint() ||
- index.getElement().matchesConstraint(index.getSchemaTypeConstraint(),element));
+ (!(index instanceof CompositeIndexType) || ((CompositeIndexType)index).getStatus()!= SchemaStatus.DISABLED);
+ }
+
+ public static boolean indexMatchesConstraints(IndexType index, JanusGraphElement element) {
+ return !index.hasSchemaTypeConstraint() ||
+ index.getElement().matchesConstraint(index.getSchemaTypeConstraint(),element);
  }
 
  public static PropertyKey[] getKeysOfRecords(IndexRecordEntry[] record) {