Skip to content

Commit

Permalink
Saving off for right now, still working on unit tests, ruturning afte…
Browse files Browse the repository at this point in the history 
…r log.trace is changing behavoir

Saving off, trying to update tests

Fix formatting

Updates to unit tests per configuration changes

Updated unit tests and initilization of max terms to print
  • Loading branch information
@MiguelRicardos
MiguelRicardos committed May 21, 2024
1 parent 3011367 commit 5f2f0de
Show file tree
Hide file tree
Showing 6 changed files with 244 additions and 19 deletions.
10 changes: 10 additions & 0 deletions warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -336,6 +336,7 @@ public class ShardQueryConfiguration extends GenericQueryConfiguration implement
private int finalMaxTermThreshold = 2500;
private int maxDepthThreshold = 2500;
private boolean expandFields = true;
private int maxTermsToPrint = 100;
private int maxUnfieldedExpansionThreshold = 500;
private boolean expandValues = true;
private int maxValueExpansionThreshold = 5000;
Expand Down Expand Up @@ -635,6 +636,7 @@ public ShardQueryConfiguration(ShardQueryConfiguration other) {
this.setIndexedMaxTermThreshold(other.getIndexedMaxTermThreshold());
this.setFinalMaxTermThreshold(other.getFinalMaxTermThreshold());
this.setMaxDepthThreshold(other.getMaxDepthThreshold());
this.setMaxTermsToPrint(other.getMaxTermsToPrint());
this.setMaxUnfieldedExpansionThreshold(other.getMaxUnfieldedExpansionThreshold());
this.setExpandFields(other.isExpandFields());
this.setMaxValueExpansionThreshold(other.getMaxValueExpansionThreshold());
Expand Down Expand Up @@ -1296,6 +1298,14 @@ public void setMaxDepthThreshold(int maxDepthThreshold) {
this.maxDepthThreshold = maxDepthThreshold;
}

public int getMaxTermsToPrint() {
return this.maxTermsToPrint;
}

public void setMaxTermsToPrint(int maxTermsToPrint) {
this.maxTermsToPrint = maxTermsToPrint;
}

public boolean isExpandFields() {
return expandFields;
}
Expand Down
144 changes: 125 additions & 19 deletions ...query-core/src/main/java/datawave/query/jexl/visitors/ExecutableDeterminationVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,10 @@
import datawave.query.config.ShardQueryConfiguration;
import datawave.query.jexl.JexlASTHelper;
import datawave.query.jexl.nodes.QueryPropertyMarker;
import datawave.query.jexl.visitors.BaseVisitor;
import datawave.query.jexl.visitors.JexlStringBuildingVisitor;
import datawave.query.jexl.visitors.PrintingVisitor;
import datawave.query.jexl.visitors.PushdownNegationVisitor;
import datawave.query.util.MetadataHelper;

/**
Expand Down Expand Up @@ -111,6 +115,11 @@ public static class StateAndReason {

private static final Logger log = Logger.getLogger(ExecutableDeterminationVisitor.class);

private static final String EXCEEDED_MAX_TERMS_MESSAGE = "Exceeded max terms to print threshold of ";
private static final String TERMS_PRINTED_MESSAGE = "; terms printed: ";
private int maxTermsToPrint;
private int termsPrinted = 0;

/**
* The output interface which allows us to write lines into the output, and provides for handling a stack of summary contributors that in the end will
* contain the terms that contribute to the final state.
Expand Down Expand Up @@ -382,6 +391,7 @@ public ExecutableDeterminationVisitor(ShardQueryConfiguration conf, MetadataHelp
if (debugOutput != null) {
output = newStringListOutput(debugOutput);
}
this.maxTermsToPrint = (this.config.getMaxTermsToPrint() == -1) ? 100 : this.config.getMaxTermsToPrint();
}

/**
Expand Down Expand Up @@ -556,8 +566,13 @@ else if (states.contains(STATE.ERROR)) {
}
}
if (output != null) {
output.writeLine(data, node, state, false);
termsPrinted++;
if (termsPrinted > maxTermsToPrint) {
log.trace(EXCEEDED_MAX_TERMS_MESSAGE + maxTermsToPrint + TERMS_PRINTED_MESSAGE + termsPrinted);
} else {

output.writeLine(data, node, state, false);
}
// determine which states to pass up. Should be the state determined.
states.clear();
states.add(state);
Expand Down Expand Up @@ -640,8 +655,13 @@ else if (states.contains(STATE.EXECUTABLE)) {
}
}
if (output != null) {
output.writeLine(data, node, state, false);
termsPrinted++;
if (termsPrinted > maxTermsToPrint) {
log.trace(EXCEEDED_MAX_TERMS_MESSAGE + maxTermsToPrint + TERMS_PRINTED_MESSAGE + termsPrinted);

} else {
output.writeLine(data, node, state, false);
}
// determine which states to pass up. Should be the state determined.
states.clear();
states.add(state);
Expand Down Expand Up @@ -918,6 +938,7 @@ public Object visit(ASTFunctionNode node, Object data) {
public Object visit(ASTNotNode node, Object data) {
// grab the recursive state because its either necessary directly or the error state of the branch needs to be checked
STATE state = allOrNone(node, negateData(data + PREFIX));

// if there is no error and executability is being checked against the global index just return non-executable
if (state == STATE.EXECUTABLE) {
if (output != null) {
Expand Down Expand Up @@ -972,14 +993,24 @@ public Object visit(ASTAndNode node, Object data) {
if (instance.isType(EXCEEDED_TERM)) {
state = STATE.NON_EXECUTABLE;
if (output != null) {
output.writeLine(data, node, "( Exceeded Term Threshold )", state, true);
termsPrinted++;
if (termsPrinted > maxTermsToPrint) {
log.trace(EXCEEDED_MAX_TERMS_MESSAGE + maxTermsToPrint + TERMS_PRINTED_MESSAGE + termsPrinted);
} else {
output.writeLine(data, node, "( Exceeded Term Threshold )", state, true);
}
}
}
// if an ivarator then return true, else check out children
else if (instance.isAnyTypeOf(EXCEEDED_VALUE, EXCEEDED_OR)) {
state = STATE.EXECUTABLE;
if (output != null) {
output.writeLine(data, node, "( Exceeded Or / Value Threshold )", state, true);
termsPrinted++;
if (termsPrinted > maxTermsToPrint) {
log.trace(EXCEEDED_MAX_TERMS_MESSAGE + maxTermsToPrint + TERMS_PRINTED_MESSAGE + termsPrinted);
} else {
output.writeLine(data, node, "( Exceeded Or / Value Threshold )", state, true);
}
}
}
// if ignored expression, then ignore it
Expand All @@ -996,19 +1027,34 @@ else if (instance.isAnyTypeOf(DELAYED, EVALUATION_ONLY)) {
state = STATE.NON_EXECUTABLE;
}
if (output != null) {
output.writeLine(data, node, "( delayed/eval only predicate )", state, true);
termsPrinted++;
if (termsPrinted > maxTermsToPrint) {
log.trace(EXCEEDED_MAX_TERMS_MESSAGE + maxTermsToPrint + TERMS_PRINTED_MESSAGE + termsPrinted);
} else {
output.writeLine(data, node, "( delayed/eval only predicate )", state, true);
}
}
}
// if we got to a bounded range, then this was expanded and is not executable against the index
else if (instance.isType(BOUNDED_RANGE)) {
state = STATE.NON_EXECUTABLE;
if (output != null) {
output.writeLine(data, node, "( bounded range )", state, true);
termsPrinted++;
if (termsPrinted > maxTermsToPrint) {
log.trace(EXCEEDED_MAX_TERMS_MESSAGE + maxTermsToPrint + TERMS_PRINTED_MESSAGE + termsPrinted);
} else {
output.writeLine(data, node, "( bounded range )", state, true);
}
}
} else if (instance.isType(INDEX_HOLE)) {
state = STATE.NON_EXECUTABLE;
if (output != null) {
output.writeLine(data, node, "( index hole )", state, true);
termsPrinted++;
if (termsPrinted > maxTermsToPrint) {
log.trace(EXCEEDED_MAX_TERMS_MESSAGE + maxTermsToPrint + TERMS_PRINTED_MESSAGE + termsPrinted);
} else {
output.writeLine(data, node, "( index hole )", state, true);
}
}
} else {
if (isNegated(data)) {
Expand Down Expand Up @@ -1171,7 +1217,12 @@ public Object visit(ASTBitwiseComplNode node, Object data) {
public Object visit(ASTIdentifier node, Object data) {
STATE state = STATE.IGNORABLE;
if (output != null) {
output.writeLine(data, node, ":" + node.getName(), state, true);
termsPrinted++;
if (termsPrinted > maxTermsToPrint) {
log.trace(EXCEEDED_MAX_TERMS_MESSAGE + maxTermsToPrint + TERMS_PRINTED_MESSAGE + termsPrinted);
} else {
output.writeLine(data, node, ":" + node.getName(), state, true);
}
}
return state;
}
Expand All @@ -1180,7 +1231,12 @@ public Object visit(ASTIdentifier node, Object data) {
public Object visit(ASTNullLiteral node, Object data) {
STATE state = STATE.NON_EXECUTABLE;
if (output != null) {
output.writeLine(data, node, state, true);
termsPrinted++;
if (termsPrinted > maxTermsToPrint) {
log.trace(EXCEEDED_MAX_TERMS_MESSAGE + maxTermsToPrint + TERMS_PRINTED_MESSAGE + termsPrinted);
} else {
output.writeLine(data, node, state, true);
}
}
return state;
}
Expand All @@ -1189,7 +1245,12 @@ public Object visit(ASTNullLiteral node, Object data) {
public Object visit(ASTTrueNode node, Object data) {
STATE state = STATE.IGNORABLE;
if (output != null) {
output.writeLine(data, node, state, true);
termsPrinted++;
if (termsPrinted > maxTermsToPrint) {
log.trace(EXCEEDED_MAX_TERMS_MESSAGE + maxTermsToPrint + TERMS_PRINTED_MESSAGE + termsPrinted);
} else {
output.writeLine(data, node, state, true);
}
}
return state;
}
Expand All @@ -1198,7 +1259,12 @@ public Object visit(ASTTrueNode node, Object data) {
public Object visit(ASTFalseNode node, Object data) {
STATE state = STATE.IGNORABLE;
if (output != null) {
output.writeLine(data, node, state, true);
termsPrinted++;
if (termsPrinted > maxTermsToPrint) {
log.trace(EXCEEDED_MAX_TERMS_MESSAGE + maxTermsToPrint + TERMS_PRINTED_MESSAGE + termsPrinted);
} else {
output.writeLine(data, node, state, true);
}
}
return state;
}
Expand All @@ -1207,7 +1273,12 @@ public Object visit(ASTFalseNode node, Object data) {
public Object visit(ASTStringLiteral node, Object data) {
STATE state = STATE.IGNORABLE;
if (output != null) {
output.writeLine(data, node, ":" + node.getLiteral(), state, true);
termsPrinted++;
if (termsPrinted > maxTermsToPrint) {
log.trace(EXCEEDED_MAX_TERMS_MESSAGE + maxTermsToPrint + TERMS_PRINTED_MESSAGE + termsPrinted);
} else {
output.writeLine(data, node, ":" + node.getLiteral(), state, true);
}
}
return state;
}
Expand Down Expand Up @@ -1243,7 +1314,12 @@ public Object visit(ASTMapEntry node, Object data) {
public Object visit(ASTEmptyFunction node, Object data) {
STATE state = STATE.IGNORABLE;
if (output != null) {
output.writeLine(data, node, state, true);
termsPrinted++;
if (termsPrinted > maxTermsToPrint) {
log.trace(EXCEEDED_MAX_TERMS_MESSAGE + maxTermsToPrint + TERMS_PRINTED_MESSAGE + termsPrinted);
} else {
output.writeLine(data, node, state, true);
}
}
return state;
}
Expand All @@ -1261,7 +1337,12 @@ public Object visit(ASTSizeFunction node, Object data) {
public Object visit(ASTMethodNode node, Object data) {
STATE state = STATE.NON_EXECUTABLE;
if (output != null) {
output.writeLine(data, node, ":" + JexlNodes.getIdentifierOrLiteral(node), state, true);
termsPrinted++;
if (termsPrinted > maxTermsToPrint) {
log.trace(EXCEEDED_MAX_TERMS_MESSAGE + maxTermsToPrint + TERMS_PRINTED_MESSAGE + termsPrinted);
} else {
output.writeLine(data, node, ":" + JexlNodes.getIdentifierOrLiteral(node), state, true);
}
}
return state;
}
Expand All @@ -1270,7 +1351,12 @@ public Object visit(ASTMethodNode node, Object data) {
public Object visit(ASTConstructorNode node, Object data) {
STATE state = STATE.IGNORABLE;
if (output != null) {
output.writeLine(data, node, state, true);
termsPrinted++;
if (termsPrinted > maxTermsToPrint) {
log.trace(EXCEEDED_MAX_TERMS_MESSAGE + maxTermsToPrint + TERMS_PRINTED_MESSAGE + termsPrinted);
} else {
output.writeLine(data, node, state, true);
}
}
return state;
}
Expand All @@ -1279,7 +1365,12 @@ public Object visit(ASTConstructorNode node, Object data) {
public Object visit(ASTArrayAccess node, Object data) {
STATE state = STATE.IGNORABLE;
if (output != null) {
output.writeLine(data, node, state, true);
termsPrinted++;
if (termsPrinted > maxTermsToPrint) {
log.trace(EXCEEDED_MAX_TERMS_MESSAGE + maxTermsToPrint + TERMS_PRINTED_MESSAGE + termsPrinted);
} else {
output.writeLine(data, node, state, true);
}
}
return state;
}
Expand All @@ -1288,7 +1379,12 @@ public Object visit(ASTArrayAccess node, Object data) {
public Object visit(ASTReturnStatement node, Object data) {
STATE state = STATE.IGNORABLE;
if (output != null) {
output.writeLine(data, node, state, true);
termsPrinted++;
if (termsPrinted > maxTermsToPrint) {
log.trace(EXCEEDED_MAX_TERMS_MESSAGE + maxTermsToPrint + TERMS_PRINTED_MESSAGE + termsPrinted);
} else {
output.writeLine(data, node, state, true);
}
}
return state;
}
Expand All @@ -1297,7 +1393,12 @@ public Object visit(ASTReturnStatement node, Object data) {
public Object visit(ASTVar node, Object data) {
STATE state = STATE.IGNORABLE;
if (output != null) {
output.writeLine(data, node, state, true);
termsPrinted++;
if (termsPrinted > maxTermsToPrint) {
log.trace(EXCEEDED_MAX_TERMS_MESSAGE + maxTermsToPrint + TERMS_PRINTED_MESSAGE + termsPrinted);
} else {
output.writeLine(data, node, state, true);
}
}
return state;
}
Expand All @@ -1306,7 +1407,12 @@ public Object visit(ASTVar node, Object data) {
public Object visit(ASTNumberLiteral node, Object data) {
STATE state = STATE.IGNORABLE;
if (output != null) {
output.writeLine(data, node, ":" + node.getLiteral(), state, true);
termsPrinted++;
if (termsPrinted > maxTermsToPrint) {
log.trace(EXCEEDED_MAX_TERMS_MESSAGE + maxTermsToPrint + TERMS_PRINTED_MESSAGE + termsPrinted);
} else {
output.writeLine(data, node, ":" + node.getLiteral(), state, true);
}
}
return state;
}
Expand Down
4 changes: 4 additions & 0 deletions warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,11 +82,15 @@ public class ShardQueryConfigurationTest {
// The set of fields that are already set via one of the other fields.
private final Set<String> alreadySet = new HashSet<>();

private int maxTermsToPrint;

@Before
public void setUp() throws Exception {
// The set of default values (optionally as predicates,
// alternate values (to test the setters/getters),
// and optional alternate predicates for testing equality.
defaultValues.put("maxTermsToPrint", 100);
updatedValues.put("maxTermsToPrint", 150);
defaultValues.put("authorizations", Collections.singleton(Authorizations.EMPTY));
updatedValues.put("authorizations", Collections.singleton(new Authorizations("FOO", "BAR")));

Expand Down
Loading

0 comments on commit 5f2f0de

Please sign in to comment.