From 57cb6c2724c140a8fe850df944db12da745f5cee Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Fri, 5 Dec 2025 14:01:03 +0000 Subject: [PATCH 01/22] map -> multiMap for pullUp rules --- .../AggregateIndexExpansionVisitor.java | 5 ++- .../BitmapAggregateIndexExpansionVisitor.java | 5 ++- .../KeyExpressionExpansionVisitor.java | 5 ++- .../record/query/plan/cascades/MatchInfo.java | 16 ++++---- .../record/query/plan/cascades/Ordering.java | 14 ++++++- .../plan/cascades/values/QuantifiedValue.java | 16 ++++---- .../query/plan/cascades/values/Value.java | 31 +++++++-------- .../CompensateRecordConstructorRule.java | 28 +++++++++----- .../MatchConstantValueRule.java | 10 +++-- ...ValueAgainstQuantifiedObjectValueRule.java | 14 ++++--- .../MatchOrCompensateFieldValueRule.java | 14 ++++--- ...ValueAgainstQuantifiedObjectValueRule.java | 16 ++++---- .../values/simplification/MatchValueRule.java | 10 +++-- .../simplification/PullUpValueRuleSet.java | 21 +++++----- .../values/translation/MaxMatchMap.java | 2 +- .../cascades/values/translation/PullUp.java | 5 ++- .../simplification/ValueComputationTest.java | 38 ++++++++++++++++--- .../recordlayer/query/Expression.java | 3 +- 18 files changed, 156 insertions(+), 97 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexExpansionVisitor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexExpansionVisitor.java index 86fe88a819..2a7dea349e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexExpansionVisitor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexExpansionVisitor.java @@ -53,6 +53,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -246,7 +247,7 @@ protected NonnullPair> constructGroupBy(@Nonnull f throw new RecordCoreException("could not pull grouped value " + groupedValue) .addLogInfo(LogMessageKeys.VALUE, groupedValue); } - argument = result.get(groupedValue); + argument = Iterables.getOnlyElement(result.get(groupedValue)); } else { throw new RecordCoreException("unable to plan group by with non-field value") .addLogInfo(LogMessageKeys.VALUE, groupedValue); @@ -272,7 +273,7 @@ protected NonnullPair> constructGroupBy(@Nonnull f throw new RecordCoreException("could not pull grouping value " + groupingValue) .addLogInfo(LogMessageKeys.VALUE, groupingValue); } - return pulledUpGroupingValuesMap.get(groupingValue); + return Iterables.getOnlyElement(pulledUpGroupingValuesMap.get(groupingValue)); }).collect(ImmutableList.toImmutableList()); final var groupingColsValue = RecordConstructorValue.ofUnnamed(pulledUpGroupingValues); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BitmapAggregateIndexExpansionVisitor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BitmapAggregateIndexExpansionVisitor.java index 44b5ffb846..d173e65d27 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BitmapAggregateIndexExpansionVisitor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BitmapAggregateIndexExpansionVisitor.java @@ -41,6 +41,7 @@ import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import javax.annotation.Nonnull; @@ -82,7 +83,7 @@ protected NonnullPair> constructGroupBy(@Nonnull f throw new RecordCoreException("could not pull grouped value " + groupedValue) .addLogInfo(LogMessageKeys.VALUE, groupedValue); } - argument = result.get(groupedValue); + argument = Iterables.getOnlyElement(result.get(groupedValue)); } else { throw new UnsupportedOperationException("unable to plan group by with non-field value " + groupedValue); } @@ -114,7 +115,7 @@ protected NonnullPair> constructGroupBy(@Nonnull f throw new RecordCoreException("could not pull grouping value " + groupingValue) .addLogInfo(LogMessageKeys.VALUE, groupingValue); } - return pulledUpGroupingValuesMap.get(groupingValue); + return Iterables.getOnlyElement(pulledUpGroupingValuesMap.get(groupingValue)); }).collect(ImmutableList.toImmutableList()); final var pulledUpGroupingValues = ImmutableList.builder().addAll(explicitPulledUpGroupingValues).add(implicitGroupingValue).build(); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java index 19b2666024..c21d9d2937 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java @@ -44,6 +44,7 @@ import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import javax.annotation.Nonnull; @@ -349,7 +350,7 @@ private static ImmutableList pullUpPlaceholders(@Nonnull final Grap final var pulledUpValue = pulledUpPlaceholderValuesMap.get(value); final var parameterAlias = Objects.requireNonNull(childExpansionPlaceholderValuesMap.get(value)); - return Placeholder.newInstanceWithoutRanges(pulledUpValue, parameterAlias); + return Placeholder.newInstanceWithoutRanges(Iterables.getOnlyElement(pulledUpValue), parameterAlias); }) .collect(ImmutableList.toImmutableList()); } @@ -372,7 +373,7 @@ private static ImmutableList> pullUpResultColumns(@Nonnu throw new RecordCoreException("could not pull expansion value " + value) .addLogInfo(LogMessageKeys.VALUE, value); } - return pulledUpValuesMap.get(value); + return Iterables.getOnlyElement(pulledUpValuesMap.get(value)); }) .map(Column::unnamedOf) .collect(ImmutableList.toImmutableList()); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchInfo.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchInfo.java index 17b6ba8842..434f34002e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchInfo.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchInfo.java @@ -121,8 +121,8 @@ static ImmutableBiMap adjustMatchedValueMap(@Nonnull final Correla candidateLowerExpression.getCorrelatedTo()), candidateAlias); final var pulledUpCandidateAggregateValue = candidatePullUpMap.get(candidateAggregateValue); - if (pulledUpCandidateAggregateValue != null) { - adjustedMatchedAggregateMapBuilder.put(queryAggregateValue, pulledUpCandidateAggregateValue); + if (!pulledUpCandidateAggregateValue.isEmpty()) { + adjustedMatchedAggregateMapBuilder.put(queryAggregateValue, Iterables.getOnlyElement(pulledUpCandidateAggregateValue)); } } return adjustedMatchedAggregateMapBuilder.build(); @@ -506,10 +506,10 @@ public static GroupByMappings pullUpGroupByMappings(@Nonnull final PartialMatch Sets.difference(queryAggregateValue.getCorrelatedToWithoutChildren(), queryExpression.getCorrelatedTo()), queryAlias); final var pulledUpQueryAggregateValue = pullUpMap.get(queryAggregateValue); - if (pulledUpQueryAggregateValue == null) { + if (pulledUpQueryAggregateValue.isEmpty()) { return GroupByMappings.empty(); } - unmatchedAggregateMapBuilder.put(unmatchedAggregateMapEntry.getKey(), pulledUpQueryAggregateValue); + unmatchedAggregateMapBuilder.put(unmatchedAggregateMapEntry.getKey(), Iterables.getOnlyElement(pulledUpQueryAggregateValue)); } return GroupByMappings.of(matchedGroupingsMap, matchedAggregatesMap, unmatchedAggregateMapBuilder.build()); @@ -527,8 +527,8 @@ private static ImmutableBiMap pullUpMatchedValueMap(@Nonnull final final var pullUpMap = queryResultValue.pullUp(ImmutableList.of(queryValue), EvaluationContext.empty(), AliasMap.emptyMap(), constantAliases, queryAlias); - final Value pulledUpQueryValue = pullUpMap.get(queryValue); - if (pulledUpQueryValue == null) { + final var pulledUpQueryValue = pullUpMap.get(queryValue); + if (pulledUpQueryValue.isEmpty()) { continue; } @@ -544,10 +544,10 @@ private static ImmutableBiMap pullUpMatchedValueMap(@Nonnull final candidateLowerExpression.getCorrelatedTo()), candidateAlias); final var pulledUpCandidateAggregateValue = candidatePullUpMap.get(candidateAggregateValue); - if (pulledUpCandidateAggregateValue == null) { + if (pulledUpCandidateAggregateValue.isEmpty()) { continue; } - matchedAggregatesMapBuilder.put(pulledUpQueryValue, pulledUpCandidateAggregateValue); + matchedAggregatesMapBuilder.put(Iterables.getOnlyElement(pulledUpQueryValue), Iterables.getOnlyElement(pulledUpCandidateAggregateValue)); } return matchedAggregatesMapBuilder.build(); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java index e372da639a..5a64da6ed1 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java @@ -57,6 +57,7 @@ import java.util.function.BiPredicate; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; /** * This class captures an ordering property. @@ -400,7 +401,11 @@ public Ordering pullUp(@Nonnull final Value value, @Nonnull EvaluationContext ev final var pulledUpBindings = translateBindings(entry.getValue(), toBePulledUpValues -> value.pullUp(toBePulledUpValues, evaluationContext, - aliasMap, constantAliases, Quantifier.current())); + aliasMap, constantAliases, Quantifier.current()).asMap().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> e.getValue().iterator().next() // first element + ))); pulledUpBindingMapBuilder.putAll(entry.getKey(), pulledUpBindings); } @@ -408,7 +413,12 @@ public Ordering pullUp(@Nonnull final Value value, @Nonnull EvaluationContext ev final var pulledUpBindingMap = pulledUpBindingMapBuilder.build(); final var pulledUpValuesMap = value.pullUp(pulledUpBindingMap.keySet(), evaluationContext, aliasMap, constantAliases, - Quantifier.current()); + Quantifier.current()) + .asMap().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> e.getValue().iterator().next() // first element + )); final var mappedOrderingSet = getOrderingSet().mapAll(pulledUpValuesMap); final var mappedValues = mappedOrderingSet.getSet(); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/QuantifiedValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/QuantifiedValue.java index d4fa8e00f2..65ee51d1d1 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/QuantifiedValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/QuantifiedValue.java @@ -26,12 +26,12 @@ import com.apple.foundationdb.record.query.plan.cascades.ConstrainedBoolean; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; import com.google.common.collect.Streams; import javax.annotation.Nonnull; -import java.util.Map; import java.util.Set; /** @@ -58,11 +58,11 @@ default ConstrainedBoolean equalsWithoutChildren(@Nonnull final Value other) { @Nonnull @Override - default Map pullUp(@Nonnull final Iterable toBePulledUpValues, - @Nonnull final EvaluationContext evaluationContext, - @Nonnull final AliasMap aliasMap, - @Nonnull final Set constantAliases, - @Nonnull final CorrelationIdentifier upperBaseAlias) { + default Multimap pullUp(@Nonnull final Iterable toBePulledUpValues, + @Nonnull final EvaluationContext evaluationContext, + @Nonnull final AliasMap aliasMap, + @Nonnull final Set constantAliases, + @Nonnull final CorrelationIdentifier upperBaseAlias) { final var alias = getAlias(); final var areSimpleReferences = Streams.stream(toBePulledUpValues) @@ -71,7 +71,7 @@ default Map pullUp(@Nonnull final Iterable toBePu if (areSimpleReferences) { final var translationMap = TranslationMap.rebaseWithAliasMap(AliasMap.ofAliases(alias, upperBaseAlias)); - final var translatedMapBuilder = ImmutableMap.builder(); + final var translatedMapBuilder = ImmutableMultimap.builder(); for (final var toBePulledUpValue : toBePulledUpValues) { translatedMapBuilder.put(toBePulledUpValue, toBePulledUpValue.translateCorrelations(translationMap)); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/Value.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/Value.java index 0b5da02fef..738d306830 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/Value.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/Value.java @@ -69,9 +69,10 @@ import com.google.common.base.Functions; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.common.collect.Multimap; import com.google.common.collect.Streams; import com.google.common.primitives.ImmutableIntArray; import com.google.protobuf.Message; @@ -81,7 +82,6 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -508,30 +508,27 @@ default Value simplify(@Nonnull final EvaluationContext evaluationContext, * resulting values of the pull-up logic */ @Nonnull - default Map pullUp(@Nonnull final Iterable toBePulledUpValues, - @Nonnull final EvaluationContext evaluationContext, - @Nonnull final AliasMap aliasMap, - @Nonnull final Set constantAliases, - @Nonnull final CorrelationIdentifier upperBaseAlias) { + default Multimap pullUp(@Nonnull final Iterable toBePulledUpValues, + @Nonnull final EvaluationContext evaluationContext, + @Nonnull final AliasMap aliasMap, + @Nonnull final Set constantAliases, + @Nonnull final CorrelationIdentifier upperBaseAlias) { final var resultPair = Simplification.compute(this, evaluationContext, toBePulledUpValues, aliasMap, constantAliases, PullUpValueRuleSet.ofPullUpValueRules()); if (resultPair == null) { - return ImmutableMap.of(); + return ImmutableMultimap.of(); } final var matchedValuesMap = resultPair.getRight(); - final var resultsMapBuilder = ImmutableMap.builder(); + final var resultsMapBuilder = ImmutableMultimap.builder(); for (final var toBePulledUpValue : toBePulledUpValues) { - final var compensation = matchedValuesMap.get(toBePulledUpValue); - if (compensation != null) { - resultsMapBuilder.put(toBePulledUpValue, - compensation.compensate( - QuantifiedObjectValue.of(upperBaseAlias, this.getResultType()))); - } + matchedValuesMap.getOrDefault(toBePulledUpValue, ImmutableList.of()) + .forEach(compensation -> resultsMapBuilder.put(toBePulledUpValue, + compensation.compensate(QuantifiedObjectValue.of(upperBaseAlias, this.getResultType())))); } - return resultsMapBuilder.buildKeepingLast(); + return resultsMapBuilder.build(); } /** @@ -547,7 +544,7 @@ default Map pullUp(@Nonnull final Iterable toBePu * @param aliasMap an alias map of equalities * @param constantAliases a set of aliases that are considered to be constant * @param upperBaseAlias an alias to be treated as current alias - * @return a map from {@link Value} to {@link Value} that related the values that the called passed in with the + * @return a list of {@link Value}s that related the values that the called passed in with the * resulting values of the pull-up logic */ @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/CompensateRecordConstructorRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/CompensateRecordConstructorRule.java index e8c4a1e3a4..1e9d28ab1e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/CompensateRecordConstructorRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/CompensateRecordConstructorRule.java @@ -26,8 +26,10 @@ import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.google.common.collect.ImmutableList; import javax.annotation.Nonnull; +import java.util.List; import java.util.Map; import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.MultiMatcher.all; @@ -44,7 +46,7 @@ */ @API(API.Status.EXPERIMENTAL) @SuppressWarnings("PMD.TooManyStaticImports") -public class CompensateRecordConstructorRule extends ValueComputationRule, Map, RecordConstructorValue> { +public class CompensateRecordConstructorRule extends ValueComputationRule, Map>, RecordConstructorValue> { @Nonnull private static final BindingMatcher rootMatcher = recordConstructorValue(all(anyValue())); @@ -54,17 +56,18 @@ public CompensateRecordConstructorRule() { } @Override - public void onMatch(@Nonnull final ValueComputationRuleCall, Map> call) { + public void onMatch(@Nonnull final ValueComputationRuleCall, Map>> call) { final var bindings = call.getBindings(); final var recordConstructorValue = bindings.get(rootMatcher); - final var resultingMatchedValuesMap = new LinkedIdentityMap(); + final var resultingMatchedValuesMap = new LinkedIdentityMap>(); final var recordConstructorValueResult = call.getResult(recordConstructorValue); - final var matchedCompensation = recordConstructorValueResult == null + final var matchedCompensations = recordConstructorValueResult == null ? null : recordConstructorValueResult.getValue().get(recordConstructorValue); - if (matchedCompensation != null) { - resultingMatchedValuesMap.put(recordConstructorValue, matchedCompensation); + if (matchedCompensations != null) { + resultingMatchedValuesMap.put(recordConstructorValue, matchedCompensations); } else { + final var compensationsBuilderMap = new LinkedIdentityMap>(); for (int i = 0; i < recordConstructorValue.getColumns().size(); ++i) { final var column = recordConstructorValue.getColumns().get(i); final var childResultPair = call.getResult(column.getValue()); @@ -78,14 +81,19 @@ public void onMatch(@Nonnull final ValueComputationRuleCall new ImmutableList.Builder<>()) + .addAll(argumentValueCompensations + .stream() + .map(compensation -> new FieldValueCompensation(FieldValue.FieldPath.ofSingle(field, columnIdx), compensation)) + .iterator()); } } + compensationsBuilderMap.forEach((key, value) -> resultingMatchedValuesMap.put(key, value.build())); } - call.yieldValue(recordConstructorValue, resultingMatchedValuesMap); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchConstantValueRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchConstantValueRule.java index e347050ef2..f8839b0379 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchConstantValueRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchConstantValueRule.java @@ -27,8 +27,10 @@ import com.apple.foundationdb.record.query.plan.cascades.values.AggregateValue; import com.apple.foundationdb.record.query.plan.cascades.values.QueriedValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.google.common.collect.ImmutableList; import javax.annotation.Nonnull; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -38,7 +40,7 @@ */ @API(API.Status.EXPERIMENTAL) @SuppressWarnings("PMD.TooManyStaticImports") -public class MatchConstantValueRule extends ValueComputationRule, Map, Value> { +public class MatchConstantValueRule extends ValueComputationRule, Map>, Value> { @Nonnull private static final BindingMatcher rootMatcher = ValueMatchers.anyValue(); @@ -54,7 +56,7 @@ public Optional> getRootOperator() { } @Override - public void onMatch(@Nonnull final ValueComputationRuleCall, Map> call) { + public void onMatch(@Nonnull final ValueComputationRuleCall, Map>> call) { if (!call.isRoot()) { return; } @@ -62,7 +64,7 @@ public void onMatch(@Nonnull final ValueComputationRuleCall(); + final var newMatchedValuesMap = new LinkedIdentityMap>(); final var resultPair = call.getResult(value); final var matchedValuesMap = resultPair == null ? null : resultPair.getRight(); if (matchedValuesMap != null) { @@ -79,7 +81,7 @@ public void onMatch(@Nonnull final ValueComputationRuleCall toBePulledUpValue); + newMatchedValuesMap.put(toBePulledUpValue, ImmutableList.of(ignored -> toBePulledUpValue)); } } call.yieldValue(value, newMatchedValuesMap); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchFieldValueAgainstQuantifiedObjectValueRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchFieldValueAgainstQuantifiedObjectValueRule.java index df8e40208a..f74e9575ee 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchFieldValueAgainstQuantifiedObjectValueRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchFieldValueAgainstQuantifiedObjectValueRule.java @@ -27,9 +27,11 @@ import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.google.common.collect.ImmutableList; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -40,7 +42,7 @@ */ @API(API.Status.EXPERIMENTAL) @SuppressWarnings("PMD.TooManyStaticImports") -public class MatchFieldValueAgainstQuantifiedObjectValueRule extends ValueComputationRule, Map, QuantifiedObjectValue> { +public class MatchFieldValueAgainstQuantifiedObjectValueRule extends ValueComputationRule, Map>, QuantifiedObjectValue> { @Nonnull private static final BindingMatcher rootMatcher = ValueMatchers.quantifiedObjectValue(); @@ -50,7 +52,7 @@ public MatchFieldValueAgainstQuantifiedObjectValueRule() { } @Override - public void onMatch(@Nonnull final ValueComputationRuleCall, Map> call) { + public void onMatch(@Nonnull final ValueComputationRuleCall, Map>> call) { final var bindings = call.getBindings(); final var quantifiedObjectValue = bindings.get(rootMatcher); final var toBePulledUpValues = Objects.requireNonNull(call.getArgument()); @@ -59,14 +61,14 @@ public void onMatch(@Nonnull final ValueComputationRuleCall(); + final var newMatchedValuesMap = new LinkedIdentityMap>(); for (final var toBePulledUpValue : toBePulledUpValues) { if (toBePulledUpValue instanceof FieldValue) { final var toBePulledUpFieldValue = (FieldValue)toBePulledUpValue; if (quantifiedObjectValue.semanticEquals(toBePulledUpFieldValue.getChild(), equivalenceMap)) { - newMatchedValuesMap.put(toBePulledUpValue, new FieldValueCompensation(toBePulledUpFieldValue.getFieldPath())); + newMatchedValuesMap.put(toBePulledUpValue, ImmutableList.of(new FieldValueCompensation(toBePulledUpFieldValue.getFieldPath()))); } } else { inheritMatchedMapEntry(matchedValuesMap, newMatchedValuesMap, toBePulledUpValue); @@ -75,8 +77,8 @@ public void onMatch(@Nonnull final ValueComputationRuleCall matchedValuesMap, - @Nonnull final Map newMatchedValuesMap, + private static void inheritMatchedMapEntry(@Nullable final Map> matchedValuesMap, + @Nonnull final Map> newMatchedValuesMap, @Nonnull final Value toBePulledUpValue) { if (matchedValuesMap != null && matchedValuesMap.containsKey(toBePulledUpValue)) { newMatchedValuesMap.put(toBePulledUpValue, matchedValuesMap.get(toBePulledUpValue)); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchOrCompensateFieldValueRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchOrCompensateFieldValueRule.java index 1fd0267101..dbf6b71a1b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchOrCompensateFieldValueRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchOrCompensateFieldValueRule.java @@ -28,8 +28,10 @@ import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.google.common.collect.ImmutableList; import javax.annotation.Nonnull; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -42,7 +44,7 @@ */ @API(API.Status.EXPERIMENTAL) @SuppressWarnings("PMD.TooManyStaticImports") -public class MatchOrCompensateFieldValueRule extends ValueComputationRule, Map, FieldValue> { +public class MatchOrCompensateFieldValueRule extends ValueComputationRule, Map>, FieldValue> { @Nonnull private static final CollectionMatcher fieldPathOrdinalsMatcher = all(anyObject()); @@ -58,7 +60,7 @@ public MatchOrCompensateFieldValueRule() { } @Override - public void onMatch(@Nonnull final ValueComputationRuleCall, Map> call) { + public void onMatch(@Nonnull final ValueComputationRuleCall, Map>> call) { final var bindings = call.getBindings(); final var fieldValue = bindings.get(rootMatcher); @@ -67,7 +69,7 @@ public void onMatch(@Nonnull final ValueComputationRuleCall(); + final var newMatchedValuesMap = new LinkedIdentityMap>(); for (final var toBePulledUpValue : toBePulledUpValues) { if (toBePulledUpValue instanceof FieldValue) { @@ -83,9 +85,9 @@ public void onMatch(@Nonnull final ValueComputationRuleCall { if (pathSuffix.isEmpty()) { - newMatchedValuesMap.put(toBePulledUpValue, ValueCompensation.noCompensation()); + newMatchedValuesMap.put(toBePulledUpValue, ImmutableList.of(ValueCompensation.noCompensation())); } else { - newMatchedValuesMap.put(toBePulledUpValue, new FieldValueCompensation(pathSuffix)); + newMatchedValuesMap.put(toBePulledUpValue, ImmutableList.of(new FieldValueCompensation(pathSuffix))); } }); } @@ -95,7 +97,7 @@ public void onMatch(@Nonnull final ValueComputationRuleCall newMatchedValuesMap.put(toBePulledUpValue, fieldValueCompensation.withSuffix(pathSuffix))); + pathSuffixOptional.ifPresent(pathSuffix -> newMatchedValuesMap.put(toBePulledUpValue, ImmutableList.of(fieldValueCompensation.withSuffix(pathSuffix)))); } } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueAgainstQuantifiedObjectValueRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueAgainstQuantifiedObjectValueRule.java index 548f1a7665..3e1d6f087f 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueAgainstQuantifiedObjectValueRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueAgainstQuantifiedObjectValueRule.java @@ -28,10 +28,12 @@ import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -42,7 +44,7 @@ */ @API(API.Status.EXPERIMENTAL) @SuppressWarnings("PMD.TooManyStaticImports") -public class MatchValueAgainstQuantifiedObjectValueRule extends ValueComputationRule, Map, QuantifiedObjectValue> { +public class MatchValueAgainstQuantifiedObjectValueRule extends ValueComputationRule, Map>, QuantifiedObjectValue> { @Nonnull private static final BindingMatcher rootMatcher = ValueMatchers.quantifiedObjectValue(); @@ -52,7 +54,7 @@ public MatchValueAgainstQuantifiedObjectValueRule() { } @Override - public void onMatch(@Nonnull final ValueComputationRuleCall, Map> call) { + public void onMatch(@Nonnull final ValueComputationRuleCall, Map>> call) { final var bindings = call.getBindings(); final var quantifiedObjectValue = bindings.get(rootMatcher); final var toBePulledUpValues = Objects.requireNonNull(call.getArgument()); @@ -60,7 +62,7 @@ public void onMatch(@Nonnull final ValueComputationRuleCall(); + final var newMatchedValuesMap = new LinkedIdentityMap>(); for (final var toBePulledUpValue : toBePulledUpValues) { if (toBePulledUpValue instanceof FieldValue || @@ -82,18 +84,18 @@ public void onMatch(@Nonnull final ValueComputationRuleCall { final var translationMapBuilder = TranslationMap.regularBuilder(); translationMapBuilder.when(alias).then(((sourceAlias, leafValue) -> value)); return toBePulledUpValue.translateCorrelations(translationMapBuilder.build()); - })); + }))); } call.yieldValue(quantifiedObjectValue, newMatchedValuesMap); } - private static void inheritMatchedMapEntry(@Nullable final Map matchedValuesMap, - @Nonnull final Map newMatchedValuesMap, + private static void inheritMatchedMapEntry(@Nullable final Map> matchedValuesMap, + @Nonnull final Map> newMatchedValuesMap, @Nonnull final Value toBePulledUpValue) { if (matchedValuesMap != null && matchedValuesMap.containsKey(toBePulledUpValue)) { newMatchedValuesMap.put(toBePulledUpValue, matchedValuesMap.get(toBePulledUpValue)); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueRule.java index 8c0f4b9069..3e31a793df 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueRule.java @@ -25,8 +25,10 @@ import com.apple.foundationdb.record.query.plan.cascades.matching.structure.BindingMatcher; import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.google.common.collect.ImmutableList; import javax.annotation.Nonnull; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -38,7 +40,7 @@ */ @API(API.Status.EXPERIMENTAL) @SuppressWarnings("PMD.TooManyStaticImports") -public class MatchValueRule extends ValueComputationRule, Map, Value> { +public class MatchValueRule extends ValueComputationRule, Map>, Value> { @Nonnull private static final BindingMatcher rootMatcher = anyValue(); @@ -53,12 +55,12 @@ public Optional> getRootOperator() { } @Override - public void onMatch(@Nonnull final ValueComputationRuleCall, Map> call) { + public void onMatch(@Nonnull final ValueComputationRuleCall, Map>> call) { final var bindings = call.getBindings(); final var value = bindings.get(rootMatcher); final var toBePulledUpValues = Objects.requireNonNull(call.getArgument()); - final var newMatchedValuesMap = new LinkedIdentityMap(); + final var newMatchedValuesMap = new LinkedIdentityMap>(); final var resultPair = call.getResult(value); final var matchedValuesMap = resultPair == null ? null : resultPair.getRight(); @@ -69,7 +71,7 @@ public void onMatch(@Nonnull final ValueComputationRuleCall, Map> { - protected static final ValueComputationRule, Map, ? extends Value> matchValueRule = new MatchValueRule(); - protected static final ValueComputationRule, Map, ? extends Value> matchValueAgainstQuantifiedObjectValueRule = new MatchValueAgainstQuantifiedObjectValueRule(); - protected static final ValueComputationRule, Map, ? extends Value> matchFieldValueAgainstQuantifiedObjectValueRule = new MatchFieldValueAgainstQuantifiedObjectValueRule(); - protected static final ValueComputationRule, Map, ? extends Value> matchOrCompensateFieldValueRule = new MatchOrCompensateFieldValueRule(); - protected static final ValueComputationRule, Map, ? extends Value> compensateRecordConstructorRule = new CompensateRecordConstructorRule(); - protected static final ValueComputationRule, Map, ? extends Value> matchConstantValueRule = new MatchConstantValueRule(); +public class PullUpValueRuleSet extends ValueComputationRuleSet, Map>> { + protected static final ValueComputationRule, Map>, ? extends Value> matchValueRule = new MatchValueRule(); + protected static final ValueComputationRule, Map>, ? extends Value> matchValueAgainstQuantifiedObjectValueRule = new MatchValueAgainstQuantifiedObjectValueRule(); + protected static final ValueComputationRule, Map>, ? extends Value> matchFieldValueAgainstQuantifiedObjectValueRule = new MatchFieldValueAgainstQuantifiedObjectValueRule(); + protected static final ValueComputationRule, Map>, ? extends Value> matchOrCompensateFieldValueRule = new MatchOrCompensateFieldValueRule(); + protected static final ValueComputationRule, Map>, ? extends Value> compensateRecordConstructorRule = new CompensateRecordConstructorRule(); + protected static final ValueComputationRule, Map>, ? extends Value> matchConstantValueRule = new MatchConstantValueRule(); - protected static final Set, Map, ? extends Value>> PULL_UP_RULES = + protected static final Set, Map>, ? extends Value>> PULL_UP_RULES = ImmutableSet.of(matchValueRule, matchValueAgainstQuantifiedObjectValueRule, matchFieldValueAgainstQuantifiedObjectValueRule, @@ -50,11 +51,11 @@ public class PullUpValueRuleSet extends ValueComputationRuleSet, Map, ? extends Value>, ValueComputationRule, Map, ? extends Value>> PULL_UP_DEPENDS_ON; + protected static final SetMultimap, Map>, ? extends Value>, ValueComputationRule, Map>, ? extends Value>> PULL_UP_DEPENDS_ON; static { final var dependsOnBuilder = - ImmutableSetMultimap., Map, ? extends Value>, ValueComputationRule, Map, ? extends Value>>builder(); + ImmutableSetMultimap., Map>, ? extends Value>, ValueComputationRule, Map>, ? extends Value>>builder(); PULL_UP_RULES.forEach(rule -> { if (rule != matchConstantValueRule) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/MaxMatchMap.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/MaxMatchMap.java index e25e27eb78..72e0770716 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/MaxMatchMap.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/MaxMatchMap.java @@ -254,7 +254,7 @@ public Optional translateQueryValueMaybe(@Nonnull final CorrelationIdenti for (final var entry : mapping.entrySet()) { final var queryPart = entry.getKey(); final var candidatePart = entry.getValue(); - final var pulledUpdateCandidatePart = pulledUpCandidateValueMap.get(candidatePart); + final var pulledUpdateCandidatePart = Iterables.getOnlyElement(pulledUpCandidateValueMap.get(candidatePart)); if (pulledUpdateCandidatePart == null) { return Optional.empty(); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/PullUp.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/PullUp.java index 29e100b2f0..cbc3b74b58 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/PullUp.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/PullUp.java @@ -33,6 +33,7 @@ import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import javax.annotation.Nonnull; @@ -177,10 +178,10 @@ public Optional pullUpCandidateValueMaybe(@Nonnull final Value value) { currentRangedOverAliases), currentCandidateAlias); final var pulledUpCandidateAggregateValue = candidatePullUpMap.get(currentValue); - if (pulledUpCandidateAggregateValue == null) { + if (pulledUpCandidateAggregateValue.isEmpty()) { return Optional.empty(); } - currentValue = pulledUpCandidateAggregateValue; + currentValue = Iterables.getOnlyElement(pulledUpCandidateAggregateValue); if (currentPullUp.getParentPullUp() == null) { return Optional.of(currentValue); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueComputationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueComputationTest.java index bdcb8ee0a0..075cc097ae 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueComputationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueComputationTest.java @@ -23,6 +23,7 @@ import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.query.expressions.Comparisons; import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.Column; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.ValueEquivalence; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; @@ -37,7 +38,7 @@ import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.tuple.TupleOrdering; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -89,13 +90,40 @@ void testPullUpFieldValue1() { final var upperCurrentValue = QuantifiedObjectValue.of(ALIAS, pulledThroughValue.getResultType()); - final var expectedMap = ImmutableMap.of( + final var expectedMap = ImmutableMultimap.of( _x_xa, FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_0", "xa")), _z, FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_1"))); Assertions.assertEquals(expectedMap, resultsMap); } + @Test + void testPullUpAmbiguous() { + final var someCurrentValue = ObjectValue.of(ALIAS, someRecordType()); + // (_.x as _0, _.x.xa as _1) + final var pulledThroughValue = RecordConstructorValue.ofColumns(ImmutableList.of( + Column.unnamedOf(FieldValue.ofFieldNames(someCurrentValue, ImmutableList.of("x"))), + Column.unnamedOf(FieldValue.ofFieldNames(someCurrentValue, ImmutableList.of("x", "xa"))) + )); + + // _.x.xa + final var _x_xa = FieldValue.ofFieldNames(someCurrentValue, ImmutableList.of("x", "xa")); + final var toBePulledUpValues = ImmutableList.of(_x_xa); + + final var resultsMap = pulledThroughValue.pullUp(toBePulledUpValues, + EvaluationContext.empty(), AliasMap.emptyMap(), ImmutableSet.of(), ALIAS); + + final var upperCurrentValue = QuantifiedObjectValue.of(ALIAS, pulledThroughValue.getResultType()); + + // _.x.xa -> { _0.xa, _1 } + final var expectedMap = ImmutableMultimap.of( + _x_xa, FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_0", "xa")), + _x_xa, FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_1")) + ); + + Assertions.assertEquals(expectedMap, resultsMap); + } + @Test void testPullUpFieldValueThroughStreamingAggregation() { // _ @@ -122,7 +150,7 @@ void testPullUpFieldValueThroughStreamingAggregation() { // _ final var someNewCurrentValue = QuantifiedObjectValue.of(ALIAS, completeResultValue.getResultType()); - final var expectedResult = ImmutableMap.of( + final var expectedResult = ImmutableMultimap.of( _x, FieldValue.ofOrdinalNumber(someNewCurrentValue, 0), _z, FieldValue.ofOrdinalNumber(someNewCurrentValue, 1)); Assertions.assertEquals(expectedResult, resultsMap); @@ -150,7 +178,7 @@ void testPullUpValue1() { final var new_a_b = FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("a", "ab")); final var new_x_b = FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("x", "xb")); - final var expectedMap = ImmutableMap.of( + final var expectedMap = ImmutableMultimap.of( record_type, new RecordTypeValue(QuantifiedObjectValue.of(UPPER_ALIAS, new Type.AnyRecord(true))), _a_ab__plus__x_xb, (Value)new ArithmeticValue.AddFn().encapsulate(ImmutableList.of(new_a_b, new_x_b))); Assertions.assertEquals(expectedMap, resultsMap); @@ -178,7 +206,7 @@ void testPullUpValue2() { final var new_a_b = FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_1", "a", "ab")); final var new_x_b = FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_1", "x", "xb")); - final var expectedMap = ImmutableMap.of( + final var expectedMap = ImmutableMultimap.of( record_type, new RecordTypeValue(FieldValue.ofFieldName(upperCurrentValue, "_1")), _a_ab__plus__x_xb, (Value)new ArithmeticValue.AddFn().encapsulate(ImmutableList.of(new_a_b, new_x_b))); Assertions.assertEquals(expectedMap, resultsMap); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expression.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expression.java index f73df9743f..c9457664c9 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expression.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expression.java @@ -42,6 +42,7 @@ import com.google.common.base.Suppliers; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; import com.google.common.collect.Streams; import javax.annotation.Nonnull; @@ -193,7 +194,7 @@ public Expression pullUp(@Nonnull Value value, @Nonnull CorrelationIdentifier co simplifiedValue.pullUp(List.of(subExpression), EvaluationContext.empty(), aliasMap, constantAliases, correlationIdentifier); if (pulledUpExpressionMap.containsKey(subExpression)) { - return pulledUpExpressionMap.get(subExpression); + return Iterables.getOnlyElement(pulledUpExpressionMap.get(subExpression)); } return subExpression; } From 38fe5ab3bf881467d30ec10df23b64ecda051e45 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Fri, 5 Dec 2025 15:18:42 +0000 Subject: [PATCH 02/22] fix tests --- .../query/plan/cascades/values/translation/MaxMatchMap.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/MaxMatchMap.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/MaxMatchMap.java index 72e0770716..b11fdf3084 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/MaxMatchMap.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/MaxMatchMap.java @@ -254,11 +254,11 @@ public Optional translateQueryValueMaybe(@Nonnull final CorrelationIdenti for (final var entry : mapping.entrySet()) { final var queryPart = entry.getKey(); final var candidatePart = entry.getValue(); - final var pulledUpdateCandidatePart = Iterables.getOnlyElement(pulledUpCandidateValueMap.get(candidatePart)); - if (pulledUpdateCandidatePart == null) { + final var pulledUpdateCandidatePart = pulledUpCandidateValueMap.get(candidatePart); + if (pulledUpdateCandidatePart.isEmpty()) { return Optional.empty(); } - pulledUpMaxMatchMapBuilder.put(queryPart, pulledUpdateCandidatePart); + pulledUpMaxMatchMapBuilder.put(queryPart, Iterables.getOnlyElement(pulledUpdateCandidatePart)); } final var pulledUpMaxMatchMap = pulledUpMaxMatchMapBuilder.build(); From 84230c8ed9919f5af5bf0c58e9d775baa28f0aa4 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Fri, 5 Dec 2025 15:22:43 +0000 Subject: [PATCH 03/22] fix tests --- .../MatchOrCompensateFieldValueRule.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchOrCompensateFieldValueRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchOrCompensateFieldValueRule.java index dbf6b71a1b..6bfd794fd0 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchOrCompensateFieldValueRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchOrCompensateFieldValueRule.java @@ -75,7 +75,7 @@ public void onMatch(@Nonnull final ValueComputationRuleCall newMatchedValuesMap.put(toBePulledUpValue, ImmutableList.of(fieldValueCompensation.withSuffix(pathSuffix)))); + final var compensations = matchedValuesMap.get(toBePulledUpValue); + for (var compensation: compensations) { + if (compensation instanceof FieldValueCompensation) { + final var fieldValueCompensation = (FieldValueCompensation)compensation; + final var pathSuffixOptional = FieldValue.stripFieldPrefixMaybe(fieldValueCompensation.getFieldPath(), fieldValue.getFieldPath()); + pathSuffixOptional.ifPresent(pathSuffix -> newMatchedValuesMap.put(toBePulledUpValue, ImmutableList.of(fieldValueCompensation.withSuffix(pathSuffix)))); + } } } } From a1f966d5bd4bdc944aa7d03985466b7aa5f477fb Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Fri, 5 Dec 2025 16:17:06 +0000 Subject: [PATCH 04/22] wip --- .../foundationdb/record/query/plan/cascades/Ordering.java | 7 ++++--- .../relational/recordlayer/query/Expressions.java | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java index 5a64da6ed1..5965e846b9 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java @@ -40,6 +40,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSetMultimap; import com.google.common.collect.Iterables; +import com.google.common.collect.Multimap; import com.google.common.collect.Multimaps; import com.google.common.collect.SetMultimap; import com.google.common.collect.Sets; @@ -394,7 +395,7 @@ public boolean satisfiesGroupingValues(@Nonnull final Set requestedGroupi } @Nonnull - public Ordering pullUp(@Nonnull final Value value, @Nonnull EvaluationContext evaluationContext, + public Ordering pullUp(@Nonnull final Value value /*to pull against*/, @Nonnull EvaluationContext evaluationContext, @Nonnull final AliasMap aliasMap, @Nonnull final Set constantAliases) { final var pulledUpBindingMapBuilder = ImmutableSetMultimap.builder(); for (final var entry : getBindingMap().asMap().entrySet()) { @@ -515,7 +516,7 @@ public boolean isSingularFixedValue(@Nonnull final Value value) { @Nonnull private static Set translateBindings(@Nonnull final Collection bindings, - @Nonnull final Function, Map> translateFunction) { + @Nonnull final Function, Multimap> translateFunction) { final var translatedBindingsBuilder = ImmutableSet.builder(); if (areAllBindingsFixed(bindings)) { @@ -535,7 +536,7 @@ private static Set translateBindings(@Nonnull final Collection if (translationMap.containsKey(valueComparison.getValue())) { final var translatedComparison = new Comparisons.ValueComparison(valueComparison.getType(), - translationMap.get(valueComparison.getValue())); + Iterables.getOnlyElement(translationMap.get(valueComparison.getValue()))); translatedBindingsBuilder.add(Binding.fixed(translatedComparison)); } } else { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java index 93c0d4604c..51c1ac0e09 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java @@ -101,7 +101,7 @@ public Expressions pullUp(@Nonnull Value value, @Nonnull CorrelationIdentifier c simplifiedValue.pullUp(List.of(subExpression), EvaluationContext.empty(), aliasMap, constantAliases, correlationIdentifier); if (pulledUpExpressionMap.containsKey(subExpression)) { - return pulledUpExpressionMap.get(subExpression); + return Iterables.getOnlyElement(pulledUpExpressionMap.get(subExpression)); } return subExpression; } From 7cbd0e8adc009cb49c12c62b65e44747a2eadcd5 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Fri, 5 Dec 2025 16:25:55 +0000 Subject: [PATCH 05/22] oldie From 8d000bf4757396d211b412ede4f4a53c3af78c5b Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Fri, 5 Dec 2025 18:24:12 +0000 Subject: [PATCH 06/22] multiple pull-up values in ordering --- .../combinatorics/PartiallyOrderedSet.java | 24 +++++++++ .../record/query/plan/cascades/Ordering.java | 42 ++++++--------- .../recordlayer/query/OrderByExpression.java | 3 +- .../src/test/java/YamlIntegrationTests.java | 5 ++ yaml-tests/src/test/resources/test.yamsql | 53 +++++++++++++++++++ 5 files changed, 101 insertions(+), 26 deletions(-) create mode 100644 yaml-tests/src/test/resources/test.yamsql diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java index efe2d10dd2..b57838926b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java @@ -29,6 +29,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSetMultimap; import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; import com.google.common.collect.SetMultimap; import com.google.common.collect.Sets; @@ -191,6 +192,29 @@ public PartiallyOrderedSet mapAll(@Nonnull final Map map) { return PartiallyOrderedSet.of(mappedElements, resultDependencyMapBuilder.build()); } + @Nonnull + public PartiallyOrderedSet mapAll(@Nonnull final Multimap map) { + final var mappedElements = Sets.newLinkedHashSet(map.values()); + + final var resultDependencyMapBuilder = ImmutableSetMultimap.builder(); + for (final var entry : getTransitiveClosure().entries()) { + final var key = entry.getKey(); + final var value = entry.getValue(); + + if (map.containsKey(key) && map.containsKey(value)) { + map.get(key).forEach(mappedKey -> resultDependencyMapBuilder.putAll(mappedKey, map.get(value))); + } else { + if (!map.containsKey(value)) { + // if key depends on value that does not exist -- do not insert the dependency and also remove key + mappedElements.removeAll(map.get(key)); + } + } + } + + // this needs the dependency map to be cleansed (which is done in the constructor) + return PartiallyOrderedSet.of(mappedElements, resultDependencyMapBuilder.build()); + } + /** * Method that computes a new partially-ordered set that only retains elements that pass the filtering predicate. * The filtering is applied in a way that we do not break dependencies. diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java index 5965e846b9..23c2bc2a39 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java @@ -35,6 +35,7 @@ import com.apple.foundationdb.record.query.plan.cascades.values.simplification.DefaultValueSimplificationRuleSet; import com.google.common.base.Suppliers; import com.google.common.base.Verify; +import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -58,7 +59,6 @@ import java.util.function.BiPredicate; import java.util.function.Function; import java.util.function.Supplier; -import java.util.stream.Collectors; /** * This class captures an ordering property. @@ -402,33 +402,26 @@ public Ordering pullUp(@Nonnull final Value value /*to pull against*/, @Nonnull final var pulledUpBindings = translateBindings(entry.getValue(), toBePulledUpValues -> value.pullUp(toBePulledUpValues, evaluationContext, - aliasMap, constantAliases, Quantifier.current()).asMap().entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - e -> e.getValue().iterator().next() // first element - ))); - pulledUpBindingMapBuilder.putAll(entry.getKey(), pulledUpBindings); + aliasMap, constantAliases, Quantifier.current())); + pulledUpBindingMapBuilder.putAll(entry.getKey() /* old value*/, pulledUpBindings); } // pull up the values we actually could also pull up some of the bindings for final var pulledUpBindingMap = pulledUpBindingMapBuilder.build(); - final var pulledUpValuesMap = + final var pulledUpValuesMultimap = value.pullUp(pulledUpBindingMap.keySet(), evaluationContext, aliasMap, constantAliases, - Quantifier.current()) - .asMap().entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - e -> e.getValue().iterator().next() // first element - )); - - final var mappedOrderingSet = getOrderingSet().mapAll(pulledUpValuesMap); + Quantifier.current()); + + final var mappedOrderingSet = getOrderingSet().mapAll(pulledUpValuesMultimap); final var mappedValues = mappedOrderingSet.getSet(); final var bindingMapBuilder = ImmutableSetMultimap.builder(); - for (final var entry : pulledUpValuesMap.entrySet()) { - if (mappedValues.contains(entry.getValue())) { - Verify.verify(pulledUpBindingMap.containsKey(entry.getKey())); - bindingMapBuilder.putAll(entry.getValue(), pulledUpBindingMap.get(entry.getKey())); + for (final var entry : pulledUpValuesMultimap.asMap().entrySet()) { + for (final var pulledUpValue: entry.getValue()) { + if (mappedValues.contains(pulledUpValue)) { + Verify.verify(pulledUpBindingMap.containsKey(entry.getKey())); + bindingMapBuilder.putAll(pulledUpValue, pulledUpBindingMap.get(entry.getKey())); + } } } @@ -447,7 +440,7 @@ public Ordering pushDown(@Nonnull final Value value, @Nonnull final EvaluationCo value.pushDown(toBePushedValues, DefaultValueSimplificationRuleSet.instance(), evaluationContext, aliasMap, constantAliases, Quantifier.current()); - final var resultMap = new LinkedIdentityMap(); + final var resultMap = HashMultimap.create(); for (int i = 0; i < toBePushedValues.size(); i++) { final Value toBePushedValue = toBePushedValues.get(i); final Value pushedValue = Objects.requireNonNull(pushedDownValues.get(i)); @@ -534,10 +527,9 @@ private static Set translateBindings(@Nonnull final Collection if (comparison instanceof Comparisons.ValueComparison) { final var valueComparison = (Comparisons.ValueComparison)comparison; if (translationMap.containsKey(valueComparison.getValue())) { - final var translatedComparison = - new Comparisons.ValueComparison(valueComparison.getType(), - Iterables.getOnlyElement(translationMap.get(valueComparison.getValue()))); - translatedBindingsBuilder.add(Binding.fixed(translatedComparison)); + translationMap.get(valueComparison.getValue()).stream() + .map(value -> new Comparisons.ValueComparison(valueComparison.getType(), Objects.requireNonNull(value))) + .forEach(translatedValueComparison -> translatedBindingsBuilder.add(Binding.fixed(translatedValueComparison))); } } else { translatedBindingsBuilder.add(binding); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderByExpression.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderByExpression.java index 728c6883c4..51cb5716ee 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderByExpression.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderByExpression.java @@ -28,6 +28,7 @@ import com.apple.foundationdb.record.query.plan.cascades.OrderingPart; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.relational.util.Assert; +import com.google.common.collect.Iterables; import javax.annotation.Nonnull; import java.util.List; @@ -92,7 +93,7 @@ public static Stream pullUp(@Nonnull final Stream Date: Fri, 5 Dec 2025 18:29:05 +0000 Subject: [PATCH 07/22] fix yaml test --- yaml-tests/src/test/resources/test.yamsql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yaml-tests/src/test/resources/test.yamsql b/yaml-tests/src/test/resources/test.yamsql index 5ffb255f0f..c8f8e71d02 100644 --- a/yaml-tests/src/test/resources/test.yamsql +++ b/yaml-tests/src/test/resources/test.yamsql @@ -18,7 +18,7 @@ # limitations under the License. --- options: - supported_version: 4.8.3.0 + supported_version: !current_version --- schema_template: create type as struct st1(y bigint, z bigint) @@ -49,5 +49,5 @@ test_block: # - error: [{1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}, {10}] - - query: select (*) from table_func1(100) order by ordering - - result: [{ID: 1, Q: {1, 100}, ORDERING: 1}] + - result: [{{ID: 1, Q: {1, 100}, ORDERING: 1}}, {{ID: 2, Q: {2, 100}, ORDERING: 2}}, {{ID: 3, Q: {3, 100}, ORDERING: 3}}, {{ID: 4, Q: {4, 100}, ORDERING: 4}}, {{ID: 5, Q: {5, 100}, ORDERING: 5}}] ... From 727fafc8122b9b705b3daad8442e689a8b354b2d Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Mon, 8 Dec 2025 17:13:09 +0000 Subject: [PATCH 08/22] fix tests --- .../query/plan/cascades/values/Value.java | 3 +-- .../values/translation/MaxMatchMap.java | 2 +- .../cascades/values/ValueTranslationTest.java | 20 ++++++++----------- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/Value.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/Value.java index 738d306830..1c63a1dfe8 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/Value.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/Value.java @@ -544,8 +544,7 @@ default Multimap pullUp(@Nonnull final Iterable t * @param aliasMap an alias map of equalities * @param constantAliases a set of aliases that are considered to be constant * @param upperBaseAlias an alias to be treated as current alias - * @return a list of {@link Value}s that related the values that the called passed in with the - * resulting values of the pull-up logic + * @return a list of resulting {@link Value}s of the push-down logic */ @Nonnull default List pushDown(@Nonnull final Iterable toBePushedDownValues, diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/MaxMatchMap.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/MaxMatchMap.java index b11fdf3084..89aef3047c 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/MaxMatchMap.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/MaxMatchMap.java @@ -94,7 +94,7 @@ * correspondences between subtrees that include this subtree. *
* In the example above we can say that the query side value and the candidate value define maximum matches between - * {@code q.a -> q.a}, {@code q.b -> q.b}, and {@code q.b -> q.b}. If the candidate side had been + * {@code q.a -> q.a}, {@code q.b -> q.b}, and {@code q.c -> q.c}. If the candidate side had been * {@code rcv(q.a as a, q.b as b, q.c as c)}, the maximum matches between query side and candidate side would just * have been {@code rcv(q.a as a, q.b as b, q.c as c) -> rcv(q.a as a, q.b as b, q.c as c)} as the entire tree under * the root would have matched. Even though {@code q.a -> q.a}, {@code q.b -> q.b}, and {@code q.c -> q.c} are also diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ValueTranslationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ValueTranslationTest.java index 56857d9c5d..4283f79681 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ValueTranslationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ValueTranslationTest.java @@ -252,7 +252,6 @@ public void testMultiLevelValueTranslation() { fv(t_, "b", "m")), fv(t_, "j", "s"), fv(t_, "j", "q"), - fv(t_, "b", "t"), fv(t_, "b", "m") ); @@ -471,7 +470,7 @@ void maxMatchValueWithUnmatchableArithmeticOperationCase2() { public void maxMatchValueWithMatchableArithmeticOperationAndOtherConstantCorrelations() { /* 1st level: - (t.a.q + s, t.a.r, (t.b.t), t.j.s) ((t'.a.q + s', t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.t, t'.b.m) + (t.a.q + s, t.a.r, (t.b.t), t.j.s) ((t'.a.q + s', t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.m) | | T | T' | | | @@ -491,7 +490,6 @@ public void maxMatchValueWithMatchableArithmeticOperationAndOtherConstantCorrela fv(t_, "b", "m")), fv(t_, "j", "s"), fv(t_, "j", "q"), - fv(t_, "b", "t"), fv(t_, "b", "m") ); @@ -534,7 +532,7 @@ translation of (t.a.q + s, t.a.r, (t.b.t), t.j.s) with correlation mapping of t | | P | p' | | | - (t.a.q + s, t.a.r, (t.b.t), t.j.s) ((t'.a.q + s', t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.t, t'.b.m) + (t.a.q + s, t.a.r, (t.b.t), t.j.s) ((t'.a.q + s', t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.m) | | T | T' | | | @@ -600,7 +598,7 @@ private Type.Record getNType() { public void maxMatchValueWithCompositionOfTranslationMaps() { /* 1st level: - (t.a.q, t.a.r, (t.b.t), t.j.s) ((t'.a.q, t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.t) + (t.a.q, t.a.r, (t.b.t), t.j.s) ((t'.a.q, t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q) | | T | T' | | | @@ -631,8 +629,7 @@ public void maxMatchValueWithCompositionOfTranslationMaps() { rcv(fv(t_, "b", "t"), fv(t_, "b", "m")), fv(t_, "j", "s"), - fv(t_, "j", "q"), - fv(t_, "b", "t") + fv(t_, "j", "q") ); final var mv = rcv( @@ -748,7 +745,7 @@ translation of (t.a.q, t.a.r, (t.b.t), t.j.s) with correlation mapping of t -> t | P' | | - ((t'.a.q, t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.t, t'.b.m) + ((t'.a.q, t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.m) | T' | | @@ -892,8 +889,7 @@ public void maxMatchDifferentCompositions() { rcv(fv(t_, "b", "t"), fv(t_, "b", "m")), fv(t_, "j", "s"), - fv(t_, "j", "q"), - fv(t_, "b", "t") + fv(t_, "j", "q") ); final var mv = rcv( @@ -920,7 +916,7 @@ public void maxMatchDifferentCompositions() { /* 1st level: - (t.a.q, t.a.r, (t.b.t), t.j.s) ((t'.a.q, t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.t) + (t.a.q, t.a.r, (t.b.t), t.j.s) ((t'.a.q, t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q) | | T | T' | | | @@ -988,7 +984,7 @@ translation of (t.a.q, t.a.r, (t.b.t), t.j.s) with correlation mapping of t -> t | P' | | - ((t'.a.q, t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.t, t'.b.m) + ((t'.a.q, t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.m) | T' | | From 2672d16b8dba75132456572ce719420673004089 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Tue, 9 Dec 2025 11:25:40 +0000 Subject: [PATCH 09/22] fix tests --- .../simplification/ValueSimplificationTest.java | 4 ++-- .../relational/recordlayer/query/Expressions.java | 2 ++ .../foundationdb/relational/api/ddl/IndexTest.java | 12 +----------- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueSimplificationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueSimplificationTest.java index c2d97cc5c6..64de387100 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueSimplificationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueSimplificationTest.java @@ -280,8 +280,8 @@ void testSimplificationPullUp() { // _._0.b final var bFieldPulledUp = FieldValue.ofFieldNameAndFuseIfPossible(zeroFieldPulledUp, "b"); - Assertions.assertEquals(iFieldPulledUp, pulledUpValuesMap.get(iField)); - Assertions.assertEquals(bFieldPulledUp, pulledUpValuesMap.get(bField)); + Assertions.assertEquals(ImmutableList.of(iFieldPulledUp), pulledUpValuesMap.get(iField)); + Assertions.assertEquals(ImmutableList.of(bFieldPulledUp), pulledUpValuesMap.get(bField)); } @Nonnull diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java index 51c1ac0e09..418f459c25 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java @@ -29,6 +29,7 @@ import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.util.Assert; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; @@ -101,6 +102,7 @@ public Expressions pullUp(@Nonnull Value value, @Nonnull CorrelationIdentifier c simplifiedValue.pullUp(List.of(subExpression), EvaluationContext.empty(), aliasMap, constantAliases, correlationIdentifier); if (pulledUpExpressionMap.containsKey(subExpression)) { + Assert.thatUnchecked(pulledUpExpressionMap.get(subExpression).size() == 1, ErrorCode.AMBIGUOUS_COLUMN, "Ambiguous columns for " + subExpression); return Iterables.getOnlyElement(pulledUpExpressionMap.get(subExpression)); } return subExpression; diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/IndexTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/IndexTest.java index 4e7f6214eb..d371e65672 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/IndexTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/IndexTest.java @@ -313,16 +313,6 @@ void createBitMapIndexWithEmptyGroupIsSupported() throws Exception { indexIs(stmt, field("P1").ungrouped(), IndexTypes.BITMAP_VALUE); } - @Test - void createBitMapIndexWithMultipleGroupByIsSupported() throws Exception { - final String stmt = "CREATE SCHEMA TEMPLATE test_template " + - "CREATE TABLE T1(p1 bigint, a bigint, b bigint, primary key(p1)) " + - "CREATE INDEX mv1 AS SELECT bitmap_construct_agg(bitmap_bit_position(p1)) as bitmap, " + - "bitmap_bucket_offset(p1), bitmap_bucket_offset(p1), bitmap_bucket_offset(p1) as offset FROM T1\n" + - "GROUP BY bitmap_bucket_offset(p1), bitmap_bucket_offset(p1), bitmap_bucket_offset(p1)"; - indexIs(stmt, field("P1").groupBy(concat(function("bitmap_bucket_offset", concat(field("P1"), value(10000))), function("bitmap_bucket_offset", concat(field("P1"), value(10000))))), IndexTypes.BITMAP_VALUE); - } - @Test void createBitMapIndexWithRedundantFunctionsIsSupported() throws Exception { final String stmt = "CREATE SCHEMA TEMPLATE test_template " + @@ -330,7 +320,7 @@ void createBitMapIndexWithRedundantFunctionsIsSupported() throws Exception { "CREATE INDEX mv1 AS SELECT bitmap_construct_agg(bitmap_bit_position(p1)) as bitmap, " + "a, bitmap_bucket_offset(p1), b, bitmap_bucket_offset(p1) as offset FROM T1\n" + "GROUP BY a, bitmap_bucket_offset(p1), b, bitmap_bucket_offset(p1)"; - indexIs(stmt, field("P1").groupBy(concat(field("A"), function("bitmap_bucket_offset", concat(field("P1"), value(10000))), field("B"))), IndexTypes.BITMAP_VALUE); + shouldFailWith(stmt, ErrorCode.AMBIGUOUS_COLUMN, "Ambiguous columns for"); } @Test From 3c86886670bdc8b7bb25f196a50b71239c4ae415 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Tue, 9 Dec 2025 16:10:18 +0000 Subject: [PATCH 10/22] more tests --- .../recordlayer/query/OrderByExpression.java | 4 +- .../relational/api/ddl/IndexTest.java | 2 +- .../src/test/java/YamlIntegrationTests.java | 5 -- .../resources/bitmap-aggregate-index.yamsql | 2 - .../src/test/resources/orderby.metrics.binpb | 54 +++++++++++++++--- .../src/test/resources/orderby.metrics.yaml | 55 +++++++++++++++---- yaml-tests/src/test/resources/orderby.yamsql | 24 ++++++++ yaml-tests/src/test/resources/test.yamsql | 53 ------------------ 8 files changed, 117 insertions(+), 82 deletions(-) delete mode 100644 yaml-tests/src/test/resources/test.yamsql diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderByExpression.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderByExpression.java index 51cb5716ee..8908663d7a 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderByExpression.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderByExpression.java @@ -92,8 +92,8 @@ public static Stream pullUp(@Nonnull final Stream -> [A: KEY[2], B: KEY[0], C: VALUE[0]]) | MAP (_.C AS C, _.B AS B) | MAP (_.C AS C) digraph G { + +M ($0,8@cCOVERING(I1 <,> -> [A: KEY[2], B: KEY[0], C: VALUE[0]]) | MAP (_.C AS C, _.B AS B) | MAP (_.C AS C) digraph G { fontname=courier; rankdir=BT; splines=polyline; @@ -12,10 +13,10 @@ 3 -> 2 [ label=< q47> label="q47" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; 4 -> 3 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; 2 -> 1 [ label=< q6> label="q6" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; -} +} 4 - orderby-tests#EXPLAIN select b from t1 order by c -^ Ҁ(+08)@cCOVERING(I2 <,> -> [A: KEY[2], B: VALUE[0], C: KEY[0]]) | MAP (_.B AS B, _.C AS C) | MAP (_.B AS B) digraph G { + orderby-tests#EXPLAIN select b from t1 order by c +r^ ¸^(+08)@cCOVERING(I2 <,> -> [A: KEY[2], B: VALUE[0], C: KEY[0]]) | MAP (_.B AS B, _.C AS C) | MAP (_.B AS B) digraph G { fontname=courier; rankdir=BT; splines=polyline; @@ -29,7 +30,7 @@ } 9 orderby-tests(EXPLAIN select c from t1 order by b desc -M ($0 8@kCOVERING(I1 <,> REVERSE -> [A: KEY[2], B: KEY[0], C: VALUE[0]]) | MAP (_.C AS C, _.B AS B) | MAP (_.C AS C)digraph G { +ȍM ($0)8@kCOVERING(I1 <,> REVERSE -> [A: KEY[2], B: KEY[0], C: VALUE[0]]) | MAP (_.C AS C, _.B AS B) | MAP (_.C AS C)digraph G { fontname=courier; rankdir=BT; splines=polyline; @@ -43,7 +44,7 @@ } 9 orderby-tests(EXPLAIN select b from t1 order by c desc - ^ Ž(+0#8)@kCOVERING(I2 <,> REVERSE -> [A: KEY[2], B: VALUE[0], C: KEY[0]]) | MAP (_.B AS B, _.C AS C) | MAP (_.B AS B)digraph G { +IJ^ (+0r8)@kCOVERING(I2 <,> REVERSE -> [A: KEY[2], B: VALUE[0], C: KEY[0]]) | MAP (_.B AS B, _.C AS C) | MAP (_.B AS B)digraph G { fontname=courier; rankdir=BT; splines=polyline; @@ -57,7 +58,7 @@ } @ orderby-tests/EXPLAIN select c, b from t5 order by c, b desc; -5 (08@vCOVERING(I8 <,> -> [A: KEY[3], B: from_ordered_bytes(KEY:[1], DESC_NULLS_LAST), C: KEY[0]]) | MAP (_.C AS C, _.B AS B) +5 (0'8@vCOVERING(I8 <,> -> [A: KEY[3], B: from_ordered_bytes(KEY:[1], DESC_NULLS_LAST), C: KEY[0]]) | MAP (_.C AS C, _.B AS B) digraph G { fontname=courier; rankdir=BT; @@ -67,4 +68,43 @@ digraph G { 3 [ label=<
Index
I8
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, LONG AS B, LONG AS C)" ]; 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; 2 -> 1 [ label=< q32> label="q32" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +Z + orderby-testsIEXPLAIN select q as "nested", q.a as "ordered" from t2 order by "ordered" +5 (0%8@5ISCAN(I4 <,>) | MAP (_.Q AS nested, _.Q.A AS ordered) +digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (q2.Q AS nested, q2.Q.A AS ordered)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, LONG AS B AS nested, LONG AS ordered)" ]; + 2 [ label=<
Index Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS P, LONG AS A, LONG AS B AS Q)" ]; + 3 [ label=<
Index
I4
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS P, LONG AS A, LONG AS B AS Q)" ]; + 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +s + orderby-testsbEXPLAIN select (T.*) from (select q as "nested", q.a as "ordered" from t2) as T order by "ordered" +Y9 E(0؏8@=ISCAN(I4 <,>) | MAP ((_.Q AS nested, _.Q.A AS ordered) AS _0) +digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP ((q2.Q AS nested, q2.Q.A AS ordered) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, LONG AS B AS nested, LONG AS ordered AS _0)" ]; + 2 [ label=<
Index Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS P, LONG AS A, LONG AS B AS Q)" ]; + 3 [ label=<
Index
I4
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS P, LONG AS A, LONG AS B AS Q)" ]; + 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +E + orderby-tests4EXPLAIN select (*) from t2_func() order by "ordered" +9 ׽(0+8@>ISCAN(I4 <,>) | MAP ((_.Q AS nesting, _.Q.A AS ordered) AS _0) +digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP ((q2.Q AS nesting, q2.Q.A AS ordered) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, LONG AS B AS nesting, LONG AS ordered AS _0)" ]; + 2 [ label=<
Index Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS P, LONG AS A, LONG AS B AS Q)" ]; + 3 [ label=<
Index
I4
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS P, LONG AS A, LONG AS B AS Q)" ]; + 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; } \ No newline at end of file diff --git a/yaml-tests/src/test/resources/orderby.metrics.yaml b/yaml-tests/src/test/resources/orderby.metrics.yaml index 1671f87a02..19df8fa9d8 100644 --- a/yaml-tests/src/test/resources/orderby.metrics.yaml +++ b/yaml-tests/src/test/resources/orderby.metrics.yaml @@ -3,9 +3,9 @@ orderby-tests: explain: 'COVERING(I1 <,> -> [A: KEY[2], B: KEY[0], C: VALUE[0]]) | MAP (_.C AS C, _.B AS B) | MAP (_.C AS C)' task_count: 302 - task_total_time_ms: 12 + task_total_time_ms: 22 transform_count: 77 - transform_time_ms: 5 + transform_time_ms: 11 transform_yield_count: 36 insert_time_ms: 0 insert_new_count: 25 @@ -14,20 +14,20 @@ orderby-tests: explain: 'COVERING(I2 <,> -> [A: KEY[2], B: VALUE[0], C: KEY[0]]) | MAP (_.B AS B, _.C AS C) | MAP (_.B AS B)' task_count: 407 - task_total_time_ms: 7 + task_total_time_ms: 240 transform_count: 94 - transform_time_ms: 3 + transform_time_ms: 198 transform_yield_count: 43 - insert_time_ms: 0 + insert_time_ms: 7 insert_new_count: 41 insert_reused_count: 4 - query: EXPLAIN select c from t1 order by b desc explain: 'COVERING(I1 <,> REVERSE -> [A: KEY[2], B: KEY[0], C: VALUE[0]]) | MAP (_.C AS C, _.B AS B) | MAP (_.C AS C)' task_count: 302 - task_total_time_ms: 5 + task_total_time_ms: 14 transform_count: 77 - transform_time_ms: 2 + transform_time_ms: 8 transform_yield_count: 36 insert_time_ms: 0 insert_new_count: 25 @@ -36,21 +36,52 @@ orderby-tests: explain: 'COVERING(I2 <,> REVERSE -> [A: KEY[2], B: VALUE[0], C: KEY[0]]) | MAP (_.B AS B, _.C AS C) | MAP (_.B AS B)' task_count: 407 - task_total_time_ms: 19 + task_total_time_ms: 36 transform_count: 94 - transform_time_ms: 5 + transform_time_ms: 16 transform_yield_count: 43 - insert_time_ms: 0 + insert_time_ms: 1 insert_new_count: 41 insert_reused_count: 4 - query: EXPLAIN select c, b from t5 order by c, b desc; explain: 'COVERING(I8 <,> -> [A: KEY[3], B: from_ordered_bytes(KEY:[1], DESC_NULLS_LAST), C: KEY[0]]) | MAP (_.C AS C, _.B AS B)' task_count: 210 - task_total_time_ms: 14 + task_total_time_ms: 18 transform_count: 53 - transform_time_ms: 6 + transform_time_ms: 9 transform_yield_count: 22 insert_time_ms: 0 insert_new_count: 20 insert_reused_count: 2 +- query: EXPLAIN select q as "nested", q.a as "ordered" from t2 order by "ordered" + explain: ISCAN(I4 <,>) | MAP (_.Q AS nested, _.Q.A AS ordered) + task_count: 194 + task_total_time_ms: 11 + transform_count: 53 + transform_time_ms: 6 + transform_yield_count: 26 + insert_time_ms: 0 + insert_new_count: 17 + insert_reused_count: 2 +- query: EXPLAIN select (T.*) from (select q as "nested", q.a as "ordered" from + t2) as T order by "ordered" + explain: ISCAN(I4 <,>) | MAP ((_.Q AS nested, _.Q.A AS ordered) AS _0) + task_count: 220 + task_total_time_ms: 187 + transform_count: 57 + transform_time_ms: 146 + transform_yield_count: 28 + insert_time_ms: 4 + insert_new_count: 21 + insert_reused_count: 2 +- query: EXPLAIN select (*) from t2_func() order by "ordered" + explain: ISCAN(I4 <,>) | MAP ((_.Q AS nesting, _.Q.A AS ordered) AS _0) + task_count: 220 + task_total_time_ms: 11 + transform_count: 57 + transform_time_ms: 7 + transform_yield_count: 28 + insert_time_ms: 0 + insert_new_count: 21 + insert_reused_count: 2 diff --git a/yaml-tests/src/test/resources/orderby.yamsql b/yaml-tests/src/test/resources/orderby.yamsql index 13c78c2ebe..9bd18c30e3 100644 --- a/yaml-tests/src/test/resources/orderby.yamsql +++ b/yaml-tests/src/test/resources/orderby.yamsql @@ -25,6 +25,7 @@ schema_template: create index i3 as select b, c from t1 order by c, b CREATE TYPE AS STRUCT st_1(a bigint, b bigint) create table t2(p bigint, q st_1, primary key(p)) + create function t2_func() as select q as "nesting", q.a as "ordered" from t2 create index i4 as select q.b, q.a from t2 order by q.a create index i5 as select q.b, q.a from t2 order by q.b, q.a create table t3(a bigint, b bigint, c bigint, d bigint, p bigint, primary key(p)) @@ -262,4 +263,27 @@ test_block: - query: select c, b from t5 order by c, b desc; - explain: "COVERING(I8 <,> -> [A: KEY:[3], B: from_ordered_bytes(KEY:[1], DESC_NULLS_LAST), C: KEY:[0]]) | MAP (_.C AS C, _.B AS B)" - result: [{0, 2}, {0, 1}, {1, 4}, {1, 3}, {5, 10}, {5, 9}, {5, 8}, {8, 7}, {8, 6}, {8, 5}] + - + # Ordering by a ambiguous projected column + - query: select q as "nested", q.a as "ordered" from t2 order by "ordered" + - supported_version: !current_version + - explain: "ISCAN(I4 <,>) | MAP (_.Q AS nested, _.Q.A AS ordered)" + - result: [ {{1, 2}, 1}, {{2, 1}, 2}, {{3, 2}, 3}, {{4, 1}, 4}, {{5, 2}, 5}, {{6, 1}, 6}, {{7, 2}, 7}, {{8, 1}, 8} ] + - + # Ordering by a ambiguous projected column in subquery + - query: select (T.*) from (select q as "nested", q.a as "ordered" from t2) as T order by "ordered" + - supported_version: !current_version + - explain: "ISCAN(I4 <,>) | MAP ((_.Q AS nested, _.Q.A AS ordered) AS _0)" + - result: [ {{{1, 2}, 1}}, {{{2, 1}, 2}}, {{{3, 2}, 3}}, {{{4, 1}, 4}}, {{{5, 2}, 5}}, {{{6, 1}, 6}}, {{{7, 2}, 7}}, {{{8, 1}, 8}} ] + - + # Ordering by a ambiguous projected column in table function + - query: select (*) from t2_func() order by "ordered" + - supported_version: !current_version + - explain: "ISCAN(I4 <,>) | MAP ((_.Q AS nesting, _.Q.A AS ordered) AS _0)" + - result: [ {{{1, 2}, 1}}, {{{2, 1}, 2}}, {{{3, 2}, 3}}, {{{4, 1}, 4}}, {{{5, 2}, 5}}, {{{6, 1}, 6}}, {{{7, 2}, 7}}, {{{8, 1}, 8}} ] + - + # qualification on order by clause is currently not supported entirely + - query: select (q as "nested", q.a as "ordered") as "st" from t2 order by "st"."ordered" + - supported_version: !current_version + - error: "42703" ... diff --git a/yaml-tests/src/test/resources/test.yamsql b/yaml-tests/src/test/resources/test.yamsql deleted file mode 100644 index c8f8e71d02..0000000000 --- a/yaml-tests/src/test/resources/test.yamsql +++ /dev/null @@ -1,53 +0,0 @@ -# -# user-defined-scalar-function-tests.yamsql -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2021-2024 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -options: - supported_version: !current_version ---- -schema_template: - create type as struct st1(y bigint, z bigint) - create table nested(id bigint, q st1, primary key(q.y, id)) - CREATE FUNCTION table_func1(IN z_param bigint) AS SELECT *, q.y as ordering FROM nested WHERE q.z = z_param - WITH OPTIONS ( INTERMINGLE_TABLES=true ) ---- -setup: - steps: - - query: INSERT INTO NESTED - VALUES (1, (1, 100)), - (2, (2, 100)), - (3, (3, 100)), - (4, (4, 100)), - (5, (5, 100)), - (6, (6, 200)), - (7, (7, 200)), - (8, (8, 200)), - (9, (9, 200)), - (10, (10, 200)) ---- -test_block: - name: user-defined-scalar-function-tests - preset: single_repetition_ordered - tests: -# - -# - query: select (*) from nested order by q.y -# - error: [{1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}, {10}] - - - - query: select (*) from table_func1(100) order by ordering - - result: [{{ID: 1, Q: {1, 100}, ORDERING: 1}}, {{ID: 2, Q: {2, 100}, ORDERING: 2}}, {{ID: 3, Q: {3, 100}, ORDERING: 3}}, {{ID: 4, Q: {4, 100}, ORDERING: 4}}, {{ID: 5, Q: {5, 100}, ORDERING: 5}}] -... From d6fd7d297bbdebc8944e799c079cdc765d02c52f Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Wed, 10 Dec 2025 12:17:55 +0000 Subject: [PATCH 11/22] fix more bitmap yaml tests --- .../bitmap-aggregate-index.metrics.binpb | 33 ++--------- .../bitmap-aggregate-index.metrics.yaml | 58 +++++-------------- .../resources/bitmap-aggregate-index.yamsql | 8 ++- 3 files changed, 26 insertions(+), 73 deletions(-) diff --git a/yaml-tests/src/test/resources/bitmap-aggregate-index.metrics.binpb b/yaml-tests/src/test/resources/bitmap-aggregate-index.metrics.binpb index 95c4850e13..44e9b70adb 100644 --- a/yaml-tests/src/test/resources/bitmap-aggregate-index.metrics.binpb +++ b/yaml-tests/src/test/resources/bitmap-aggregate-index.metrics.binpb @@ -1,7 +1,7 @@  bitmap-agg-index-testsEXPLAIN SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, bitmap_bucket_offset(id) as offset FROM T1 GROUP BY bitmap_bucket_offset(id) -N @(40y8%@hAISCAN(BITMAPINDEX1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS BITMAP, _._0 AS OFFSET) +=g )( 08@hAISCAN(BITMAPINDEX1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS BITMAP, _._0 AS OFFSET) digraph G { fontname=courier; rankdir=BT; @@ -11,35 +11,10 @@ digraph G { 3 [ label=<
Index
BITMAPINDEX1
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, STRING AS CATEGORY)" ]; 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; 2 -> 1 [ label=< q6> label="q6" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; -} +}  -bitmap-agg-index-testsEXPLAIN SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, category, bitmap_bucket_offset(id) as offset FROM T1 GROUP BY category, bitmap_bucket_offset(id) -O ̡B(408%@AISCAN(BITMAPINDEX2 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: VALUE:[0]]) | MAP (_._2 AS BITMAP, _._0 AS CATEGORY, _._1 AS OFFSET) digraph G { - fontname=courier; - rankdir=BT; - splines=polyline; - 1 [ label=<
Value Computation
MAP (q6._2 AS BITMAP, q6._0 AS CATEGORY, q6._1 AS OFFSET)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(BYTES AS BITMAP, STRING AS CATEGORY, LONG AS OFFSET)" ]; - 2 [ label=<
Index Scan
scan type: BY_GROUP
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(STRING AS _0, LONG AS _1, BYTES AS _2)" ]; - 3 [ label=<
Index
BITMAPINDEX2
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, STRING AS CATEGORY)" ]; - 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 2 -> 1 [ label=< q6> label="q6" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; -} - -bitmap-agg-index-testsEXPLAIN SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, bitmap_bucket_offset(id) as offset FROM T1 GROUP BY bitmap_bucket_offset(id), bitmap_bucket_offset(id), bitmap_bucket_offset(id) -פ ُ(408%@hAISCAN(BITMAPINDEX1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS BITMAP, _._0 AS OFFSET) -digraph G { - fontname=courier; - rankdir=BT; - splines=polyline; - 1 [ label=<
Value Computation
MAP (q6._1 AS BITMAP, q6._0 AS OFFSET)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(BYTES AS BITMAP, LONG AS OFFSET)" ]; - 2 [ label=<
Index Scan
scan type: BY_GROUP
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS _0, BYTES AS _1)" ]; - 3 [ label=<
Index
BITMAPINDEX1
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, STRING AS CATEGORY)" ]; - 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 2 -> 1 [ label=< q6> label="q6" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; -} - -bitmap-agg-index-testsEXPLAIN SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, category, bitmap_bucket_offset(id) as offset FROM T1 GROUP BY bitmap_bucket_offset(id), category, bitmap_bucket_offset(id) -O B(408%@AISCAN(BITMAPINDEX2 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: VALUE:[0]]) | MAP (_._2 AS BITMAP, _._0 AS CATEGORY, _._1 AS OFFSET) digraph G { +bitmap-agg-index-testsEXPLAIN SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, category, bitmap_bucket_offset(id) as offset FROM T1 GROUP BY category, bitmap_bucket_offset(id) +g ( 0٣L8@AISCAN(BITMAPINDEX2 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: VALUE:[0]]) | MAP (_._2 AS BITMAP, _._0 AS CATEGORY, _._1 AS OFFSET) digraph G { fontname=courier; rankdir=BT; splines=polyline; diff --git a/yaml-tests/src/test/resources/bitmap-aggregate-index.metrics.yaml b/yaml-tests/src/test/resources/bitmap-aggregate-index.metrics.yaml index 6cadf1e00d..f26d326477 100644 --- a/yaml-tests/src/test/resources/bitmap-aggregate-index.metrics.yaml +++ b/yaml-tests/src/test/resources/bitmap-aggregate-index.metrics.yaml @@ -3,52 +3,26 @@ bitmap-agg-index-tests: bitmap_bucket_offset(id) as offset FROM T1 GROUP BY bitmap_bucket_offset(id) explain: 'AISCAN(BITMAPINDEX1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS BITMAP, _._0 AS OFFSET)' - task_count: 483 - task_total_time_ms: 165 - transform_count: 148 - transform_time_ms: 135 - transform_yield_count: 52 - insert_time_ms: 1 - insert_new_count: 37 - insert_reused_count: 2 + task_count: 332 + task_total_time_ms: 128 + transform_count: 103 + transform_time_ms: 87 + transform_yield_count: 32 + insert_time_ms: 3 + insert_new_count: 24 + insert_reused_count: 1 - query: EXPLAIN SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, category, bitmap_bucket_offset(id) as offset FROM T1 GROUP BY category, bitmap_bucket_offset(id) explain: 'AISCAN(BITMAPINDEX2 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: VALUE:[0]]) | MAP (_._2 AS BITMAP, _._0 AS CATEGORY, _._1 AS OFFSET)' - task_count: 483 - task_total_time_ms: 166 - transform_count: 148 - transform_time_ms: 139 - transform_yield_count: 52 - insert_time_ms: 2 - insert_new_count: 37 - insert_reused_count: 2 -- query: EXPLAIN SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, - bitmap_bucket_offset(id) as offset FROM T1 GROUP BY bitmap_bucket_offset(id), - bitmap_bucket_offset(id), bitmap_bucket_offset(id) - explain: 'AISCAN(BITMAPINDEX1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | - MAP (_._1 AS BITMAP, _._0 AS OFFSET)' - task_count: 483 - task_total_time_ms: 57 - transform_count: 148 - transform_time_ms: 39 - transform_yield_count: 52 - insert_time_ms: 2 - insert_new_count: 37 - insert_reused_count: 2 -- query: EXPLAIN SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, - category, bitmap_bucket_offset(id) as offset FROM T1 GROUP BY bitmap_bucket_offset(id), - category, bitmap_bucket_offset(id) - explain: 'AISCAN(BITMAPINDEX2 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: VALUE:[0]]) - | MAP (_._2 AS BITMAP, _._0 AS CATEGORY, _._1 AS OFFSET)' - task_count: 483 - task_total_time_ms: 167 - transform_count: 148 - transform_time_ms: 140 - transform_yield_count: 52 - insert_time_ms: 3 - insert_new_count: 37 - insert_reused_count: 2 + task_count: 332 + task_total_time_ms: 60 + transform_count: 103 + transform_time_ms: 38 + transform_yield_count: 32 + insert_time_ms: 1 + insert_new_count: 24 + insert_reused_count: 1 - query: EXPLAIN SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, bitmap_bucket_offset(id) as offset FROM T2 GROUP BY bitmap_bucket_offset(id) explain: ISCAN(AGG_INDEX_1 <,>) | MAP (_ AS _0) | AGG (bitmap_construct_agg_l((_._0.ID) diff --git a/yaml-tests/src/test/resources/bitmap-aggregate-index.yamsql b/yaml-tests/src/test/resources/bitmap-aggregate-index.yamsql index 562c770498..1840d92f66 100644 --- a/yaml-tests/src/test/resources/bitmap-aggregate-index.yamsql +++ b/yaml-tests/src/test/resources/bitmap-aggregate-index.yamsql @@ -57,14 +57,18 @@ test_block: {BITMAP: xStartsWith_1250'0400008', 'CATEGORY': 'world', 'OFFSET':0}] - - query: SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, bitmap_bucket_offset(id) as offset FROM T1 GROUP BY bitmap_bucket_offset(id), bitmap_bucket_offset(id), bitmap_bucket_offset(id) - - explain: "AISCAN(BITMAPINDEX1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS BITMAP, _._0 AS OFFSET)" + - initialVersionLessThan: !current_version - unorderedResult: [{BITMAP: xStartsWith_1250'060000c', 'OFFSET':0}, {BITMAP: xStartsWith_1250'02', 'OFFSET':10000}] + - initialVersionAtLeast: !current_version + - error: "42702" - - query: SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, category, bitmap_bucket_offset(id) as offset FROM T1 GROUP BY bitmap_bucket_offset(id), category, bitmap_bucket_offset(id) - - explain: "AISCAN(BITMAPINDEX2 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: VALUE:[0]]) | MAP (_._2 AS BITMAP, _._0 AS CATEGORY, _._1 AS OFFSET)" + - initialVersionLessThan: !current_version - unorderedResult: [{BITMAP: xStartsWith_1250'0200004', 'CATEGORY': 'hello', 'OFFSET':0}, {BITMAP: xStartsWith_1250'02', 'CATEGORY': 'hello', 'OFFSET':10000}, {BITMAP: xStartsWith_1250'0400008', 'CATEGORY': 'world', 'OFFSET':0}] + - initialVersionAtLeast: !current_version + - error: "42702" - - query: SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, bitmap_bucket_offset(id) as offset FROM T2 GROUP BY bitmap_bucket_offset(id) - explain: "ISCAN(AGG_INDEX_1 <,>) | MAP (_ AS _0) | AGG (bitmap_construct_agg_l((_._0.ID) bitmap_bit_position 10000) AS _0) GROUP BY ((_._0.ID) bitmap_bucket_offset 10000 AS _0) | MAP (_._1._0 AS BITMAP, _._0._0 AS OFFSET)" From 68fe4bd422bffb10352ccbb4b827986da413960b Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Wed, 10 Dec 2025 15:34:19 +0000 Subject: [PATCH 12/22] add tests --- .../record/query/plan/cascades/Ordering.java | 12 +-- .../query/plan/cascades/OrderingTest.java | 88 +++++++++++++++++++ yaml-tests/src/test/resources/orderby.yamsql | 12 ++- 3 files changed, 105 insertions(+), 7 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java index 23c2bc2a39..7b02b0fcef 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java @@ -35,9 +35,9 @@ import com.apple.foundationdb.record.query.plan.cascades.values.simplification.DefaultValueSimplificationRuleSet; import com.google.common.base.Suppliers; import com.google.common.base.Verify; -import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSetMultimap; import com.google.common.collect.Iterables; @@ -395,7 +395,7 @@ public boolean satisfiesGroupingValues(@Nonnull final Set requestedGroupi } @Nonnull - public Ordering pullUp(@Nonnull final Value value /*to pull against*/, @Nonnull EvaluationContext evaluationContext, + public Ordering pullUp(@Nonnull final Value value, @Nonnull EvaluationContext evaluationContext, @Nonnull final AliasMap aliasMap, @Nonnull final Set constantAliases) { final var pulledUpBindingMapBuilder = ImmutableSetMultimap.builder(); for (final var entry : getBindingMap().asMap().entrySet()) { @@ -440,18 +440,18 @@ public Ordering pushDown(@Nonnull final Value value, @Nonnull final EvaluationCo value.pushDown(toBePushedValues, DefaultValueSimplificationRuleSet.instance(), evaluationContext, aliasMap, constantAliases, Quantifier.current()); - final var resultMap = HashMultimap.create(); + final var resultMapBuilder = ImmutableMultimap.builder(); for (int i = 0; i < toBePushedValues.size(); i++) { final Value toBePushedValue = toBePushedValues.get(i); final Value pushedValue = Objects.requireNonNull(pushedDownValues.get(i)); - resultMap.put(toBePushedValue, pushedValue); + resultMapBuilder.put(toBePushedValue, pushedValue); } - return resultMap; + return resultMapBuilder.build(); }); pushedBindingMapBuilder.putAll(entry.getKey(), pushedBindings); } - // pull up the values we actually could also pull up some of the the bindings for + // push down the values for which we actually could push down some of the bindings final var pushedBindingMap = pushedBindingMapBuilder.build(); final var values = pushedBindingMap.keySet(); final var pushedValues = diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java index 29e2f500e5..956f223135 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java @@ -327,6 +327,94 @@ void testPullUp4() { assertEquals(expectedOrdering, pulledUpOrdering); } + @Test + void testPullUp5() { + final var qov = ValueTestHelpers.qov(); + final var a = ValueTestHelpers.field(qov, "a"); + final var b = ValueTestHelpers.field(qov, "b"); + final var c = ValueTestHelpers.field(qov, "c"); + final var innerOrderedSet = PartiallyOrderedSet.of(ImmutableSet.of(a, b, c), ImmutableSetMultimap.of(b, a)); + final var innerOrdering = + Ordering.ofOrderingSet(bindingMap(a, ProvidedSortOrder.ASCENDING, + b, ProvidedSortOrder.ASCENDING, + c, ProvidedSortOrder.ASCENDING), innerOrderedSet, false); + + final var rcv2 = RecordConstructorValue.ofColumns(ImmutableList.of( + Column.of(Type.Record.Field.of(Type.primitiveType(Type.TypeCode.STRING), Optional.of("a_1")), FieldValue.ofFieldNames(ValueTestHelpers.qov(), ImmutableList.of("a"))), + Column.of(Type.Record.Field.of(Type.primitiveType(Type.TypeCode.STRING), Optional.of("b_1")), FieldValue.ofFieldNames(ValueTestHelpers.qov(), ImmutableList.of("b"))), + Column.of(Type.Record.Field.of(Type.primitiveType(Type.TypeCode.STRING), Optional.of("a_2")), FieldValue.ofFieldNames(ValueTestHelpers.qov(), ImmutableList.of("a"))), + Column.of(Type.Record.Field.of(Type.primitiveType(Type.TypeCode.STRING), Optional.of("b_2")), FieldValue.ofFieldNames(ValueTestHelpers.qov(), ImmutableList.of("b"))) + )); + final var pulledUpOrdering = + innerOrdering.pullUp(rcv2, EvaluationContext.empty(), AliasMap.emptyMap(), Set.of()); + + final var qovCurrent = QuantifiedObjectValue.of(Quantifier.current(), rcv2.getResultType()); + final var ap1 = ValueTestHelpers.field(qovCurrent, "a_1"); + final var bp1 = ValueTestHelpers.field(qovCurrent, "b_1"); + final var ap2 = ValueTestHelpers.field(qovCurrent, "a_2"); + final var bp2 = ValueTestHelpers.field(qovCurrent, "b_2"); + + final var expectedOrdering = + Ordering.ofOrderingSet(bindingMap(ap1, ProvidedSortOrder.ASCENDING, + bp1, ProvidedSortOrder.ASCENDING, + ap2, ProvidedSortOrder.ASCENDING, + bp2, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(ap1, ap2, bp1, bp2), ImmutableSetMultimap.of(bp2, ap1, bp2, ap2, bp1, ap1, bp1, ap2)), + false); + + assertEquals( + expectedOrdering, + pulledUpOrdering); + } + + @Test + void testPushDown1() { + final var rcv = select("a", "b", "c"); + + final var qovCurrent = QuantifiedObjectValue.of(Quantifier.current(), rcv.getResultType()); + final var ap = ValueTestHelpers.field(qovCurrent, "ap"); + final var ordering = + Ordering.ofOrderingSet(bindingMap(ap, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(ap), ImmutableSetMultimap.of()), + false); + + final var pushedDownOrdering = ordering.pushDown(rcv, EvaluationContext.empty(), AliasMap.emptyMap(), Set.of()); + + final var a = ValueTestHelpers.field(ValueTestHelpers.qov(), "a"); + final var expectedOrdering = + Ordering.ofOrderingSet(bindingMap(a, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(a), ImmutableSetMultimap.of()), + false); + + assertEquals(expectedOrdering, pushedDownOrdering); + } + + @Test + void testPushDown2() { + final var rcv = select("a", "b", "c"); + + final var qovCurrent = QuantifiedObjectValue.of(Quantifier.current(), rcv.getResultType()); + final var ap = ValueTestHelpers.field(qovCurrent, "ap"); + final var bp = ValueTestHelpers.field(qovCurrent, "bp"); + final var ordering = + Ordering.ofOrderingSet(bindingMap(ap, ProvidedSortOrder.ASCENDING, + bp, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(ap, bp), ImmutableSetMultimap.of(bp, ap)), + false); + + final var pushedDownOrdering = ordering.pushDown(rcv, EvaluationContext.empty(), AliasMap.emptyMap(), Set.of()); + + final var a = ValueTestHelpers.field(ValueTestHelpers.qov(), "a"); + final var b = ValueTestHelpers.field(ValueTestHelpers.qov(), "b"); + final var expectedOrdering = + Ordering.ofOrderingSet(bindingMap(a, ProvidedSortOrder.ASCENDING, + b, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(a, b), ImmutableSetMultimap.of(b, a)), + false); + + assertEquals(expectedOrdering, pushedDownOrdering); + } + @Test void testMergePartialOrdersNAry() { final var qov = ValueTestHelpers.qov(); diff --git a/yaml-tests/src/test/resources/orderby.yamsql b/yaml-tests/src/test/resources/orderby.yamsql index 9bd18c30e3..11d0773fbb 100644 --- a/yaml-tests/src/test/resources/orderby.yamsql +++ b/yaml-tests/src/test/resources/orderby.yamsql @@ -26,6 +26,7 @@ schema_template: CREATE TYPE AS STRUCT st_1(a bigint, b bigint) create table t2(p bigint, q st_1, primary key(p)) create function t2_func() as select q as "nesting", q.a as "ordered" from t2 + create function t2_func_2() as select q as "nesting", q.a as "ordered", q.b as "ordered" from t2 create index i4 as select q.b, q.a from t2 order by q.a create index i5 as select q.b, q.a from t2 order by q.b, q.a create table t3(a bigint, b bigint, c bigint, d bigint, p bigint, primary key(p)) @@ -269,6 +270,10 @@ test_block: - supported_version: !current_version - explain: "ISCAN(I4 <,>) | MAP (_.Q AS nested, _.Q.A AS ordered)" - result: [ {{1, 2}, 1}, {{2, 1}, 2}, {{3, 2}, 3}, {{4, 1}, 4}, {{5, 2}, 5}, {{6, 1}, 6}, {{7, 2}, 7}, {{8, 1}, 8} ] + - + # Ordering by a ambiguous projected column + - query: select q as "nested", q.a as "ordered", q.b as "ordered" from t2 order by "ordered" + - error: "42702" - # Ordering by a ambiguous projected column in subquery - query: select (T.*) from (select q as "nested", q.a as "ordered" from t2) as T order by "ordered" @@ -276,11 +281,16 @@ test_block: - explain: "ISCAN(I4 <,>) | MAP ((_.Q AS nested, _.Q.A AS ordered) AS _0)" - result: [ {{{1, 2}, 1}}, {{{2, 1}, 2}}, {{{3, 2}, 3}}, {{{4, 1}, 4}}, {{{5, 2}, 5}}, {{{6, 1}, 6}}, {{{7, 2}, 7}}, {{{8, 1}, 8}} ] - - # Ordering by a ambiguous projected column in table function + # Ordering by a column in table function that has been projected in multiple ways - query: select (*) from t2_func() order by "ordered" - supported_version: !current_version - explain: "ISCAN(I4 <,>) | MAP ((_.Q AS nesting, _.Q.A AS ordered) AS _0)" - result: [ {{{1, 2}, 1}}, {{{2, 1}, 2}}, {{{3, 2}, 3}}, {{{4, 1}, 4}}, {{{5, 2}, 5}}, {{{6, 1}, 6}}, {{{7, 2}, 7}}, {{{8, 1}, 8}} ] + - + # Ordering by a column in table function that has been projected in multiple ways + - query: select (*) from t2_func_2() order by "ordered" + # This should probably return AMBIGUOUS_COLUMN, but currently it returns UNKNOWN_COLUMN. + - error: "42703" - # qualification on order by clause is currently not supported entirely - query: select (q as "nested", q.a as "ordered") as "st" from t2 order by "st"."ordered" From 187aac8c1c2fd8fcbbb73efc29a30062bd3a6faa Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Wed, 10 Dec 2025 22:45:53 +0000 Subject: [PATCH 13/22] nit --- .../values/simplification/MatchOrCompensateFieldValueRule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchOrCompensateFieldValueRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchOrCompensateFieldValueRule.java index 6bfd794fd0..b99f7ba1c6 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchOrCompensateFieldValueRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchOrCompensateFieldValueRule.java @@ -75,7 +75,7 @@ public void onMatch(@Nonnull final ValueComputationRuleCall Date: Fri, 5 Dec 2025 16:25:55 +0000 Subject: [PATCH 14/22] oldie From 798a3c698d8b1372b733fdb22aaac96c2b9340f3 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Tue, 16 Dec 2025 13:15:54 +0000 Subject: [PATCH 15/22] ip --- .../record/query/plan/cascades/Ordering.java | 35 ++++++++++++++++--- yaml-tests/src/test/resources/orderby.yamsql | 5 +++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java index 7b02b0fcef..e823ad957f 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java @@ -59,6 +59,7 @@ import java.util.function.BiPredicate; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; /** * This class captures an ordering property. @@ -292,7 +293,29 @@ public Set deriveRequestedOrderings(@Nonnull final RequestedO } public boolean satisfies(@Nonnull RequestedOrdering requestedOrdering) { - return !Iterables.isEmpty(enumerateCompatibleRequestedOrderings(requestedOrdering)); + if (requestedOrdering.isDistinct() && !isDistinct()) { + return false; + } + + final var eligibleElements = orderingSet.eligibleSet().eligibleElements(); + var isFirst = true; + for (final var requestedOrderingPart : requestedOrdering.getOrderingParts()) { + if (!bindingMap.containsKey(requestedOrderingPart.getValue())) { + return false; + } + if (isFirst) { + if (!eligibleElements.contains(requestedOrderingPart.getValue())) { + return false; + } + isFirst = false; + } + final var bindings = bindingMap.get(requestedOrderingPart.getValue()); + final var sortOrder = sortOrder(bindings); + if (!sortOrder.isCompatibleWithRequestedSortOrder(requestedOrderingPart.getSortOrder())) { + return false; + } + } + return satisfiesGroupingValues(requestedOrdering.getOrderingParts().stream().map(OrderingPart::getValue).collect(Collectors.toSet())); } /** @@ -326,7 +349,6 @@ public Iterable> enumerateCompatibleRequestedOrderin return ImmutableList.of(); } - final var requestedOrderingValuesBuilder = ImmutableList.builder(); final var requestedOrderingValuesMapBuilder = ImmutableMap.builder(); for (final var requestedOrderingPart : requestedOrdering.getOrderingParts()) { if (!bindingMap.containsKey(requestedOrderingPart.getValue())) { @@ -337,8 +359,6 @@ public Iterable> enumerateCompatibleRequestedOrderin if (!sortOrder.isCompatibleWithRequestedSortOrder(requestedOrderingPart.getSortOrder())) { return ImmutableList.of(); } - - requestedOrderingValuesBuilder.add(requestedOrderingPart.getValue()); requestedOrderingValuesMapBuilder.put(requestedOrderingPart.getValue(), requestedOrderingPart); } final var requestedOrderingValuesMap = requestedOrderingValuesMapBuilder.build(); @@ -371,6 +391,11 @@ public boolean satisfiesGroupingValues(@Nonnull final Set requestedGroupi return false; } + final var eligibleElements = orderingSet.eligibleSet().eligibleElements(); + if (requestedGroupingValues.stream().noneMatch(eligibleElements::contains)) { + return false; + } + if (requestedGroupingValues .stream() .anyMatch(requestedGroupingValue -> { @@ -383,7 +408,7 @@ public boolean satisfiesGroupingValues(@Nonnull final Set requestedGroupi return false; } - final var permutations = TopologicalSort.topologicalOrderPermutations(orderingSet); + final var permutations = TopologicalSort.topologicalOrderPermutations(requestedGroupingValues, orderingSet.getTransitiveClosure()); for (final var permutation : permutations) { final var containsAll = requestedGroupingValues.containsAll(permutation.subList(0, requestedGroupingValues.size())); diff --git a/yaml-tests/src/test/resources/orderby.yamsql b/yaml-tests/src/test/resources/orderby.yamsql index 11d0773fbb..5fa5ca4949 100644 --- a/yaml-tests/src/test/resources/orderby.yamsql +++ b/yaml-tests/src/test/resources/orderby.yamsql @@ -27,6 +27,7 @@ schema_template: create table t2(p bigint, q st_1, primary key(p)) create function t2_func() as select q as "nesting", q.a as "ordered" from t2 create function t2_func_2() as select q as "nesting", q.a as "ordered", q.b as "ordered" from t2 + create function t2_func_3() as select q as "nesting", q.a as "ordered1", q.b as "ordered2" from t2 create index i4 as select q.b, q.a from t2 order by q.a create index i5 as select q.b, q.a from t2 order by q.b, q.a create table t3(a bigint, b bigint, c bigint, d bigint, p bigint, primary key(p)) @@ -296,4 +297,8 @@ test_block: - query: select (q as "nested", q.a as "ordered") as "st" from t2 order by "st"."ordered" - supported_version: !current_version - error: "42703" + - + # Ordering by a column in table function that has been projected in multiple ways + - query: select (*) from t2_func_3() order by "ordered2", "ordered1" + - result: [ {{{2, 1}, 2, 1}}, {{{4, 1}, 4, 1}}, {{{6, 1}, 6, 1}}, {{{8, 1}, 8, 1}}, {{{1, 2}, 1, 2}}, {{{3, 2}, 3, 2}}, {{{5, 2}, 5, 2}}, {{{7, 2}, 7, 2}} ] ... From f978fd5e10845ed689d086bd19c826aa10325f2d Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Tue, 16 Dec 2025 18:44:53 +0000 Subject: [PATCH 16/22] fix poset mapAll --- .../combinatorics/PartiallyOrderedSet.java | 73 +++++++++++-------- .../record/query/plan/cascades/Ordering.java | 34 +-------- .../query/plan/cascades/OrderingTest.java | 6 +- yaml-tests/src/test/resources/orderby.yamsql | 1 + 4 files changed, 51 insertions(+), 63 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java index b57838926b..582e60b0b0 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java @@ -35,6 +35,7 @@ import javax.annotation.Nonnull; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -168,25 +169,43 @@ public PartiallyOrderedSet mapAll(@Nonnull final Function PartiallyOrderedSet mapAll(@Nonnull final Map map) { - final var mappedElements = Sets.newLinkedHashSet(map.values()); - - final var resultDependencyMapBuilder = ImmutableSetMultimap.builder(); - for (final var entry : getTransitiveClosure().entries()) { - final var key = entry.getKey(); - final var value = entry.getValue(); - - if (map.containsKey(key) && map.containsKey(value)) { - resultDependencyMapBuilder.put(map.get(key), map.get(value)); - } else { - if (!map.containsKey(value)) { - // if key depends on value that does not exist -- do not insert the dependency and also remove key - final var mappedKey = map.get(key); - if (mappedKey != null) { - mappedElements.remove(mappedKey); + final var allRemovedBuilder = ImmutableSet.builder(); + final var leftToRemove = new HashSet<>(Sets.difference(set, map.keySet())); + + final var degreeMap = dependencyMap.entries().stream() + .collect(Collectors.groupingBy( + Map.Entry::getKey, + HashMap::new, + Collectors.summingInt(e -> 1))); + + while (!leftToRemove.isEmpty()) { + final T toRemove = leftToRemove.iterator().next(); + for (final var entry: dependencyMap.entries()) { + final var key = entry.getKey(); + if (entry.getValue().equals(toRemove) && degreeMap.containsKey(key)) { + final var updatedDegree = degreeMap.get(key) - 1; + if (updatedDegree == 0) { + leftToRemove.add(key); + degreeMap.remove(key); + } else { + degreeMap.put(key, updatedDegree); } } } + leftToRemove.remove(toRemove); + allRemovedBuilder.add(toRemove); } + final var allRemovedElements = allRemovedBuilder.build(); + + final var mappedElements = set.stream() + .filter(value -> !allRemovedElements.contains(value)) + .map(map::get) + .collect(Collectors.toSet()); + final var resultDependencyMapBuilder = ImmutableSetMultimap.builder(); + dependencyMap.entries().stream() + .filter(entry -> !allRemovedElements.contains(entry.getKey()) && !allRemovedElements.contains(entry.getValue())) + .map(entry -> Map.entry(map.get(entry.getKey()), map.get(entry.getValue()))) + .forEach(resultDependencyMapBuilder::put); // this needs the dependency map to be cleansed (which is done in the constructor) return PartiallyOrderedSet.of(mappedElements, resultDependencyMapBuilder.build()); @@ -194,25 +213,21 @@ public PartiallyOrderedSet mapAll(@Nonnull final Map map) { @Nonnull public PartiallyOrderedSet mapAll(@Nonnull final Multimap map) { - final var mappedElements = Sets.newLinkedHashSet(map.values()); + final var identityMapped = this.filterElements(map::containsKey); - final var resultDependencyMapBuilder = ImmutableSetMultimap.builder(); - for (final var entry : getTransitiveClosure().entries()) { - final var key = entry.getKey(); - final var value = entry.getValue(); + final var resultSet = identityMapped.getSet().stream() + .flatMap(value -> map.get(value).stream()) + .collect(Collectors.toSet()); - if (map.containsKey(key) && map.containsKey(value)) { - map.get(key).forEach(mappedKey -> resultDependencyMapBuilder.putAll(mappedKey, map.get(value))); - } else { - if (!map.containsKey(value)) { - // if key depends on value that does not exist -- do not insert the dependency and also remove key - mappedElements.removeAll(map.get(key)); - } - } + final var resultDependencyMapBuilder = ImmutableSetMultimap.builder(); + for (final var entry: identityMapped.dependencyMap.entries()) { + Verify.verify(map.containsKey(entry.getKey()) && map.containsKey(entry.getValue())); + map.get(entry.getKey()).forEach(mappedKey -> resultDependencyMapBuilder.putAll(mappedKey, map.get(entry.getValue()))); + resultSet.addAll(map.get(entry.getKey())); } // this needs the dependency map to be cleansed (which is done in the constructor) - return PartiallyOrderedSet.of(mappedElements, resultDependencyMapBuilder.build()); + return PartiallyOrderedSet.of(resultSet, resultDependencyMapBuilder.build()); } /** diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java index e823ad957f..3a787d2807 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java @@ -59,7 +59,6 @@ import java.util.function.BiPredicate; import java.util.function.Function; import java.util.function.Supplier; -import java.util.stream.Collectors; /** * This class captures an ordering property. @@ -293,29 +292,7 @@ public Set deriveRequestedOrderings(@Nonnull final RequestedO } public boolean satisfies(@Nonnull RequestedOrdering requestedOrdering) { - if (requestedOrdering.isDistinct() && !isDistinct()) { - return false; - } - - final var eligibleElements = orderingSet.eligibleSet().eligibleElements(); - var isFirst = true; - for (final var requestedOrderingPart : requestedOrdering.getOrderingParts()) { - if (!bindingMap.containsKey(requestedOrderingPart.getValue())) { - return false; - } - if (isFirst) { - if (!eligibleElements.contains(requestedOrderingPart.getValue())) { - return false; - } - isFirst = false; - } - final var bindings = bindingMap.get(requestedOrderingPart.getValue()); - final var sortOrder = sortOrder(bindings); - if (!sortOrder.isCompatibleWithRequestedSortOrder(requestedOrderingPart.getSortOrder())) { - return false; - } - } - return satisfiesGroupingValues(requestedOrdering.getOrderingParts().stream().map(OrderingPart::getValue).collect(Collectors.toSet())); + return !Iterables.isEmpty(enumerateCompatibleRequestedOrderings(requestedOrdering)); } /** @@ -365,7 +342,7 @@ public Iterable> enumerateCompatibleRequestedOrderin final var satisfyingValuePermutations = TopologicalSort.satisfyingPermutations( - getOrderingSet(), + getOrderingSet().filterElements(t -> requestedOrdering.getOrderingParts().stream().anyMatch(part -> part.getValue().equals(t))), ImmutableList.copyOf(requestedOrderingValuesMap.keySet()), Function.identity(), permutation -> requestedOrdering.getOrderingParts().size()); @@ -391,11 +368,6 @@ public boolean satisfiesGroupingValues(@Nonnull final Set requestedGroupi return false; } - final var eligibleElements = orderingSet.eligibleSet().eligibleElements(); - if (requestedGroupingValues.stream().noneMatch(eligibleElements::contains)) { - return false; - } - if (requestedGroupingValues .stream() .anyMatch(requestedGroupingValue -> { @@ -408,7 +380,7 @@ public boolean satisfiesGroupingValues(@Nonnull final Set requestedGroupi return false; } - final var permutations = TopologicalSort.topologicalOrderPermutations(requestedGroupingValues, orderingSet.getTransitiveClosure()); + final var permutations = TopologicalSort.topologicalOrderPermutations(getOrderingSet().filterElements(requestedGroupingValues::contains)); for (final var permutation : permutations) { final var containsAll = requestedGroupingValues.containsAll(permutation.subList(0, requestedGroupingValues.size())); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java index 956f223135..f50449d458 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java @@ -49,7 +49,7 @@ import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; class OrderingTest { @Test @@ -519,7 +519,7 @@ void testCommonOrdering() { final var satisfyingOrderingsIterable = mergedOrdering.enumerateCompatibleRequestedOrderings(requestedOrdering); final var onlySatisfyingOrdering = Iterables.getOnlyElement(satisfyingOrderingsIterable); - assertEquals(requested(a, b, c, d, RequestedSortOrder.ANY), onlySatisfyingOrdering); + assertEquals(requested(a, b, c), onlySatisfyingOrdering); } @Test @@ -628,7 +628,7 @@ void testCommonOrdering4() { RequestedOrdering.ofPrimitiveParts(requested(a, b, x), RequestedOrdering.Distinctness.PRESERVE_DISTINCTNESS, false); - assertFalse(mergedOrdering.satisfies(requestedOrdering)); + assertTrue(mergedOrdering.satisfies(requestedOrdering)); } @Nonnull diff --git a/yaml-tests/src/test/resources/orderby.yamsql b/yaml-tests/src/test/resources/orderby.yamsql index 5fa5ca4949..2f6a5a6500 100644 --- a/yaml-tests/src/test/resources/orderby.yamsql +++ b/yaml-tests/src/test/resources/orderby.yamsql @@ -300,5 +300,6 @@ test_block: - # Ordering by a column in table function that has been projected in multiple ways - query: select (*) from t2_func_3() order by "ordered2", "ordered1" + - supported_version: !current_version - result: [ {{{2, 1}, 2, 1}}, {{{4, 1}, 4, 1}}, {{{6, 1}, 6, 1}}, {{{8, 1}, 8, 1}}, {{{1, 2}, 1, 2}}, {{{3, 2}, 3, 2}}, {{{5, 2}, 5, 2}}, {{{7, 2}, 7, 2}} ] ... From 9588cd3dc56bd844cc69cc4a7409bb120ce5962f Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Tue, 16 Dec 2025 22:34:34 +0000 Subject: [PATCH 17/22] wip --- .../combinatorics/PartiallyOrderedSet.java | 26 ++++ .../record/query/plan/cascades/Ordering.java | 2 +- .../foundationdb/query/FDBInQueryTest.java | 4 +- .../query/FDBOrQueryToUnionTest.java | 42 +++--- .../query/plan/cascades/OrderingTest.java | 141 +++++++++++++++++- 5 files changed, 188 insertions(+), 27 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java index 582e60b0b0..60f765f70e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java @@ -211,6 +211,32 @@ public PartiallyOrderedSet mapAll(@Nonnull final Map map) { return PartiallyOrderedSet.of(mappedElements, resultDependencyMapBuilder.build()); } +// @Nonnull +// public PartiallyOrderedSet mapAll(@Nonnull final Map map) { +// final var mappedElements = Sets.newLinkedHashSet(map.values()); +// +// final var resultDependencyMapBuilder = ImmutableSetMultimap.builder(); +// for (final var entry : getTransitiveClosure().entries()) { +// final var key = entry.getKey(); +// final var value = entry.getValue(); +// +// if (map.containsKey(key) && map.containsKey(value)) { +// resultDependencyMapBuilder.put(map.get(key), map.get(value)); +// } else { +// if (!map.containsKey(value)) { +// // if key depends on value that does not exist -- do not insert the dependency and also remove key +// final var mappedKey = map.get(key); +// if (mappedKey != null) { +// mappedElements.remove(mappedKey); +// } +// } +// } +// } +// +// // this needs the dependency map to be cleansed (which is done in the constructor) +// return PartiallyOrderedSet.of(mappedElements, resultDependencyMapBuilder.build()); +// } + @Nonnull public PartiallyOrderedSet mapAll(@Nonnull final Multimap map) { final var identityMapped = this.filterElements(map::containsKey); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java index 3a787d2807..d5af2c44ed 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java @@ -382,7 +382,7 @@ public boolean satisfiesGroupingValues(@Nonnull final Set requestedGroupi final var permutations = TopologicalSort.topologicalOrderPermutations(getOrderingSet().filterElements(requestedGroupingValues::contains)); for (final var permutation : permutations) { - final var containsAll = + final var containsAll = permutation.size() >= requestedGroupingValues.size() && requestedGroupingValues.containsAll(permutation.subList(0, requestedGroupingValues.size())); if (containsAll) { return true; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBInQueryTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBInQueryTest.java index b43482cdf1..102328d3af 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBInQueryTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBInQueryTest.java @@ -2293,7 +2293,7 @@ void testInQueryOrDifferentCondition() throws Exception { inUnionOnValuesPlan( predicatesFilterPlan(indexPlan().where(indexName("MySimpleRecord$num_value_unique")).and(scanComparisons(range("([990],>")))) .where(predicates(valuePredicate(fieldValueWithFieldNames("num_value_2"), anyValueComparison())))) - .where(comparisonKeyValues(exactly(fieldValueWithFieldNames("num_value_unique"), fieldValueWithFieldNames("rec_no"), fieldValueWithFieldNames("num_value_2")))))); + .where(comparisonKeyValues(exactly(fieldValueWithFieldNames("num_value_unique"), fieldValueWithFieldNames("rec_no")))))); assertEquals(1521186153, plan.planHash(PlanHashable.CURRENT_LEGACY)); assertEquals(-455093906, plan.planHash(PlanHashable.CURRENT_FOR_CONTINUATION)); } @@ -2804,7 +2804,7 @@ void testInUnionWithIntersectionOnTwoPredicates(int replans, boolean dropNumValu ) ).where(inUnionValuesSources(only(inUnionInParameter(equalsObject("str_list"))))) ).where(inUnionValuesSources(only(inUnionInParameter(equalsObject("nv2_list")))))); - assertEquals(-1515083625, plan.planHash(PlanHashable.CURRENT_LEGACY)); + assertEquals(-1515112545, plan.planHash(PlanHashable.CURRENT_LEGACY)); assertEquals(-1918411257, plan.planHash(PlanHashable.CURRENT_FOR_CONTINUATION)); singleIndexScan = false; } else if (replans < 0 || dropNumValue3Index) { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java index 000321fe1f..368264f92e 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java @@ -1197,7 +1197,7 @@ void testOrderedOrQueryWithMultipleValuesForEarlierColumn(OrQueryParams orQueryP // Index(multi_index [[odd, 0],[odd, 0]]) ∪[Field { 'num_value_3_indexed' None}, Field { 'rec_no' None}] Index(multi_index [[odd, 2],[odd, 2]]) orQueryParams.setPlannerConfiguration(this); RecordQueryPlan plan = planQuery(query); - final KeyExpression comparisonKey = planner instanceof CascadesPlanner ? concat(field("num_value_3_indexed"), primaryKey("MySimpleRecord"), field("num_value_2")) : concat(field("num_value_3_indexed"), primaryKey("MySimpleRecord")); + final KeyExpression comparisonKey = planner instanceof CascadesPlanner ? concat(field("num_value_3_indexed"), field("num_value_2"), primaryKey("MySimpleRecord")) : concat(field("num_value_3_indexed"), primaryKey("MySimpleRecord")); final BindingMatcher planMatcher = queryPlanMatcher(orQueryParams, List.of( indexPlan().where(indexName("multi_index")).and(scanComparisons(range("[[odd, 0],[odd, 0]]"))), indexPlan().where(indexName("multi_index")).and(scanComparisons(range("[[odd, 2],[odd, 2]]"))) @@ -1213,8 +1213,8 @@ void testOrderedOrQueryWithMultipleValuesForEarlierColumn(OrQueryParams orQueryP assertEquals(-521036388, plan.planHash(CURRENT_FOR_CONTINUATION)); } } else { - assertEquals(-2004621044, plan.planHash(CURRENT_LEGACY)); - assertEquals(661700575, plan.planHash(CURRENT_FOR_CONTINUATION)); + assertEquals(-2004620414, plan.planHash(CURRENT_LEGACY)); + assertEquals(661700665, plan.planHash(CURRENT_FOR_CONTINUATION)); } try (FDBRecordContext context = openContext()) { @@ -1288,7 +1288,7 @@ void testOrderedOrQueryWithIntermediateUnorderedColumn(OrQueryParams orQueryPara final BindingMatcher planMatcher = queryPlanMatcher(orQueryParams, List.of( indexPlan().where(indexName("multi_index")).and(scanComparisons(range("[[odd],[odd]]"))), indexPlan().where(indexName("multi_index")).and(scanComparisons(range("[[even],[even]]"))) - ), planner instanceof CascadesPlanner ? concat(field("num_value_2"), field("num_value_3_indexed"), primaryKey("MySimpleRecord"), field("str_value_indexed")) : concat(field("num_value_2"), field("num_value_3_indexed"), primaryKey("MySimpleRecord"))); + ), planner instanceof CascadesPlanner ? concat(field("num_value_2"), field("num_value_3_indexed"), field("str_value_indexed"), primaryKey("MySimpleRecord")) : concat(field("num_value_2"), field("num_value_3_indexed"), primaryKey("MySimpleRecord"))); assertMatchesExactly(plan, planMatcher); if (planner instanceof RecordQueryPlanner) { @@ -1300,8 +1300,8 @@ void testOrderedOrQueryWithIntermediateUnorderedColumn(OrQueryParams orQueryPara assertEquals(1294497974, plan.planHash(CURRENT_FOR_CONTINUATION)); } } else { - assertEquals(471565558, plan.planHash(CURRENT_LEGACY)); - assertEquals(1167320211, plan.planHash(CURRENT_FOR_CONTINUATION)); + assertEquals(471565768, plan.planHash(CURRENT_LEGACY)); + assertEquals(1167320241, plan.planHash(CURRENT_FOR_CONTINUATION)); } try (FDBRecordContext context = openContext()) { @@ -1365,12 +1365,12 @@ void testNestedPredicates(OrQueryParams orQueryParams) throws Exception { indexPlan().where(indexName(index.getName())).and(scanComparisons(range("[EQUALS 0, EQUALS $town1]"))), indexPlan().where(indexName(index.getName())).and(scanComparisons(range("[EQUALS 0, EQUALS $town2]"))) ), - isUseCascadesPlanner() ? concat(field("stats").nest("start_date"), field("name"), field("id"), field("stats").nest("hometown")) : concat(field("stats").nest("start_date"), field("name"), primaryKey("RestaurantReviewer"))); + isUseCascadesPlanner() ? concat(field("stats").nest("start_date"), field("name"), field("stats").nest("hometown"), field("id")) : concat(field("stats").nest("start_date"), field("name"), primaryKey("RestaurantReviewer"))); assertMatchesExactly(plan, planMatcher); assertEquals(orQueryParams.isSortReverse(), plan.isReverse()); if (isUseCascadesPlanner()) { - assertEquals(orQueryParams.isSortReverse() ? 1539206407 : 1539206374, plan.planHash(CURRENT_LEGACY)); - assertEquals(orQueryParams.isSortReverse() ? 1716842733 : 1722562791, plan.planHash(CURRENT_FOR_CONTINUATION)); + assertEquals(orQueryParams.isSortReverse() ? 1539265987 : 1539265954, plan.planHash(CURRENT_LEGACY)); + assertEquals(orQueryParams.isSortReverse() ? 1716902313 : 1722622371, plan.planHash(CURRENT_FOR_CONTINUATION)); } else { assertEquals(orQueryParams.isSortReverse() ? 1766220 : 1766187, plan.planHash(CURRENT_LEGACY)); if (orQueryParams.shouldDeferFetch()) { @@ -1413,7 +1413,7 @@ void testNestedPredicatesWithExtraUnorderedColumns(OrQueryParams orQueryParams) indexPlan().where(indexName(index.getName())).and(scanComparisons(range("[EQUALS 0, EQUALS $town1]"))), indexPlan().where(indexName(index.getName())).and(scanComparisons(range("[EQUALS 0, EQUALS $town2]"))) ), - isUseCascadesPlanner() ? concat(field("stats").nest("start_date"), field("name"), field("id"), field("stats.hometown")) : concat(field("stats").nest("start_date"), field("name"), primaryKey("RestaurantReviewer"))); + isUseCascadesPlanner() ? concat(field("stats").nest("start_date"), field("stats.hometown"), field("name"), field("id")) : concat(field("stats").nest("start_date"), field("name"), primaryKey("RestaurantReviewer"))); } else if (orQueryParams.shouldNormalizeNestedFields()) { planMatcher = filterPlan( indexPlan() @@ -1440,8 +1440,8 @@ void testNestedPredicatesWithExtraUnorderedColumns(OrQueryParams orQueryParams) assertMatchesExactly(plan, planMatcher); assertEquals(orQueryParams.isSortReverse(), plan.isReverse()); if (isUseCascadesPlanner()) { - assertEquals(orQueryParams.isSortReverse() ? 1539206407 : 1539206374, plan.planHash(CURRENT_LEGACY)); - assertEquals(orQueryParams.isSortReverse() ? 1716842733 : 1722562791, plan.planHash(CURRENT_FOR_CONTINUATION)); + assertEquals(orQueryParams.isSortReverse() ? 1541112037 : 1541112004, plan.planHash(CURRENT_LEGACY)); + assertEquals(orQueryParams.isSortReverse() ? 1718748363 : 1724468421, plan.planHash(CURRENT_FOR_CONTINUATION)); } else { if (orQueryParams.shouldNormalizeNestedFields()) { if (orQueryParams.shouldOmitPrimaryKeyInOrderingKey()) { @@ -1577,12 +1577,12 @@ void testOrderedUnionHasRepeatedColumnsInPrefixAndOrdering(OrQueryParams orQuery final BindingMatcher planMatcher = unionPlanMatcher(orQueryParams, List.of( indexPlan().where(indexName(index.getName())).and(scanComparisons(range("[EQUALS $" + value2Param + ", EQUALS $" + strValue1Param + "]"))), indexPlan().where(indexName(index.getName())).and(scanComparisons(range("[EQUALS $" + value2Param + ", EQUALS $" + strValue2Param + "]"))) - ), isUseCascadesPlanner() ? concatenateFields("num_value_3_indexed", "rec_no", "str_value_indexed") : concatenateFields("num_value_3_indexed", "rec_no")); + ), isUseCascadesPlanner() ? concatenateFields("num_value_3_indexed", "str_value_indexed", "rec_no") : concatenateFields("num_value_3_indexed", "rec_no")); assertMatchesExactly(plan, planMatcher); assertEquals(orQueryParams.isSortReverse(), plan.isReverse()); if (isUseCascadesPlanner()) { - assertEquals(orQueryParams.isSortReverse() ? 973132071L : 973132038L, plan.planHash(CURRENT_LEGACY)); - assertEquals(orQueryParams.isSortReverse() ? 1977118957L : 1982839015L, plan.planHash(CURRENT_FOR_CONTINUATION)); + assertEquals(orQueryParams.isSortReverse() ? 973132101L : 973132068L, plan.planHash(CURRENT_LEGACY)); + assertEquals(orQueryParams.isSortReverse() ? 1977118987L : 1982839045L, plan.planHash(CURRENT_FOR_CONTINUATION)); } else { assertEquals(orQueryParams.isSortReverse() ? -2058994186L : -2058994219L, plan.planHash(CURRENT_LEGACY)); if (orQueryParams.shouldDeferFetch()) { @@ -1666,12 +1666,12 @@ void testOrderedUnionHasRepeatedColumnsNotInPrefix(OrQueryParams orQueryParams) final BindingMatcher planMatcher = unionPlanMatcher(orQueryParams, List.of( indexPlan().where(indexName(index.getName())).and(scanComparisons(range("[EQUALS $" + strValue1Param + "]"))), indexPlan().where(indexName(index.getName())).and(scanComparisons(range("[EQUALS $" + strValue2Param + "]"))) - ), isUseCascadesPlanner() ? concatenateFields("num_value_2", "num_value_3_indexed", "rec_no", "str_value_indexed") : concatenateFields("num_value_2", "num_value_3_indexed", "rec_no")); + ), isUseCascadesPlanner() ? concatenateFields("num_value_2", "num_value_3_indexed", "str_value_indexed", "rec_no") : concatenateFields("num_value_2", "num_value_3_indexed", "rec_no")); assertMatchesExactly(plan, planMatcher); assertEquals(orQueryParams.isSortReverse(), plan.isReverse()); if (isUseCascadesPlanner()) { - assertEquals(orQueryParams.isSortReverse() ? 1380805606 : 1380805573, plan.planHash(CURRENT_LEGACY)); - assertEquals(orQueryParams.isSortReverse() ? -1854527892 : -1848807834, plan.planHash(CURRENT_FOR_CONTINUATION)); + assertEquals(orQueryParams.isSortReverse() ? 1380805636 : 1380805603, plan.planHash(CURRENT_LEGACY)); + assertEquals(orQueryParams.isSortReverse() ? -1854527862 : -1848807804, plan.planHash(CURRENT_FOR_CONTINUATION)); } else { assertEquals(orQueryParams.isSortReverse() ? 336481815 : 336481782, plan.planHash(CURRENT_LEGACY)); if (orQueryParams.shouldDeferFetch()) { @@ -1759,7 +1759,7 @@ void orderByIncludingValuePortion(@Nonnull OrQueryParams orQueryParams, @Nonnull if (sortKeyContainsPrimaryKey) { expectedComparisonKey = concatenateFields("num_value_2", "rec_no", "num_value_3_indexed"); } else { - expectedComparisonKey = concatenateFields("num_value_2", "rec_no", "num_value_3_indexed"); + expectedComparisonKey = concatenateFields("num_value_2", "num_value_3_indexed", "rec_no"); } } else { expectedComparisonKey = concatenateFields("num_value_2", "rec_no"); @@ -1778,8 +1778,8 @@ void orderByIncludingValuePortion(@Nonnull OrQueryParams orQueryParams, @Nonnull assertEquals(orQueryParams.isSortReverse() ? 951372169 : 951372136, legacyHash); assertEquals(orQueryParams.isSortReverse() ? 477401359 : 483121417, continuationHash); } else { - assertEquals(orQueryParams.isSortReverse() ? 951372169 : 951372136, legacyHash); - assertEquals(orQueryParams.isSortReverse() ? 477401359 : 483121417, continuationHash); + assertEquals(orQueryParams.isSortReverse() ? 951372289 : 951372256, legacyHash); + assertEquals(orQueryParams.isSortReverse() ? 477401479 : 483121537, continuationHash); } } else { if (orQueryParams.shouldDeferFetch()) { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java index f50449d458..ad515f1c83 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java @@ -47,8 +47,10 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; class OrderingTest { @@ -362,9 +364,142 @@ void testPullUp5() { PartiallyOrderedSet.of(ImmutableSet.of(ap1, ap2, bp1, bp2), ImmutableSetMultimap.of(bp2, ap1, bp2, ap2, bp1, ap1, bp1, ap2)), false); - assertEquals( - expectedOrdering, - pulledUpOrdering); + assertEquals(expectedOrdering, pulledUpOrdering); + } + + @Test + void testSatisfies() { + final var qov = ValueTestHelpers.qov(); + final var a = ValueTestHelpers.field(qov, "a"); + final var b = ValueTestHelpers.field(qov, "b"); + final var c = ValueTestHelpers.field(qov, "c"); + final var d = ValueTestHelpers.field(qov, "d"); + final var e = ValueTestHelpers.field(qov, "e"); + final var x = ValueTestHelpers.field(qov, "x"); + + // a < c, b < c, c < d, d < e, d < x + final var ordering = Ordering.ofOrderingSet(bindingMap(a, ProvidedSortOrder.ASCENDING, + b, ProvidedSortOrder.ASCENDING, + c, ProvidedSortOrder.ASCENDING, + d, ProvidedSortOrder.ASCENDING, + e, ProvidedSortOrder.ASCENDING, + x, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(a, b, c, d, e, x), ImmutableSetMultimap.of(c, a, c, b, d, c, e, d, x, d)), + false); + + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(b))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, c))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, b))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(b, c))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, b, c))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, c, d))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, c, d, e))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(b, c, d, x))); + + assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(c))); + assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(d))); + assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(e))); + assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(x))); + assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(a, d, x))); + assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(c, d, x))); + assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(e, d, x))); + } + + @Test + void testSatisfies2() { + final var qov = ValueTestHelpers.qov(); + final var a = ValueTestHelpers.field(qov, "a"); + final var b = ValueTestHelpers.field(qov, "b"); + final var c = ValueTestHelpers.field(qov, "c"); + final var d = ValueTestHelpers.field(qov, "d"); + final var e = ValueTestHelpers.field(qov, "e"); + final var x = ValueTestHelpers.field(qov, "x"); + + // a < c, b < c, c < d, e < d, x < d + final var ordering = Ordering.ofOrderingSet(bindingMap(a, ProvidedSortOrder.ASCENDING, + b, ProvidedSortOrder.ASCENDING, + c, ProvidedSortOrder.ASCENDING, + d, ProvidedSortOrder.ASCENDING, + e, ProvidedSortOrder.ASCENDING, + x, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(a, b, c, d, e, x), ImmutableSetMultimap.of(c, a, c, b, d, c, d, e, d, x)), + false); + + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(b))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(e))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(x))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, c))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, b))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(b, c))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, b, c))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, c, d))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, d, x))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(e, d, x))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, c, d, e))); + assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(b, c, d, x))); + + assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(c))); + assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(c, d))); + assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(d))); + assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(c, d, x))); + } + + @Test + void testEnumeration() { + final var qov = ValueTestHelpers.qov(); + final var a = ValueTestHelpers.field(qov, "a"); + final var b = ValueTestHelpers.field(qov, "b"); + final var c = ValueTestHelpers.field(qov, "c"); + final var d = ValueTestHelpers.field(qov, "d"); + final var e = ValueTestHelpers.field(qov, "e"); + final var x = ValueTestHelpers.field(qov, "x"); + + // a < c, b < c, c < d, e < d, x < d + final var ordering = Ordering.ofOrderingSet(bindingMap(a, ProvidedSortOrder.ASCENDING, + b, ProvidedSortOrder.ASCENDING, + c, ProvidedSortOrder.ASCENDING, + d, ProvidedSortOrder.ASCENDING, + e, ProvidedSortOrder.ASCENDING, + x, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(a, b, c, d, e, x), ImmutableSetMultimap.of(c, a, c, b, d, c, d, e, d, x)), + false); + + final var expectedSet1 = Stream.of( + List.of(a, c, e, d), + List.of(a, e, c, d), + List.of(a, e, d, c), + List.of(e, a, c, d), + List.of(e, a, d, c), + List.of(e, d, a, c)) + .map(permutation -> requested(permutation.toArray())) + .collect(Collectors.toSet()); + var actual1 = ordering.enumerateCompatibleRequestedOrderings(RequestedOrdering.ofParts(requested(a, c, e, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of())); + actual1.forEach(actualPermutation -> assertTrue(expectedSet1.contains(actualPermutation))); + + final var expectedSet2 = Stream.of( + List.of(b, x, d), + List.of(x, b, d), + List.of(x, d, b)) + .map(permutation -> requested(permutation.toArray())) + .collect(Collectors.toSet()); + final var actual2 = ordering.enumerateCompatibleRequestedOrderings(RequestedOrdering.ofParts(requested(b, x, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of())); + actual2.forEach(actualPermutation -> assertTrue(expectedSet2.contains(actualPermutation))); + + final var expectedSet3 = Stream.of( + List.of()) + .map(permutation -> requested(permutation.toArray())) + .collect(Collectors.toSet()); + final var actual3 = ordering.enumerateCompatibleRequestedOrderings(RequestedOrdering.ofParts(requested(c, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of())); + actual3.forEach(actualPermutation -> assertTrue(expectedSet3.contains(actualPermutation))); + + final var expectedSet4 = Stream.of( + List.of(a, c, d)) + .map(permutation -> requested(permutation.toArray())) + .collect(Collectors.toSet()); + final var actual4 = ordering.enumerateCompatibleRequestedOrderings(RequestedOrdering.ofParts(requested(a, c, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of())); + actual4.forEach(actualPermutation -> assertTrue(expectedSet4.contains(actualPermutation))); } @Test From 364cf848b59e3ef339fbf1187b302801a7d5c158 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Wed, 17 Dec 2025 14:49:03 +0000 Subject: [PATCH 18/22] fix mapAll in poset and satifies --- .../record/query/plan/cascades/Ordering.java | 95 ++++++++++++++++--- .../foundationdb/query/FDBInQueryTest.java | 4 +- .../query/plan/cascades/OrderingTest.java | 5 + 3 files changed, 91 insertions(+), 13 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java index d5af2c44ed..768f416597 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java @@ -59,6 +59,7 @@ import java.util.function.BiPredicate; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; /** * This class captures an ordering property. @@ -291,15 +292,67 @@ public Set deriveRequestedOrderings(@Nonnull final RequestedO .collect(ImmutableSet.toImmutableSet()); } + /** + * Method to verify that a given {@link RequestedOrdering} can be used to construct an ordering sequence that is + * supported by the current ordering from the list of {@link Value}s that composes the {@link RequestedOrdering}. + * This method is similar to {@link Ordering#satisfiesGroupingValues(Set)} in the sense that then both check for + * existence of a sequence given a bunch of {@link Value}. They differ on the satisfiability criteria such that + * current method requires the candidate sequence's prefix to match the ordering of {@link Value} in the + * {@link RequestedOrdering}. + * Example 1: + * 
+     *     {@code
+     *     requested ordering:  a↑, e↑, c↑
+     *     this ordering:
+     *         partially ordered set:
+     *             members: a, b, x, c, d, e
+     *             dependencies: a ← c, b ← c, c ← d, e ← d, x ← d
+     *         bindings a↑, b↑, x↑, c↑, d↑, e↑
+     *
+     *     It should be possible to satisfy the requested ordering from the induced sub-poset of the original partially
+     *     ordered set over elements {a, e, c}:
+     *     induced sub-poset:
+     *         members: a, c, e
+     *         dependencies: a ← c
+     *
+     *     enumerated orderings (exhaustive) of the induced sub-poset:
+     *         a↑, c↑, e↑
+     *         a↑, e↑, c↑
+     *         e↑, a↑, c↑
+     *     }
+     *
+ * @param requestedOrdering the set of values which needs to match a prefix of a valid sequence + * @return the result of satisfiability of set of values in the ordering + */ public boolean satisfies(@Nonnull RequestedOrdering requestedOrdering) { - return !Iterables.isEmpty(enumerateCompatibleRequestedOrderings(requestedOrdering)); + if (requestedOrdering.isDistinct() && !isDistinct()) { + return false; + } + + for (final var requestedOrderingPart : requestedOrdering.getOrderingParts()) { + if (!bindingMap.containsKey(requestedOrderingPart.getValue())) { + return false; + } + final var bindings = bindingMap.get(requestedOrderingPart.getValue()); + final var sortOrder = sortOrder(bindings); + if (!sortOrder.isCompatibleWithRequestedSortOrder(requestedOrderingPart.getSortOrder())) { + return false; + } + } + final var requestedValuesSet = ImmutableSet.copyOf(requestedOrdering.getOrderingParts().stream().map(OrderingPart::getValue).iterator()); + return !Iterables.isEmpty( + TopologicalSort.satisfyingPermutations( + getOrderingSet().filterElements(requestedValuesSet::contains), + requestedOrdering.getOrderingParts().stream().map(OrderingPart::getValue).collect(Collectors.toList()), + Function.identity(), + permutation -> requestedOrdering.getOrderingParts().size())); } /** * Method to, given a constraining {@link RequestedOrdering}, enumerates all ordering sequences supported by this - * ordering that are compatible with the given {@link RequestedOrdering}. This functionality is needed when a - * provided order is used to again drive requested orderings for another quantifier participating in e.g. a set - * operation or similar. + * ordering that are compatible with the given {@link RequestedOrdering} having all the elements of the partially + * ordered set. This functionality is needed when a provided order is used to again drive requested orderings for + * another quantifier participating in e.g. a set operation or similar. * Example: * 
      *     {@code
@@ -311,14 +364,16 @@ public boolean satisfies(@Nonnull RequestedOrdering requestedOrdering) {
      *         bindings a↑, b=, x=, c↑, d↑, e↑
      *
      *     enumerated requested orderings (among others):
-     *         a↑, b↓, c↑
-     *         a↑, x↕, b↓, c↑
      *         a↑, x↕, b↓, c↑, d↕, e↕
      *         a↑, x↕, b↓, c↑, e↕, d↕
+     *
+     *     however, the following won't be in the enumerated sequences:
+     *         a↑, b↓, c↑
+     *         a↑, x↕, b↓, c↑
      *     }
      *
* @param requestedOrdering the {@link RequestedOrdering} this ordering needs to be compatible with - * @return an iterable of all compatible {@link RequestedOrdering}s + * @return boolean result of the compatibility check. */ @Nonnull public Iterable> enumerateCompatibleRequestedOrderings(@Nonnull final RequestedOrdering requestedOrdering) { @@ -342,7 +397,7 @@ public Iterable> enumerateCompatibleRequestedOrderin final var satisfyingValuePermutations = TopologicalSort.satisfyingPermutations( - getOrderingSet().filterElements(t -> requestedOrdering.getOrderingParts().stream().anyMatch(part -> part.getValue().equals(t))), + getOrderingSet(), ImmutableList.copyOf(requestedOrderingValuesMap.keySet()), Function.identity(), permutation -> requestedOrdering.getOrderingParts().size()); @@ -358,6 +413,25 @@ public Iterable> enumerateCompatibleRequestedOrderin .collect(ImmutableList.toImmutableList())); } + /** + * Method to verify that a given set of grouping values forms a valid prefix of an enumerated sequence that is + * supported by this ordering. + * Example: + * 
+     *     {@code
+     *     requested grouping values set:  {a, e, c}
+     *     this ordering:
+     *         partially ordered set:
+     *             members: a, b, x, c, d, e
+     *             dependencies: a ← c, b ← c, c ← d, e ← d, x ← d
+     *         bindings a↑, b↑, x↑, c↑, d↑, e↑
+     *
+     *     It should be possible to satisfy {a, c, e} as a and e are independent and c depends on a.
+     *     }
+     *
+ * @param requestedGroupingValues the set of values which needs to match a prefix of a valid sequence + * @return the result of satisfiability of set of values in the ordering + */ public boolean satisfiesGroupingValues(@Nonnull final Set requestedGroupingValues) { // no ordering left worth further considerations if (requestedGroupingValues.isEmpty()) { @@ -379,12 +453,11 @@ public boolean satisfiesGroupingValues(@Nonnull final Set requestedGroupi })) { return false; } - final var permutations = TopologicalSort.topologicalOrderPermutations(getOrderingSet().filterElements(requestedGroupingValues::contains)); for (final var permutation : permutations) { - final var containsAll = permutation.size() >= requestedGroupingValues.size() && + final var satisfies = permutation.size() >= requestedGroupingValues.size() && requestedGroupingValues.containsAll(permutation.subList(0, requestedGroupingValues.size())); - if (containsAll) { + if (satisfies) { return true; } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBInQueryTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBInQueryTest.java index 102328d3af..2cfc7cb0d7 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBInQueryTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBInQueryTest.java @@ -2293,7 +2293,7 @@ void testInQueryOrDifferentCondition() throws Exception { inUnionOnValuesPlan( predicatesFilterPlan(indexPlan().where(indexName("MySimpleRecord$num_value_unique")).and(scanComparisons(range("([990],>")))) .where(predicates(valuePredicate(fieldValueWithFieldNames("num_value_2"), anyValueComparison())))) - .where(comparisonKeyValues(exactly(fieldValueWithFieldNames("num_value_unique"), fieldValueWithFieldNames("rec_no")))))); + .where(comparisonKeyValues(exactly(fieldValueWithFieldNames("num_value_unique"), fieldValueWithFieldNames("rec_no"), fieldValueWithFieldNames("num_value_2")))))); assertEquals(1521186153, plan.planHash(PlanHashable.CURRENT_LEGACY)); assertEquals(-455093906, plan.planHash(PlanHashable.CURRENT_FOR_CONTINUATION)); } @@ -2805,7 +2805,7 @@ void testInUnionWithIntersectionOnTwoPredicates(int replans, boolean dropNumValu ).where(inUnionValuesSources(only(inUnionInParameter(equalsObject("str_list"))))) ).where(inUnionValuesSources(only(inUnionInParameter(equalsObject("nv2_list")))))); assertEquals(-1515112545, plan.planHash(PlanHashable.CURRENT_LEGACY)); - assertEquals(-1918411257, plan.planHash(PlanHashable.CURRENT_FOR_CONTINUATION)); + assertEquals(-1918440177, plan.planHash(PlanHashable.CURRENT_FOR_CONTINUATION)); singleIndexScan = false; } else if (replans < 0 || dropNumValue3Index) { // Before replanning, we always end up with an intersection. Neither index alone is enough to satisfy both diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java index ad515f1c83..85d040c9b4 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java @@ -475,6 +475,11 @@ void testEnumeration() { List.of(e, d, a, c)) .map(permutation -> requested(permutation.toArray())) .collect(Collectors.toSet()); + + + var actual0 = ordering.enumerateCompatibleRequestedOrderings(RequestedOrdering.ofParts(requested(), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of())); + + var actual1 = ordering.enumerateCompatibleRequestedOrderings(RequestedOrdering.ofParts(requested(a, c, e, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of())); actual1.forEach(actualPermutation -> assertTrue(expectedSet1.contains(actualPermutation))); From 30443f246a8ba5be242faefadea2cfb6b4c70367 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Wed, 17 Dec 2025 16:01:37 +0000 Subject: [PATCH 19/22] tests --- .../query/plan/cascades/OrderingTest.java | 244 ++++++++++++------ 1 file changed, 166 insertions(+), 78 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java index 85d040c9b4..caca69fd93 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java @@ -368,7 +368,7 @@ void testPullUp5() { } @Test - void testSatisfies() { + void testSatisfiesGroupingValues() { final var qov = ValueTestHelpers.qov(); final var a = ValueTestHelpers.field(qov, "a"); final var b = ValueTestHelpers.field(qov, "b"); @@ -387,27 +387,35 @@ void testSatisfies() { PartiallyOrderedSet.of(ImmutableSet.of(a, b, c, d, e, x), ImmutableSetMultimap.of(c, a, c, b, d, c, e, d, x, d)), false); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(b))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, c))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, b))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(b, c))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, b, c))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, c, d))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, c, d, e))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(b, c, d, x))); - - assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(c))); - assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(d))); - assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(e))); - assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(x))); - assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(a, d, x))); - assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(c, d, x))); - assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(e, d, x))); + Stream.of( + ImmutableSet.of(a), + ImmutableSet.of(b), + ImmutableSet.of(a, c), + ImmutableSet.of(a, b), + ImmutableSet.of(b, c), + ImmutableSet.of(a, b, c), + ImmutableSet.of(a, c, d), + ImmutableSet.of(a, c, d, e), + ImmutableSet.of(b, c, d, x), + ImmutableSet.of(b, c, d, x, e) + ).forEach(valueSet -> assertTrue(ordering.satisfiesGroupingValues(valueSet))); + + Stream.of( + ImmutableSet.of(c), + ImmutableSet.of(d), + ImmutableSet.of(e), + ImmutableSet.of(x), + ImmutableSet.of(a, d, x), + ImmutableSet.of(c, d, x), + ImmutableSet.of(e, d, x), + ImmutableSet.of(a, c, e), + ImmutableSet.of(b, c, x), + ImmutableSet.of(a, b, e, x) + ).forEach(valueSet -> assertFalse(ordering.satisfiesGroupingValues(valueSet))); } @Test - void testSatisfies2() { + void testSatisfiesGroupingValues2() { final var qov = ValueTestHelpers.qov(); final var a = ValueTestHelpers.field(qov, "a"); final var b = ValueTestHelpers.field(qov, "b"); @@ -426,28 +434,130 @@ void testSatisfies2() { PartiallyOrderedSet.of(ImmutableSet.of(a, b, c, d, e, x), ImmutableSetMultimap.of(c, a, c, b, d, c, d, e, d, x)), false); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(b))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(e))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(x))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, c))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, b))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(b, c))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, b, c))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, c, d))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, d, x))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(e, d, x))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(a, c, d, e))); - assertTrue(ordering.satisfiesGroupingValues(ImmutableSet.of(b, c, d, x))); - - assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(c))); - assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(c, d))); - assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(d))); - assertFalse(ordering.satisfiesGroupingValues(ImmutableSet.of(c, d, x))); + Stream.of( + ImmutableSet.of(a), + ImmutableSet.of(b), + ImmutableSet.of(e), + ImmutableSet.of(x), + ImmutableSet.of(a, c), + ImmutableSet.of(a, b), + ImmutableSet.of(b, c), + ImmutableSet.of(a, b, c), + ImmutableSet.of(a, c, d), + ImmutableSet.of(a, d, x), + ImmutableSet.of(e, d, x), + ImmutableSet.of(a, c, d, e), + ImmutableSet.of(b, c, d, x) + ).forEach(valueSet -> assertTrue(ordering.satisfiesGroupingValues(valueSet))); + + Stream.of( + ImmutableSet.of(c), + ImmutableSet.of(d), + ImmutableSet.of(c, d), + ImmutableSet.of(c, d, x) + ).forEach(valueSet -> assertFalse(ordering.satisfiesGroupingValues(valueSet))); } @Test - void testEnumeration() { + void testSatisfiesRequiredOrdering() { + final var qov = ValueTestHelpers.qov(); + final var a = ValueTestHelpers.field(qov, "a"); + final var b = ValueTestHelpers.field(qov, "b"); + final var c = ValueTestHelpers.field(qov, "c"); + final var d = ValueTestHelpers.field(qov, "d"); + final var e = ValueTestHelpers.field(qov, "e"); + final var x = ValueTestHelpers.field(qov, "x"); + + // a < c, b < c, c < d, d < e, d < x + final var ordering = Ordering.ofOrderingSet(bindingMap(a, ProvidedSortOrder.ASCENDING, + b, ProvidedSortOrder.ASCENDING, + c, ProvidedSortOrder.ASCENDING, + d, ProvidedSortOrder.ASCENDING, + e, ProvidedSortOrder.ASCENDING, + x, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(a, b, c, d, e, x), ImmutableSetMultimap.of(c, a, c, b, d, c, e, d, x, d)), + false); + + Stream.of( + RequestedOrdering.ofParts(requested(a), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(b), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, c), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, b), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(b, c), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, b, c), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, c, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, c, d, e), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(b, c, d, x), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(b, c, d, x, e), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()) + ).forEach(requestedOrdering -> assertTrue(ordering.satisfies(requestedOrdering))); + + Stream.of( + RequestedOrdering.ofParts(requested(c), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(e), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(x), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(c, a), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(c, b), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, d, c), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, d, x), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(c, d, x), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(e, d, x), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, c, e, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(b, c, x, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(b, c, x, d, e), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()) + ).forEach(requestedOrdering -> assertFalse(ordering.satisfies(requestedOrdering))); + } + + @Test + void testSatisfiesRequiredOrdering2() { + final var qov = ValueTestHelpers.qov(); + final var a = ValueTestHelpers.field(qov, "a"); + final var b = ValueTestHelpers.field(qov, "b"); + final var c = ValueTestHelpers.field(qov, "c"); + final var d = ValueTestHelpers.field(qov, "d"); + final var e = ValueTestHelpers.field(qov, "e"); + final var x = ValueTestHelpers.field(qov, "x"); + + // a < c, b < c, c < d, e < d, x < d + final var ordering = Ordering.ofOrderingSet(bindingMap(a, ProvidedSortOrder.ASCENDING, + b, ProvidedSortOrder.ASCENDING, + c, ProvidedSortOrder.ASCENDING, + d, ProvidedSortOrder.ASCENDING, + e, ProvidedSortOrder.ASCENDING, + x, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(a, b, c, d, e, x), ImmutableSetMultimap.of(c, a, c, b, d, c, d, e, d, x)), + false); + + Stream.of( + RequestedOrdering.ofParts(requested(a), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(b), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(e), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(x), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, c), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, b), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(b, c), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, b, c), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, c, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(x, a, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, x, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(e, x, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(x, e, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, c, e, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, e, c, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(e, a, c, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(b, c, x, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()) + ).forEach(requestedOrdering -> assertTrue(ordering.satisfies(requestedOrdering))); + + Stream.of( + RequestedOrdering.ofParts(requested(c), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(c, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(c, d, x), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()) + ).forEach(requestedOrdering -> assertFalse(ordering.satisfies(requestedOrdering))); + } + + @Test + void testEnumerateCompatibleRequestedOrderings() { final var qov = ValueTestHelpers.qov(); final var a = ValueTestHelpers.field(qov, "a"); final var b = ValueTestHelpers.field(qov, "b"); @@ -466,45 +576,23 @@ void testEnumeration() { PartiallyOrderedSet.of(ImmutableSet.of(a, b, c, d, e, x), ImmutableSetMultimap.of(c, a, c, b, d, c, d, e, d, x)), false); - final var expectedSet1 = Stream.of( - List.of(a, c, e, d), - List.of(a, e, c, d), - List.of(a, e, d, c), - List.of(e, a, c, d), - List.of(e, a, d, c), - List.of(e, d, a, c)) - .map(permutation -> requested(permutation.toArray())) - .collect(Collectors.toSet()); - - - var actual0 = ordering.enumerateCompatibleRequestedOrderings(RequestedOrdering.ofParts(requested(), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of())); - - - var actual1 = ordering.enumerateCompatibleRequestedOrderings(RequestedOrdering.ofParts(requested(a, c, e, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of())); - actual1.forEach(actualPermutation -> assertTrue(expectedSet1.contains(actualPermutation))); - - final var expectedSet2 = Stream.of( - List.of(b, x, d), - List.of(x, b, d), - List.of(x, d, b)) - .map(permutation -> requested(permutation.toArray())) - .collect(Collectors.toSet()); - final var actual2 = ordering.enumerateCompatibleRequestedOrderings(RequestedOrdering.ofParts(requested(b, x, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of())); - actual2.forEach(actualPermutation -> assertTrue(expectedSet2.contains(actualPermutation))); - - final var expectedSet3 = Stream.of( - List.of()) - .map(permutation -> requested(permutation.toArray())) - .collect(Collectors.toSet()); - final var actual3 = ordering.enumerateCompatibleRequestedOrderings(RequestedOrdering.ofParts(requested(c, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of())); - actual3.forEach(actualPermutation -> assertTrue(expectedSet3.contains(actualPermutation))); - - final var expectedSet4 = Stream.of( - List.of(a, c, d)) - .map(permutation -> requested(permutation.toArray())) - .collect(Collectors.toSet()); - final var actual4 = ordering.enumerateCompatibleRequestedOrderings(RequestedOrdering.ofParts(requested(a, c, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of())); - actual4.forEach(actualPermutation -> assertTrue(expectedSet4.contains(actualPermutation))); + + var requestedOrdering = RequestedOrdering.ofParts(requested(), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()); + // #pattern of _, _, c, _, _, d = 2!*2! = 4 + // #pattern of _, _, _, c, _, d = 2*3!*1! = 12 + // #pattern of _, _, _, _, c, d = 4! = 24 + assertEquals(40, ImmutableList.copyOf(ordering.enumerateCompatibleRequestedOrderings(requestedOrdering)).size()); + + requestedOrdering = RequestedOrdering.ofParts(requested(a), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()); + // #pattern of a, _, c, _, _, d = 1!*2! = 2 + // #pattern of a, _, _, c, _, d = 2*2!*1! = 4 + // #pattern of a, _, _, _, c, d = 3! = 6 + assertEquals(12, ImmutableList.copyOf(ordering.enumerateCompatibleRequestedOrderings(requestedOrdering)).size()); + + requestedOrdering = RequestedOrdering.ofParts(requested(e, a), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()); + // #pattern of e, a, _, c, _, d = 1 + // #pattern of e, a, _, _, c, d = 2! = 2 + assertEquals(3, ImmutableList.copyOf(ordering.enumerateCompatibleRequestedOrderings(requestedOrdering)).size()); } @Test @@ -659,7 +747,7 @@ void testCommonOrdering() { final var satisfyingOrderingsIterable = mergedOrdering.enumerateCompatibleRequestedOrderings(requestedOrdering); final var onlySatisfyingOrdering = Iterables.getOnlyElement(satisfyingOrderingsIterable); - assertEquals(requested(a, b, c), onlySatisfyingOrdering); + assertEquals(requested(a, b, c, d, RequestedSortOrder.ANY), onlySatisfyingOrdering); } @Test From ac5c6831cccbe4efc8f41682bf42fe97e3ae9b3d Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Wed, 17 Dec 2025 17:12:13 +0000 Subject: [PATCH 20/22] tests --- .../combinatorics/PartiallyOrderedSet.java | 26 ------- .../PartiallyOrderedSetTest.java | 72 +++++++++++++++++++ 2 files changed, 72 insertions(+), 26 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java index 60f765f70e..582e60b0b0 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java @@ -211,32 +211,6 @@ public PartiallyOrderedSet mapAll(@Nonnull final Map map) { return PartiallyOrderedSet.of(mappedElements, resultDependencyMapBuilder.build()); } -// @Nonnull -// public PartiallyOrderedSet mapAll(@Nonnull final Map map) { -// final var mappedElements = Sets.newLinkedHashSet(map.values()); -// -// final var resultDependencyMapBuilder = ImmutableSetMultimap.builder(); -// for (final var entry : getTransitiveClosure().entries()) { -// final var key = entry.getKey(); -// final var value = entry.getValue(); -// -// if (map.containsKey(key) && map.containsKey(value)) { -// resultDependencyMapBuilder.put(map.get(key), map.get(value)); -// } else { -// if (!map.containsKey(value)) { -// // if key depends on value that does not exist -- do not insert the dependency and also remove key -// final var mappedKey = map.get(key); -// if (mappedKey != null) { -// mappedElements.remove(mappedKey); -// } -// } -// } -// } -// -// // this needs the dependency map to be cleansed (which is done in the constructor) -// return PartiallyOrderedSet.of(mappedElements, resultDependencyMapBuilder.build()); -// } - @Nonnull public PartiallyOrderedSet mapAll(@Nonnull final Multimap map) { final var identityMapped = this.filterElements(map::containsKey); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSetTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSetTest.java index b2189a897f..3846aec861 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSetTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSetTest.java @@ -23,6 +23,7 @@ import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; import org.junit.jupiter.api.Test; import java.util.Map; @@ -160,4 +161,75 @@ void testEligibleSetsSingle() { eligibleSet = eligibleSet.removeEligibleElements(eligibleSet.eligibleElements()); assertTrue(eligibleSet.eligibleElements().isEmpty()); } + + @Test + void testFilterElements() { + final var a = CorrelationIdentifier.of("a"); + final var b = CorrelationIdentifier.of("b"); + final var c = CorrelationIdentifier.of("c"); + final var d = CorrelationIdentifier.of("d"); + final var e = CorrelationIdentifier.of("e"); + final var f = CorrelationIdentifier.of("f"); + final var g = CorrelationIdentifier.of("g"); + + // a < c, b < c, c < d, d < e, d < f, e < g, f < g + final var dependencyMapBuilder = ImmutableSetMultimap.builder(); + dependencyMapBuilder.putAll(c, b, a); + dependencyMapBuilder.putAll(d, c); + dependencyMapBuilder.putAll(e, d); + dependencyMapBuilder.putAll(f, d); + dependencyMapBuilder.putAll(g, e, f); + final var partialOrder = PartiallyOrderedSet.of(ImmutableSet.of(a, b, c, d, e, f, g), dependencyMapBuilder.build()); + + // a < c, b < c + var filteredSet = ImmutableSet.of(a, b, c); + var actualSubset = partialOrder.filterElements(filteredSet::contains); + var expectedDependencyEntries = ImmutableSetMultimap.of(c, a, c, b); + actualSubset.getDependencyMap().entries().forEach(entry -> assertTrue(expectedDependencyEntries.get(entry.getKey()).contains(entry.getValue()))); + + // {a} + filteredSet = ImmutableSet.of(a); + actualSubset = partialOrder.filterElements(filteredSet::contains); + assertTrue(actualSubset.getDependencyMap().isEmpty()); + assertTrue(actualSubset.getSet().containsAll(filteredSet)); + + filteredSet = ImmutableSet.of(c, d, e, f, g); + actualSubset = partialOrder.filterElements(filteredSet::contains); + assertTrue(actualSubset.getDependencyMap().isEmpty()); + assertTrue(actualSubset.getSet().isEmpty()); + + filteredSet = ImmutableSet.of(a, e, g); + actualSubset = partialOrder.filterElements(filteredSet::contains); + assertTrue(actualSubset.getDependencyMap().isEmpty()); + assertTrue(actualSubset.getSet().containsAll(ImmutableSet.of(a))); + } + + @Test + void testMapAllWithMultiMap() { + final var a = CorrelationIdentifier.of("a"); + final var b = CorrelationIdentifier.of("b"); + final var c = CorrelationIdentifier.of("c"); + final var d = CorrelationIdentifier.of("d"); + + // a < b, a < c, c < d, b < d + final var dependencyMapBuilder = ImmutableSetMultimap.builder(); + dependencyMapBuilder.putAll(c, a); + dependencyMapBuilder.putAll(b, a); + dependencyMapBuilder.putAll(d, b, c); + final var partialOrder = PartiallyOrderedSet.of(ImmutableSet.of(a, b, c, d), dependencyMapBuilder.build()); + + var mapToBuilder = ImmutableSetMultimap.builder(); + final var a_1 = CorrelationIdentifier.of("a_1"); + final var a_2 = CorrelationIdentifier.of("a_2"); + final var b_1 = CorrelationIdentifier.of("b_1"); + final var d_1 = CorrelationIdentifier.of("d_1"); + final var d_2 = CorrelationIdentifier.of("d_2"); + mapToBuilder.putAll(a, a_1, a_2); + mapToBuilder.putAll(b, b_1); + mapToBuilder.putAll(d, d_1, d_2); + + var actualSubset = partialOrder.mapAll(mapToBuilder.build()); + var expectedDependencyEntries = ImmutableSetMultimap.of(b_1, a_1, b_1, a_2, d_1, b_1, d_2, b_1); + actualSubset.getDependencyMap().entries().forEach(entry -> assertTrue(expectedDependencyEntries.get(entry.getKey()).contains(entry.getValue()))); + } } From 7d35af28969536dc779c13d8a8aabfa5f1b67742 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Wed, 17 Dec 2025 22:52:33 +0000 Subject: [PATCH 21/22] fix rule to match complex values correctly --- ...ValueAgainstQuantifiedObjectValueRule.java | 4 ++ .../simplification/ValueComputationTest.java | 54 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueAgainstQuantifiedObjectValueRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueAgainstQuantifiedObjectValueRule.java index 3e1d6f087f..07aef27efb 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueAgainstQuantifiedObjectValueRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueAgainstQuantifiedObjectValueRule.java @@ -84,6 +84,10 @@ public void onMatch(@Nonnull final ValueComputationRuleCall { final var translationMapBuilder = TranslationMap.regularBuilder(); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueComputationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueComputationTest.java index 075cc097ae..ab5bc77416 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueComputationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueComputationTest.java @@ -44,6 +44,7 @@ import org.junit.jupiter.api.Test; import javax.annotation.Nonnull; +import java.util.List; import java.util.Optional; /** @@ -124,6 +125,31 @@ void testPullUpAmbiguous() { Assertions.assertEquals(expectedMap, resultsMap); } + @Test + void testPullUpComplexValue() { + final var qa = CorrelationIdentifier.of("qa"); + final var someCurrentValue = QuantifiedObjectValue.of(qa, someRecordType()); + // (_.x as _0, _.x.xa as _1) + final var pulledThroughValue = RecordConstructorValue.ofUnnamed(List.of(someCurrentValue)); + + // _.x.xa + final var _x_xa = FieldValue.ofFieldNames(someCurrentValue, ImmutableList.of("x", "xa")); + final var toBePulledUpValues = ImmutableList.of(_x_xa); + + final var resultsMap = pulledThroughValue.pullUp(toBePulledUpValues, + EvaluationContext.empty(), AliasMap.emptyMap(), ImmutableSet.of(), ALIAS); + + final var upperCurrentValue = QuantifiedObjectValue.of(ALIAS, pulledThroughValue.getResultType()); + + // _.x.xa -> { _0.xa, _1 } + final var expectedMap = ImmutableMultimap.of( + _x_xa, FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_0", "xa")), + _x_xa, FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_1")) + ); + + Assertions.assertEquals(expectedMap, resultsMap); + } + @Test void testPullUpFieldValueThroughStreamingAggregation() { // _ @@ -212,6 +238,34 @@ record_type, new RecordTypeValue(FieldValue.ofFieldName(upperCurrentValue, "_1") Assertions.assertEquals(expectedMap, resultsMap); } + @Test + void testPullUpValue3() { + final var alias1 = CorrelationIdentifier.of("blah1"); + final var qov1 = QuantifiedObjectValue.of(alias1, someRecordType()); + final var alias2 = CorrelationIdentifier.of("blah2"); + final var qov2 = QuantifiedObjectValue.of(alias2, Type.Record.fromFields(List.of( + Type.Record.Field.of(Type.primitiveType(Type.TypeCode.STRING), Optional.of("r2_a"))))); + + final var pulledThroughValue = RecordConstructorValue.ofUnnamed(ImmutableList.of(qov1, qov2)); + + final var _a_b = FieldValue.ofFieldNames(qov1, ImmutableList.of("a", "ab")); + final var _x_b = FieldValue.ofFieldNames(qov1, ImmutableList.of("x", "xb")); + // _.a.ab + _.x.xb + final var _a_ab__plus__x_xb = (Value)new ArithmeticValue.AddFn().encapsulate(ImmutableList.of(_a_b, _x_b)); + final var toBePulledUpValues = ImmutableList.of(_a_ab__plus__x_xb); + + final var resultsMap = pulledThroughValue.pullUp(toBePulledUpValues, + EvaluationContext.empty(), AliasMap.emptyMap(), ImmutableSet.of(), ALIAS); + final var upperCurrentValue = QuantifiedObjectValue.of(ALIAS, pulledThroughValue.getResultType()); + + final var new_a_b = FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_0", "a", "ab")); + final var new_x_b = FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_0", "x", "xb")); + + final var expectedMap = ImmutableMultimap.of( + _a_ab__plus__x_xb, (Value)new ArithmeticValue.AddFn().encapsulate(ImmutableList.of(new_a_b, new_x_b))); + Assertions.assertEquals(expectedMap, resultsMap); + } + @Test void testMatchAndCompensateRemainder1() { final var someCurrentValue = ObjectValue.of(ALIAS, someRecordType()); From 3c945af42f70ea9da5895829be40ee45ccacc0b8 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Wed, 17 Dec 2025 23:45:22 +0000 Subject: [PATCH 22/22] fix fix --- ...ValueAgainstQuantifiedObjectValueRule.java | 3 ++- .../simplification/ValueComputationTest.java | 25 ------------------- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueAgainstQuantifiedObjectValueRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueAgainstQuantifiedObjectValueRule.java index 07aef27efb..89b7c72c69 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueAgainstQuantifiedObjectValueRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueAgainstQuantifiedObjectValueRule.java @@ -22,6 +22,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.query.plan.cascades.LinkedIdentityMap; +import com.apple.foundationdb.record.query.plan.cascades.Quantifier; import com.apple.foundationdb.record.query.plan.cascades.matching.structure.BindingMatcher; import com.apple.foundationdb.record.query.plan.cascades.matching.structure.ValueMatchers; import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; @@ -84,7 +85,7 @@ public void onMatch(@Nonnull final ValueComputationRuleCallof(_x_xa); - - final var resultsMap = pulledThroughValue.pullUp(toBePulledUpValues, - EvaluationContext.empty(), AliasMap.emptyMap(), ImmutableSet.of(), ALIAS); - - final var upperCurrentValue = QuantifiedObjectValue.of(ALIAS, pulledThroughValue.getResultType()); - - // _.x.xa -> { _0.xa, _1 } - final var expectedMap = ImmutableMultimap.of( - _x_xa, FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_0", "xa")), - _x_xa, FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_1")) - ); - - Assertions.assertEquals(expectedMap, resultsMap); - } - @Test void testPullUpFieldValueThroughStreamingAggregation() { // _