WIP: Attach __ROW_VERSION pseudo-field to types if the record meta-data stores versions

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java

@@ -29,9 +29,9 @@
 import com.apple.foundationdb.record.metadata.RecordType;
 import com.apple.foundationdb.record.metadata.SyntheticRecordType;
 import com.apple.foundationdb.record.metadata.UnnestedRecordType;
+import com.apple.foundationdb.record.metadata.View;
 import com.apple.foundationdb.record.metadata.expressions.KeyExpression;
 import com.apple.foundationdb.record.metadata.expressions.LiteralKeyExpression;
-import com.apple.foundationdb.record.metadata.View;
 import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction;
 import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
 import com.apple.foundationdb.record.query.plan.synthetic.SyntheticRecordPlanner;
@@ -45,15 +45,13 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 /**
  * Meta-data for Record Layer record stores.
@@ -720,11 +718,6 @@
         return builder.build();
     }
 
-    @Nonnull
-    public Map<String, Descriptors.FieldDescriptor> getFieldDescriptorMapFromNames(@Nonnull final Collection<String> recordTypeNames) {
-        return getFieldDescriptorMap(recordTypeNames.stream().map(this::getRecordType));
-    }
-
     @Nonnull
     public Map<String, UserDefinedFunction> getUserDefinedFunctionMap() {
         return userDefinedFunctionMap;
@@ -736,39 +729,50 @@
     }
 
     @Nonnull
-    public static Map<String, Descriptors.FieldDescriptor> getFieldDescriptorMapFromTypes(@Nonnull final Collection<RecordType> recordTypes) {
-        if (recordTypes.size() == 1) {
-            final var recordType = Iterables.getOnlyElement(recordTypes);
-            return Type.Record.toFieldDescriptorMap(recordType.getDescriptor().getFields());
+    public Type.Record getPlannerType(@Nonnull String recordTypeName) {
+        final RecordType recordType = getRecordType(recordTypeName);
+        Type.Record plannerType = Type.Record.fromDescriptor(recordType.getDescriptor());
+        if (storeRecordVersions) {
+            plannerType = plannerType.addPseudoFields();
         }
-        return getFieldDescriptorMap(recordTypes.stream());
+        return plannerType;
     }
 
     @Nonnull
-    private static Map<String, Descriptors.FieldDescriptor> getFieldDescriptorMap(@Nonnull final Stream<RecordType> recordTypeStream) {
+    public Type.Record getPlannerType(@Nonnull Collection<String> recordTypeNames) {
+        if (recordTypeNames.size() == 1) {
+            final String recordTypeName = Iterables.getOnlyElement(recordTypeNames);
+            return getPlannerType(recordTypeName);
+        }
         // todo: should be removed https://github.com/FoundationDB/fdb-record-layer/issues/1884
-        return recordTypeStream
-                .sorted(Comparator.comparing(RecordType::getName))
-                .flatMap(recordType -> recordType.getDescriptor().getFields().stream())
-                .collect(Collectors.groupingBy(Descriptors.FieldDescriptor::getName,
+        LinkedHashMap<String, Type.Record.Field> fieldsByName = recordTypeNames.stream()
+                .map(this::getPlannerType)
+                .flatMap(type -> type.getFields().stream())
+                .collect(Collectors.groupingBy(Type.Record.Field::getFieldName,
                         LinkedHashMap::new,
-                        Collectors.reducing(null,
-                                (fieldDescriptor, fieldDescriptor2) -> {
-                                    Verify.verify(fieldDescriptor != null || fieldDescriptor2 != null);
-                                    if (fieldDescriptor == null) {
-                                        return fieldDescriptor2;
-                                    }
-                                    if (fieldDescriptor2 == null) {
-                                        return fieldDescriptor;
-                                    }
-                                    // TODO improve
-                                    if (fieldDescriptor.getType().getJavaType() ==
-                                            fieldDescriptor2.getType().getJavaType()) {
-                                        return fieldDescriptor;
-                                    }
-
-                                    throw new IllegalArgumentException("cannot form union type of complex fields");
-                                })));
+                        Collectors.reducing(null, (Type.Record.Field f1, Type.Record.Field f2) -> {
+                            Verify.verify(f1 != null || f2 != null);
+                            if (f1 == null) {
+                                return Type.Record.Field.of(f2.getFieldType(), f2.getFieldNameOptional());
+                            }
+                            if (f2 == null) {
+                                return Type.Record.Field.of(f1.getFieldType(), f1.getFieldNameOptional());
+                            }
+                            // TODO improve
+                            final Type f1Type = f1.getFieldType();
+                            final Type f2Type = f2.getFieldType();
+                            if (f1Type.equals(f2Type)) {
+                                return Type.Record.Field.of(f1Type, f1.getFieldNameOptional());
+                            } else if (f2Type.isNullable() && f1Type.nullable().equals(f2Type)) {
+                                return Type.Record.Field.of(f2Type, f2.getFieldNameOptional());
+                            } else if (f1Type.isNullable() && f2Type.nullable().equals(f1Type)) {
+                                return Type.Record.Field.of(f1Type, f1.getFieldNameOptional());
+                            }
+
+                            throw new IllegalArgumentException("cannot form union type of complex fields");
+                        })
+                ));
+        return Type.Record.fromFields(false, List.copyOf(fieldsByName.values()));
     }
 
     @Nonnull

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java

@@ -36,13 +36,13 @@
 import com.apple.foundationdb.record.metadata.SyntheticRecordType;
 import com.apple.foundationdb.record.metadata.SyntheticRecordTypeBuilder;
 import com.apple.foundationdb.record.metadata.UnnestedRecordTypeBuilder;
+import com.apple.foundationdb.record.metadata.View;
 import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression;
 import com.apple.foundationdb.record.metadata.expressions.KeyExpression;
 import com.apple.foundationdb.record.metadata.expressions.LiteralKeyExpression;
-import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistry;
 import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactoryRegistryImpl;
+import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistry;
 import com.apple.foundationdb.record.provider.foundationdb.MetaDataProtoEditor;
-import com.apple.foundationdb.record.metadata.View;
 import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction;
 import com.google.common.base.Verify;
 import com.google.common.collect.ImmutableMap;

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java

@@ -29,10 +29,11 @@
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecordVersion;
 import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier;
 import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
+import com.apple.foundationdb.record.query.plan.cascades.typing.PseudoField;
 import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
-import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedRecordValue;
+import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue;
+import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
-import com.apple.foundationdb.record.query.plan.cascades.values.VersionValue;
 import com.google.protobuf.Descriptors;
 import com.google.protobuf.Message;
 
@@ -116,7 +117,7 @@ public <S extends KeyExpressionVisitor.State, R> R expand(@Nonnull final KeyExpr
     @Nonnull
     @Override
     public Value toValue(@Nonnull final CorrelationIdentifier baseAlias, @Nonnull final Type baseType) {
-        return new VersionValue(QuantifiedRecordValue.of(baseAlias, baseType));
+        return FieldValue.ofFieldName(QuantifiedObjectValue.of(baseAlias, baseType), PseudoField.ROW_VERSION.getFieldName());
     }
 
     @Override

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/QueryRecordFunctionWithComparison.java

@@ -36,10 +36,11 @@
 import com.apple.foundationdb.record.query.plan.cascades.Quantifier;
 import com.apple.foundationdb.record.query.plan.cascades.Reference;
 import com.apple.foundationdb.record.query.plan.cascades.predicates.ValuePredicate;
+import com.apple.foundationdb.record.query.plan.cascades.typing.PseudoField;
+import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue;
-import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedRecordValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.RankValue;
-import com.apple.foundationdb.record.query.plan.cascades.values.VersionValue;
+import com.apple.foundationdb.record.query.plan.cascades.values.Value;
 import com.google.common.collect.Lists;
 import com.google.protobuf.Descriptors;
 import com.google.protobuf.Message;
@@ -162,7 +163,7 @@ public GraphExpansion expand(@Nonnull final Quantifier.ForEach baseQuantifier,
                     Quantifier.existential(Reference.initialOf(rankSelectExpression));
             return GraphExpansion.ofExists(rankComparisonQuantifier);
         } else if (function instanceof StoreRecordFunction<?> && FunctionNames.VERSION.equals(function.getName())) {
-            final VersionValue versionValue = new VersionValue(QuantifiedRecordValue.of(baseQuantifier));
+            final Value versionValue = FieldValue.ofFieldNameAndFuseIfPossible(baseQuantifier.getFlowedObjectValue(), PseudoField.ROW_VERSION.getFieldName());
             return GraphExpansion.builder()
                     .addPredicate(new ValuePredicate(versionValue, comparison))
                     .build();

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/IndexKeyValueToPartialRecord.java

@@ -715,7 +715,9 @@ public Builder addField(@Nonnull final String field, @Nonnull final TupleSource
                                 @Nonnull final AvailableFields.CopyIfPredicate copyIfPredicate,
                                 @Nonnull final ImmutableIntArray ordinalPath,
                                 @Nullable final String invertibleFunction) {
-            validateField(field);
+            if (!validateField(field)) {
+                return this;
+            }
             FieldCopier copier = new FieldCopier(field, source, copyIfPredicate, ordinalPath, invertibleFunction);
             if (fields.put(field, copier) != null) {
                 throw new RecordCoreException("setting field more than once: " + field);
@@ -725,7 +727,9 @@ public Builder addField(@Nonnull final String field, @Nonnull final TupleSource
 
         public Builder addField(@Nonnull final String field,
                                 @Nonnull final Value extractFromIndexEntryValue) {
-            validateField(field);
+            if (!validateField(field)) {
+                return this;
+            }
             final Copier copier =
                     new FieldWithValueCopier(Quantifier.current(),
                             ConstantPredicate.TRUE,
@@ -737,15 +741,18 @@ public Builder addField(@Nonnull final String field,
             return this;
         }
 
-        private void validateField(final @Nonnull String field) {
+        private boolean validateField(final @Nonnull String field) {
             final Descriptors.FieldDescriptor fieldDescriptor = recordDescriptor.findFieldByName(field);
             if (fieldDescriptor == null) {
-                throw new MetaDataException("field not found: " + field);
+                // Field not in record descriptor. This can happen when the PseudoFields are in the index
+                // definition. Just skip over it
+                return false;
             }
             if (fieldDescriptor.getType() == Descriptors.FieldDescriptor.Type.MESSAGE &&
                     !TupleFieldsHelper.isTupleField(fieldDescriptor.getMessageType())) {
                 throw new RecordCoreException("must set nested message field-by-field: " + field);
             }
+            return true;
         }
 
         public Builder getFieldBuilder(@Nonnull String field) {

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/bitmap/ComposedBitmapIndexQueryPlan.java

@@ -106,7 +106,7 @@ public <M extends Message> RecordCursor<QueryResult> executePlan(@Nonnull final
                 // Composers can return null bitmaps when empty, which is then left out of the result set.
                 .filter(indexEntry -> indexEntry.getValue().get(0) != null)
                 .map(indexPlans.get(0).indexEntryToQueriedRecord(store))
-                .map(QueryResult::fromQueriedRecord);
+                .map(queriedRecord -> QueryResult.fromQueriedRecord(getResultValue().getResultType(), context, queriedRecord));
     }
 
     @Override

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexMatchCandidate.java

@@ -23,7 +23,6 @@
 import com.apple.foundationdb.record.EvaluationContext;
 import com.apple.foundationdb.record.IndexScanType;
 import com.apple.foundationdb.record.RecordCoreException;
-import com.apple.foundationdb.record.RecordMetaData;
 import com.apple.foundationdb.record.logging.LogMessageKeys;
 import com.apple.foundationdb.record.metadata.Index;
 import com.apple.foundationdb.record.metadata.IndexOptions;
@@ -399,8 +398,6 @@ public RecordQueryPlan toEquivalentPlan(@Nonnull final PartialMatch partialMatch
                                             @Nonnull final Memoizer memoizer,
                                             @Nonnull final List<ComparisonRange> comparisonRanges,
                                             final boolean reverseScanOrder) {
-        final var baseRecordType = Type.Record.fromFieldDescriptorsMap(RecordMetaData.getFieldDescriptorMapFromTypes(recordTypes));
-
         final var selectHavingResultValue = selectHavingExpression.getResultValue();
         final var resultType = (Type.Record)selectHavingResultValue.getResultType();
         final var messageDescriptor =
@@ -420,7 +417,7 @@ public RecordQueryPlan toEquivalentPlan(@Nonnull final PartialMatch partialMatch
                 reverseScanOrder,
                 false,
                 partialMatch.getMatchCandidate(),
-                baseRecordType,
+                baseType.narrowRecordMaybe().orElseThrow(() -> new RecordCoreException("type is of wrong implementor")),
                 QueryPlanConstraint.noConstraint());
 
         var plan = new RecordQueryAggregateIndexPlan(aggregateIndexScan,

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/IndexExpansionInfo.java

@@ -25,6 +25,7 @@
 import com.apple.foundationdb.record.metadata.Index;
 import com.apple.foundationdb.record.metadata.RecordType;
 import com.apple.foundationdb.record.metadata.expressions.KeyExpression;
+import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
 import com.google.common.collect.ImmutableSet;
 
 import javax.annotation.Nonnull;
@@ -51,18 +52,22 @@ public final class IndexExpansionInfo {
     private final Collection<RecordType> indexedRecordTypes;
     @Nonnull
     private final Set<String> indexedRecordTypeNames;
+    @Nonnull
+    private final Type.Record plannerBaseType;
 
     private IndexExpansionInfo(@Nonnull RecordMetaData metaData,
                                @Nonnull Index index,
                                boolean reverse,
                                @Nonnull Collection<RecordType> indexedRecordTypes,
                                @Nonnull Set<String> indexedRecordTypeNames,
+                               @Nonnull Type.Record plannerBaseType,
                                @Nullable KeyExpression commonPrimaryKeyForTypes) {
         this.metaData = metaData;
         this.index = index;
         this.reverse = reverse;
         this.indexedRecordTypes = indexedRecordTypes;
         this.indexedRecordTypeNames = indexedRecordTypeNames;
+        this.plannerBaseType = plannerBaseType;
         this.commonPrimaryKeyForTypes = commonPrimaryKeyForTypes;
     }
 
@@ -105,6 +110,11 @@ public Set<String> getAvailableRecordTypeNames() {
         return metaData.getRecordTypes().keySet();
     }
 
+    @Nonnull
+    public Type.Record getPlannerBaseType() {
+        return plannerBaseType;
+    }
+
     /**
      * Create an {@link IndexExpansionInfo} for a given index.
      * This wraps the given parameters into a single object, as well
@@ -126,10 +136,12 @@ public static IndexExpansionInfo createInfo(@Nonnull RecordMetaData metaData,
         final Set<String> indexedRecordTypeNames = indexedRecordTypes.stream()
                 .map(RecordType::getName)
                 .collect(ImmutableSet.toImmutableSet());
+        @Nonnull
+        final Type.Record plannerBaseType = metaData.getPlannerType(indexedRecordTypeNames);
         @Nullable
         final KeyExpression commonPrimaryKeyForTypes = RecordMetaData.commonPrimaryKey(indexedRecordTypes);
 
-        return new IndexExpansionInfo(metaData, index, reverse, indexedRecordTypes, indexedRecordTypeNames, commonPrimaryKeyForTypes);
+        return new IndexExpansionInfo(metaData, index, reverse, indexedRecordTypes, indexedRecordTypeNames, plannerBaseType, commonPrimaryKeyForTypes);
     }
 
 }