diff --git a/pbj-core/pbj-compiler/src/main/java/com/hedera/pbj/compiler/impl/generators/protobuf/CodecGenerator.java b/pbj-core/pbj-compiler/src/main/java/com/hedera/pbj/compiler/impl/generators/protobuf/CodecGenerator.java index 327c6ca1..ab887014 100644 --- a/pbj-core/pbj-compiler/src/main/java/com/hedera/pbj/compiler/impl/generators/protobuf/CodecGenerator.java +++ b/pbj-core/pbj-compiler/src/main/java/com/hedera/pbj/compiler/impl/generators/protobuf/CodecGenerator.java @@ -59,6 +59,8 @@ public void generate( } final String writeMethod = CodecWriteMethodGenerator.generateWriteMethod(modelClassName, schemaClassName, fields); + final String writeByteArrayMethod = + CodecWriteByteArrayMethodGenerator.generateWriteMethod(modelClassName, schemaClassName, fields); final String staticModifier = Generator.isInner(msgDef) ? " static" : ""; @@ -66,6 +68,8 @@ public void generate( writer.addImport("com.hedera.pbj.runtime.io.*"); writer.addImport("com.hedera.pbj.runtime.io.buffer.*"); writer.addImport("com.hedera.pbj.runtime.io.stream.EOFException"); + writer.addImport("com.hedera.pbj.runtime.io.stream.WritableStreamingData"); + writer.addImport("com.hedera.pbj.runtime.ProtoArrayWriterTools"); writer.addImport("java.io.IOException"); writer.addImport("java.nio.*"); writer.addImport("java.nio.charset.*"); @@ -78,6 +82,7 @@ public void generate( writer.addImport("static com.hedera.pbj.runtime.ProtoWriterTools.*"); writer.addImport("static com.hedera.pbj.runtime.ProtoParserTools.*"); writer.addImport("static com.hedera.pbj.runtime.ProtoConstants.*"); + writer.addImport("static com.hedera.pbj.runtime.Utf8Tools.*"); // spotless:off writer.append(""" @@ -104,6 +109,7 @@ public void generate( $unsetOneOfConstants $parseMethod $writeMethod + $writeByteArrayMethod $measureDataMethod $measureRecordMethod $fastEqualsMethod @@ -116,6 +122,7 @@ public void generate( .replace("$unsetOneOfConstants", CodecParseMethodGenerator.generateUnsetOneOfConstants(fields)) .replace("$parseMethod", CodecParseMethodGenerator.generateParseMethod(modelClassName, schemaClassName, fields)) .replace("$writeMethod", writeMethod) + .replace("$writeByteArrayMethod", writeByteArrayMethod) .replace("$measureDataMethod", CodecMeasureDataMethodGenerator.generateMeasureMethod(modelClassName, fields)) .replace("$measureRecordMethod", CodecMeasureRecordMethodGenerator.generateMeasureMethod(modelClassName, fields)) .replace("$fastEqualsMethod", CodecFastEqualsMethodGenerator.generateFastEqualsMethod(modelClassName, fields)) diff --git a/pbj-core/pbj-compiler/src/main/java/com/hedera/pbj/compiler/impl/generators/protobuf/CodecWriteByteArrayMethodGenerator.java b/pbj-core/pbj-compiler/src/main/java/com/hedera/pbj/compiler/impl/generators/protobuf/CodecWriteByteArrayMethodGenerator.java new file mode 100644 index 00000000..b11bd9c2 --- /dev/null +++ b/pbj-core/pbj-compiler/src/main/java/com/hedera/pbj/compiler/impl/generators/protobuf/CodecWriteByteArrayMethodGenerator.java @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: Apache-2.0 +package com.hedera.pbj.compiler.impl.generators.protobuf; + +import static com.hedera.pbj.compiler.impl.Common.DEFAULT_INDENT; + +import com.hedera.pbj.compiler.impl.Common; +import com.hedera.pbj.compiler.impl.Field; +import com.hedera.pbj.compiler.impl.MapField; +import com.hedera.pbj.compiler.impl.OneOfField; +import com.hedera.pbj.compiler.impl.SingleField; +import java.util.Comparator; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Code to generate the write method for Codec classes. + */ +final class CodecWriteByteArrayMethodGenerator { + + static String generateWriteMethod( + final String modelClassName, final String schemaClassName, final List fields) { + final String fieldWriteLines = buildFieldWriteLines( + modelClassName, + schemaClassName, + fields, + field -> " data.%s()".formatted(field.nameCamelFirstLower()), + true); + // spotless:off + return + """ + /** + * Writes an item to the given byte array, this is a performance focused method. In non-performance centric use + * cases there are simpler methods such as toBytes() or writing to a {@link WritableStreamingData}. + * + * @param data The item to write. Must not be null. + * @param output The byte array to write to, this must be large enough to hold the entire item. + * @param startOffset The offset in the output array to start writing at. + * @return The number of bytes written to the output array. + * @throws IndexOutOfBoundsException If the output array is not large enough to hold the entire item. + */ + public int write(@NonNull $modelClass data, @NonNull byte[] output, final int startOffset) { + int offset = startOffset; + $fieldWriteLines + // Write unknown fields if there are any + for (final UnknownField uf : data.getUnknownFields()) { + final int tag = (uf.field() << TAG_FIELD_OFFSET) | uf.wireType().ordinal(); + offset += ProtoArrayWriterTools.writeUnsignedVarInt(output, offset, tag); + offset += uf.bytes().writeTo(output, offset); + } + return offset - startOffset; + } + """ + .replace("$modelClass", modelClassName) + .replace("$fieldWriteLines", fieldWriteLines) + .indent(DEFAULT_INDENT); + // spotless:on + } + + private static String buildFieldWriteLines( + final String modelClassName, + final String schemaClassName, + final List fields, + final Function getValueBuilder, + final boolean skipDefault) { + return fields.stream() + .flatMap(field -> field.type() == Field.FieldType.ONE_OF + ? ((OneOfField) field).fields().stream() + : Stream.of(field)) + .sorted(Comparator.comparingInt(Field::fieldNumber)) + .map(field -> generateFieldWriteLines( + field, modelClassName, schemaClassName, getValueBuilder.apply(field), skipDefault)) + .collect(Collectors.joining("\n")) + .indent(DEFAULT_INDENT); + } + + /** + * Generate lines of code for writing field + * + * @param field The field to generate writing line of code for + * @param modelClassName The model class name for model class for message type we are generating writer for + * @param getValueCode java code to get the value of field + * @param skipDefault skip writing the field if it has default value (for non-oneOf only) + * @return java code to write field to output + */ + private static String generateFieldWriteLines( + final Field field, + final String modelClassName, + final String schemaClassName, + String getValueCode, + boolean skipDefault) { + final String fieldDef = schemaClassName + "." + Common.camelToUpperSnake(field.name()); + String prefix = "// [%d] - %s%n".formatted(field.fieldNumber(), field.name()); + + if (field.parent() != null) { + final OneOfField oneOfField = field.parent(); + final String oneOfType = "%s.%sOneOfType".formatted(modelClassName, oneOfField.nameCamelFirstUpper()); + getValueCode = "(%s)data.%s().as()".formatted(field.javaFieldType(), oneOfField.nameCamelFirstLower()); + prefix += "if (data.%s().kind() == %s.%s)%n" + .formatted(oneOfField.nameCamelFirstLower(), oneOfType, Common.camelToUpperSnake(field.name())); + } + // spotless:off + final String writeMethodName = field.methodNameType(); + if (field.optionalValueType()) { + return prefix + switch (field.messageType()) { + case "StringValue" -> "offset += ProtoArrayWriterTools.writeOptionalString(output, offset, %s, %s);" + .formatted(fieldDef,getValueCode); + case "BoolValue" -> "offset += ProtoArrayWriterTools.writeOptionalBoolean(output, offset, %s, %s);" + .formatted(fieldDef, getValueCode); + case "Int32Value" -> "offset += ProtoArrayWriterTools.writeOptionalInt32Value(output, offset, %s, %s);" + .formatted(fieldDef, getValueCode); + case "UInt32Value" -> "offset += ProtoArrayWriterTools.writeOptionalUInt32Value(output, offset, %s, %s);" + .formatted(fieldDef, getValueCode); + case "Int64Value","UInt64Value" -> "offset += ProtoArrayWriterTools.writeOptionalInt64Value(output, offset, %s, %s);" + .formatted(fieldDef, getValueCode); + case "FloatValue" -> "offset += ProtoArrayWriterTools.writeOptionalFloat(output, offset, %s, %s);" + .formatted(fieldDef, getValueCode); + case "DoubleValue" -> "offset += ProtoArrayWriterTools.writeOptionalDouble(output, offset, %s, %s);" + .formatted(fieldDef, getValueCode); + case "BytesValue" -> "offset += ProtoArrayWriterTools.writeOptionalBytes(output, offset, %s, %s);" + .formatted(fieldDef, getValueCode); + default -> throw new UnsupportedOperationException( + "Unhandled optional message type:%s".formatted(field.messageType())); + }; + } else { + String codecReference = ""; + if (Field.FieldType.MESSAGE.equals(field.type())) { + codecReference = "%s.%s.PROTOBUF".formatted(((SingleField) field).messageTypeModelPackage(), + ((SingleField) field).completeClassName()); + } + if (field.repeated()) { + return prefix + switch(field.type()) { + case ENUM -> "offset += ProtoArrayWriterTools.writeEnumList(output, offset, %s, %s);" + .formatted(fieldDef, getValueCode); + case MESSAGE -> "offset += ProtoArrayWriterTools.writeMessageList(output, offset, %s, %s, %s);" + .formatted(fieldDef, getValueCode, codecReference); + case INT32 -> "offset += ProtoArrayWriterTools.writeInt32List(output, offset, %s, %s);" + .formatted(fieldDef, getValueCode); + case UINT32 -> "offset += ProtoArrayWriterTools.writeUInt32List(output, offset, %s, %s);" + .formatted(fieldDef, getValueCode); + case SINT32 -> "offset += ProtoArrayWriterTools.writeSInt32List(output, offset, %s, %s);" + .formatted(fieldDef, getValueCode); + case FIXED32, SFIXED32 -> "offset += ProtoArrayWriterTools.writeFixed32List(output, offset, %s, %s);" + .formatted(fieldDef, getValueCode); + case INT64, UINT64 -> "offset += ProtoArrayWriterTools.writeInt64List(output, offset, %s, %s);" + .formatted(fieldDef, getValueCode); + case SINT64 -> "offset += ProtoArrayWriterTools.writeSInt64List(output, offset, %s, %s);" + .formatted(fieldDef, getValueCode); + case FIXED64, SFIXED64 -> "offset += ProtoArrayWriterTools.writeFixed64List(output, offset, %s, %s);" + .formatted(fieldDef, getValueCode); + + default -> "offset += ProtoArrayWriterTools.write%sList(output, offset, %s, %s);" + .formatted(writeMethodName, fieldDef, getValueCode); + }; + } else if (field.type() == Field.FieldType.MAP) { + // https://protobuf.dev/programming-guides/proto3/#maps + // On the wire, a map is equivalent to: + // message MapFieldEntry { + // key_type key = 1; + // value_type value = 2; + // } + // repeated MapFieldEntry map_field = N; + // NOTE: we serialize the map in the natural order of keys by design, + // so that the binary representation of the map is deterministic. + // NOTE: protoc serializes default values (e.g. "") in maps, so we should too. + final MapField mapField = (MapField) field; + final List mapEntryFields = List.of(mapField.keyField(), mapField.valueField()); + final Function getValueBuilder = mapEntryField -> + mapEntryField == mapField.keyField() ? "k" : (mapEntryField == mapField.valueField() ? "v" : null); + final String fieldWriteLines = buildFieldWriteLines( + field.name(), + schemaClassName, + mapEntryFields, + getValueBuilder, + false); + final String fieldSizeOfLines = CodecMeasureRecordMethodGenerator.buildFieldSizeOfLines( + field.name(), + mapEntryFields, + getValueBuilder, + false); + return prefix + """ + if (!$map.isEmpty()) { + final Pbj$javaFieldType pbjMap = (Pbj$javaFieldType) $map; + final int mapSize = pbjMap.size(); + for (int i = 0; i < mapSize; i++) { + offset += ProtoArrayWriterTools.writeTag(output, offset, $fieldDef, WIRE_TYPE_DELIMITED); + $K k = pbjMap.getSortedKeys().get(i); + $V v = pbjMap.get(k); + int size = 0; + $fieldSizeOfLines + offset += ProtoArrayWriterTools.writeUnsignedVarInt(output, offset, size); + $fieldWriteLines + } + } + """ + .replace("$fieldDef", fieldDef) + .replace("$map", getValueCode) + .replace("$javaFieldType", mapField.javaFieldType()) + .replace("$K", mapField.keyField().type().boxedType) + .replace("$V", mapField.valueField().type() == Field.FieldType.MESSAGE ? ((SingleField)mapField.valueField()).messageType() : mapField.valueField().type().boxedType) + .replace("$fieldWriteLines", fieldWriteLines.indent(DEFAULT_INDENT)) + .replace("$fieldSizeOfLines", fieldSizeOfLines.indent(DEFAULT_INDENT)); + } else { + return prefix + switch(field.type()) { + case ENUM -> "offset += ProtoArrayWriterTools.writeEnum(output, offset, %s, %s);" + .formatted(fieldDef, getValueCode); + case STRING -> "offset += ProtoArrayWriterTools.writeString(output, offset, %s, %s, %s);" + .formatted(fieldDef, getValueCode, skipDefault); + case MESSAGE -> "offset += ProtoArrayWriterTools.writeMessage(output, offset, %s, %s, %s);" + .formatted(fieldDef, getValueCode, codecReference); + case BOOL -> "offset += ProtoArrayWriterTools.writeBoolean(output, offset, %s, %s, %s);" + .formatted(fieldDef, getValueCode, skipDefault); + case INT32 -> "offset += ProtoArrayWriterTools.writeInt32(output, offset, %s, %s, %s);" + .formatted(fieldDef, getValueCode, skipDefault); + case UINT32 -> "offset += ProtoArrayWriterTools.writeUInt32(output, offset, %s, %s, %s);" + .formatted(fieldDef, getValueCode, skipDefault); + case SINT32 -> "offset += ProtoArrayWriterTools.writSInt32(output, offset, %s, %s, %s);" + .formatted(fieldDef, getValueCode, skipDefault); + case FIXED32, SFIXED32 -> "offset += ProtoArrayWriterTools.writeFixed32(output, offset, %s, %s, %s);" + .formatted(fieldDef, getValueCode, skipDefault); + case INT64, UINT64 -> "offset += ProtoArrayWriterTools.writeInt64(output, offset, %s, %s, %s);" + .formatted(fieldDef, getValueCode, skipDefault); + case SINT64 -> "offset += ProtoArrayWriterTools.writeSInt64(output, offset, %s, %s, %s);" + .formatted(fieldDef, getValueCode, skipDefault); + case FIXED64, SFIXED64 -> "offset += ProtoArrayWriterTools.writeFixed64(output, offset, %s, %s, %s);" + .formatted(fieldDef, getValueCode, skipDefault); + case BYTES -> "offset += ProtoArrayWriterTools.writeBytes(output, offset, %s, %s, %s);" + .formatted(fieldDef, getValueCode, skipDefault); + default -> "offset += ProtoArrayWriterTools.write%s(output, offset, %s, %s);" + .formatted(writeMethodName, fieldDef, getValueCode); + }; + } + } + // spotless:on + } +} diff --git a/pbj-core/pbj-grpc-helidon/src/test/java/com/hedera/pbj/grpc/helidon/PbjTest.java b/pbj-core/pbj-grpc-helidon/src/test/java/com/hedera/pbj/grpc/helidon/PbjTest.java index 9dd48df6..08dfb0b1 100644 --- a/pbj-core/pbj-grpc-helidon/src/test/java/com/hedera/pbj/grpc/helidon/PbjTest.java +++ b/pbj-core/pbj-grpc-helidon/src/test/java/com/hedera/pbj/grpc/helidon/PbjTest.java @@ -696,6 +696,8 @@ private GrpcStatus grpcStatus(Http2ClientResponse response) { try { return grpcStatus(response.headers()); } catch (NoSuchElementException e) { + // We cannot request trailers before requesting an entity, so: + response.entity(); return grpcStatus(response.trailers()); } } diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/Codec.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/Codec.java index 32a5bd45..8eb06c5a 100644 --- a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/Codec.java +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/Codec.java @@ -15,11 +15,7 @@ * * @param The type of object to serialize and deserialize */ -public interface Codec { - - // NOTE: When services has finished migrating to protobuf based objects in state, - // then we should strongly enforce Codec works with Records. This will reduce bugs - // where people try to use a mutable object. +public interface Codec { /** * Parses an object from the {@link ReadableSequentialData} and returns it. @@ -157,6 +153,27 @@ default T parseStrict(@NonNull Bytes bytes) throws ParseException { */ void write(@NonNull T item, @NonNull WritableSequentialData output) throws IOException; + /** + * Writes an item to the given byte array, this is a performance focused method. In non-performance centric use + * cases there are simpler methods such as {@link #toBytes(T)} or writing to a {@link WritableStreamingData}. + * + * @param item The item to write. Must not be null. + * @param output The byte array to write to, this must be large enough to hold the entire item. + * @param startOffset The offset in the output array to start writing at. + * @return The number of bytes written to the output array. + * @throws UncheckedIOException If the there is a problem writing to the output array. + * @throws IndexOutOfBoundsException If the output array is not large enough to hold the entire item. + */ + default int write(@NonNull T item, @NonNull byte[] output, final int startOffset) { + final BufferedData bufferedData = BufferedData.wrap(output, startOffset, output.length - startOffset); + try { + write(item, bufferedData); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return (int) bufferedData.position(); + } + /** * Reads from this data input the length of the data within the input. The implementation may * read all the data, or just some special serialized data, as needed to find out the length of diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/MalformedUtf8Exception.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/MalformedUtf8Exception.java new file mode 100644 index 00000000..f3258753 --- /dev/null +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/MalformedUtf8Exception.java @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +package com.hedera.pbj.runtime; + +/** + * Thrown during the UTF-8 encoding process when it is malformed. + */ +public class MalformedUtf8Exception extends RuntimeException { + + /** + * Construct new MalformedUtf8Exception + * + * @param message error message + */ + public MalformedUtf8Exception(final String message) { + super(message); + } + + public MalformedUtf8Exception(final String message, final Throwable cause) { + super(message, cause); + } +} diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ProtoArrayWriterTools.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ProtoArrayWriterTools.java new file mode 100644 index 00000000..66c1894f --- /dev/null +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ProtoArrayWriterTools.java @@ -0,0 +1,1159 @@ +// SPDX-License-Identifier: Apache-2.0 +package com.hedera.pbj.runtime; + +import static com.hedera.pbj.runtime.FieldType.FIXED32; +import static com.hedera.pbj.runtime.FieldType.FIXED64; +import static com.hedera.pbj.runtime.FieldType.INT32; +import static com.hedera.pbj.runtime.FieldType.INT64; +import static com.hedera.pbj.runtime.FieldType.SFIXED32; +import static com.hedera.pbj.runtime.FieldType.SFIXED64; +import static com.hedera.pbj.runtime.FieldType.SINT32; +import static com.hedera.pbj.runtime.FieldType.SINT64; +import static com.hedera.pbj.runtime.FieldType.UINT32; +import static com.hedera.pbj.runtime.FieldType.UINT64; +import static com.hedera.pbj.runtime.ProtoConstants.WIRE_TYPE_DELIMITED; +import static com.hedera.pbj.runtime.ProtoConstants.WIRE_TYPE_FIXED_32_BIT; +import static com.hedera.pbj.runtime.ProtoConstants.WIRE_TYPE_FIXED_64_BIT; +import static com.hedera.pbj.runtime.ProtoConstants.WIRE_TYPE_VARINT_OR_ZIGZAG; +import static com.hedera.pbj.runtime.ProtoWriterTools.FIXED32_SIZE; +import static com.hedera.pbj.runtime.ProtoWriterTools.FIXED64_SIZE; +import static com.hedera.pbj.runtime.ProtoWriterTools.TAG_TYPE_BITS; +import static com.hedera.pbj.runtime.ProtoWriterTools.sizeOfBoolean; +import static com.hedera.pbj.runtime.ProtoWriterTools.sizeOfBytes; +import static com.hedera.pbj.runtime.ProtoWriterTools.sizeOfDouble; +import static com.hedera.pbj.runtime.ProtoWriterTools.sizeOfFloat; +import static com.hedera.pbj.runtime.ProtoWriterTools.sizeOfString; +import static com.hedera.pbj.runtime.ProtoWriterTools.sizeOfStringNoTag; +import static com.hedera.pbj.runtime.ProtoWriterTools.sizeOfTag; +import static com.hedera.pbj.runtime.ProtoWriterTools.sizeOfUnsignedVarInt32; +import static com.hedera.pbj.runtime.ProtoWriterTools.sizeOfUnsignedVarInt64; +import static com.hedera.pbj.runtime.ProtoWriterTools.sizeOfVarInt32; + +import com.hedera.pbj.runtime.io.buffer.Bytes; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; +import java.nio.ByteOrder; +import java.util.List; + +/** + * Suite of static utility methods to assist in writing protobuf messages into Java byte arrays. Its number one focus + * is performance. + */ +@SuppressWarnings({"DuplicatedCode", "ForLoopReplaceableByForEach"}) +public final class ProtoArrayWriterTools { + /** Table to determine the length of a varint based on the number of leading zeros */ + private static final int[] VAR_INT_LENGTHS = new int[65]; + + static { + for (int i = 0; i <= 64; ++i) VAR_INT_LENGTHS[i] = ((63 - i) / 7); + } + /** VarHandle to write little-endian integers to byte arrays */ + private static final VarHandle INTEGER_LITTLE_ENDIAN = + MethodHandles.byteArrayViewVarHandle(int[].class, ByteOrder.LITTLE_ENDIAN); + /** VarHandle to write little-endian longs to byte arrays */ + private static final VarHandle LONG_LITTLE_ENDIAN = + MethodHandles.byteArrayViewVarHandle(long[].class, ByteOrder.LITTLE_ENDIAN); + + /** + * Write an unsigned varint to the output. + * + * @param output The byte array to write to + * @param offset The offset to start writing at + * @param value The value to write + * @return The number of bytes written + */ + @SuppressWarnings("fallthrough") + public static int writeUnsignedVarInt(@NonNull byte[] output, final int offset, final long value) { + int length = VAR_INT_LENGTHS[Long.numberOfLeadingZeros(value)]; + output[offset + length] = (byte) (value >>> (length * 7)); + switch (length - 1) { + case 8: + output[offset + 8] = (byte) ((value >>> 56) | 0x80); + // Deliberate fallthrough + case 7: + output[offset + 7] = (byte) ((value >>> 49) | 0x80); + // Deliberate fallthrough + case 6: + output[offset + 6] = (byte) ((value >>> 42) | 0x80); + // Deliberate fallthrough + case 5: + output[offset + 5] = (byte) ((value >>> 35) | 0x80); + // Deliberate fallthrough + case 4: + output[offset + 4] = (byte) ((value >>> 28) | 0x80); + // Deliberate fallthrough + case 3: + output[offset + 3] = (byte) ((value >>> 21) | 0x80); + // Deliberate fallthrough + case 2: + output[offset + 2] = (byte) ((value >>> 14) | 0x80); + // Deliberate fallthrough + case 1: + output[offset + 1] = (byte) ((value >>> 7) | 0x80); + // Deliberate fallthrough + case 0: + output[offset] = (byte) (value | 0x80); + } + return length; + } + + /** + * Write a signed zigzag encoded varint to the output. + * + * @param output The byte array to write to + * @param offset The offset to start writing at + * @param value The value to write + * @return The number of bytes written + */ + public static int writeSignedVarInt(@NonNull byte[] output, final int offset, final long value) { + final long zigZag = (value << 1) ^ (value >> 63); + return writeUnsignedVarInt(output, offset, zigZag); + } + + /** + * Write a protobuf tag to the output. + * + * @param offset The offset to start writing at + * @param field The field to include in tag + * @param wireType The field wire type to include in tag + * @return The number of bytes written + */ + public static int writeTag( + @NonNull byte[] output, + final int offset, + @NonNull final FieldDefinition field, + @NonNull final ProtoConstants wireType) { + return writeUnsignedVarInt(output, offset, ((long) field.number() << TAG_TYPE_BITS) | wireType.ordinal()); + } + + /** + * Write a string to data output, assuming the field is non-repeated. + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing, the field must be non-repeated + * @param value the string value to write + * @param skipDefault default value results in no-op for non-oneOf + * @return the number of bytes written + */ + public static int writeString( + @NonNull byte[] output, + final int offset, + @NonNull final FieldDefinition field, + final String value, + final boolean skipDefault) { + assert field.type() == FieldType.STRING : "Not a string type " + field; + assert !field.repeated() : "Use writeStringList with repeated types"; + return writeStringNoChecks(output, offset, field, value, skipDefault); + } + + /** + * Write an integer to data output - no validation checks. + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param value the string value to write + * @param skipDefault default value results in no-op for non-oneOf + * @return the number of bytes written + */ + private static int writeStringNoChecks( + @NonNull byte[] output, + final int offset, + @NonNull final FieldDefinition field, + final String value, + final boolean skipDefault) { + int bytesWritten = 0; + // When not a oneOf don't write default value + if (skipDefault && !field.oneOf() && (value == null || value.isEmpty())) { + return 0; + } + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, sizeOfStringNoTag(value)); + bytesWritten += Utf8Tools.encodeUtf8(output, offset + bytesWritten, value); + return bytesWritten; + } + + /** + * Write an optional string to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the field definition for the string field + * @param value the optional string value to write + * @return the number of bytes written + */ + public static int writeOptionalString( + @NonNull byte[] output, + final int offset, + @NonNull final FieldDefinition field, + @Nullable final String value) { + int bytesWritten = 0; + if (value != null) { + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + final var newField = field.type().optionalFieldDefinition; + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, sizeOfString(newField, value)); + // bytesWritten += writeStringNoChecks(output, offset + bytesWritten, newField, value, true); + + // When not a oneOf don't write default value + if (field.oneOf() || !value.isEmpty()) { + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, sizeOfStringNoTag(value)); + bytesWritten += Utf8Tools.encodeUtf8(output, offset + bytesWritten, value); + } + } + return bytesWritten; + } + + /** + * Write a boolean to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param value the boolean value to write + * @param skipDefault default value results in no-op for non-oneOf + * @return the number of bytes written + */ + public static int writeBoolean( + @NonNull final byte[] output, + final int offset, + @NonNull final FieldDefinition field, + final boolean value, + final boolean skipDefault) { + assert field.type() == FieldType.BOOL : "Not a boolean type " + field; + assert !field.repeated() : "Use writeBooleanList with repeated types"; + int bytesWritten = 0; + // In the case of oneOf we write the value even if it is default value of false + if (value || field.oneOf() || !skipDefault) { + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_VARINT_OR_ZIGZAG); + output[offset + bytesWritten] = value ? (byte) 1 : 0; + bytesWritten++; + } + return bytesWritten; + } + + /** + * Write an optional boolean to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param value the optional boolean value to write + * @return the number of bytes written + */ + public static int writeOptionalBoolean( + @NonNull final byte[] output, final int offset, @NonNull FieldDefinition field, @Nullable Boolean value) { + int bytesWritten = 0; + if (value != null) { + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + final var newField = field.type().optionalFieldDefinition; + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, sizeOfBoolean(newField, value)); + bytesWritten += writeBoolean(output, offset + bytesWritten, newField, value, true); + } + return bytesWritten; + } + + /** + * Write an Int32 to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param value the int value to write + * @param skipDefault default value results in no-op for non-oneOf + * @return the number of bytes written + */ + public static int writeInt32( + @NonNull final byte[] output, + final int offset, + @NonNull final FieldDefinition field, + final int value, + boolean skipDefault) { + assert field.type() == INT32 : "Not an Int32 type " + field; + assert !field.repeated() : "Use writeIntegerList with repeated types"; + if (skipDefault && !field.oneOf() && value == 0) return 0; + int bytesWritten = 0; + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_VARINT_OR_ZIGZAG); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, value); + return bytesWritten; + } + + /** + * Write an UInt32 to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param value the int value to write + * @param skipDefault default value results in no-op for non-oneOf + * @return the number of bytes written + */ + public static int writeUInt32( + @NonNull final byte[] output, + final int offset, + @NonNull final FieldDefinition field, + final int value, + boolean skipDefault) { + assert field.type() == UINT32 : "Not a UInt32 type " + field; + assert !field.repeated() : "Use writeIntegerList with repeated types"; + if (skipDefault && !field.oneOf() && value == 0) return 0; + int bytesWritten = 0; + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_VARINT_OR_ZIGZAG); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, Integer.toUnsignedLong(value)); + return bytesWritten; + } + + /** + * Write an SInt32 to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param value the int value to write + * @param skipDefault default value results in no-op for non-oneOf + * @return the number of bytes written + */ + public static int writSInt32( + @NonNull final byte[] output, + final int offset, + @NonNull final FieldDefinition field, + final int value, + boolean skipDefault) { + assert field.type() == SINT32 : "Not a SInt32 type " + field; + assert !field.repeated() : "Use writeIntegerList with repeated types"; + if (skipDefault && !field.oneOf() && value == 0) return 0; + int bytesWritten = 0; + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_VARINT_OR_ZIGZAG); + bytesWritten += writeSignedVarInt(output, offset + bytesWritten, value); + return bytesWritten; + } + + /** + * Write a SFixed32 or Fixed32 to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param value the int value to write + * @param skipDefault default value results in no-op for non-oneOf + * @return the number of bytes written + */ + public static int writeFixed32( + @NonNull final byte[] output, + final int offset, + @NonNull final FieldDefinition field, + final int value, + boolean skipDefault) { + assert field.type() == FieldType.FIXED32 || field.type() == FieldType.SFIXED32 + : "Not a Fixed32 or SFixed32 type " + field; + assert !field.repeated() : "Use writeIntegerList with repeated types"; + if (skipDefault && !field.oneOf() && value == 0) return 0; + int bytesWritten = 0; + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_FIXED_32_BIT); + INTEGER_LITTLE_ENDIAN.set(output, offset + bytesWritten, value); + bytesWritten += Integer.BYTES; + return bytesWritten; + } + + /** + * Write an optional signed integer to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param value the optional integer value to write + * @return the number of bytes written + */ + public static int writeOptionalInt32Value( + @NonNull final byte[] output, final int offset, FieldDefinition field, @Nullable Integer value) { + int bytesWritten = 0; + if (value != null) { + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + final var newField = field.type().optionalFieldDefinition; + bytesWritten += writeUnsignedVarInt( + output, + offset + bytesWritten, + sizeOfTag(field, WIRE_TYPE_VARINT_OR_ZIGZAG) + sizeOfVarInt32(value)); + bytesWritten += writeInt32(output, offset + bytesWritten, newField, value, true); + } + return bytesWritten; + } + + /** + * Write an optional unsigned integer to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param value the optional integer value to write + * @return the number of bytes written + */ + public static int writeOptionalUInt32Value( + @NonNull final byte[] output, final int offset, FieldDefinition field, @Nullable Integer value) { + int bytesWritten = 0; + if (value != null) { + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + final var newField = field.type().optionalFieldDefinition; + bytesWritten += writeUnsignedVarInt( + output, + offset + bytesWritten, + sizeOfTag(field, WIRE_TYPE_VARINT_OR_ZIGZAG) + sizeOfVarInt32(value)); + bytesWritten += writeUInt32(output, offset + bytesWritten, newField, value, true); + } + return bytesWritten; + } + + /** + * Write an Int64 or UInt64 to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param value the int value to write + * @param skipDefault default value results in no-op for non-oneOf + * @return the number of bytes written + */ + public static int writeInt64( + @NonNull final byte[] output, + final int offset, + @NonNull final FieldDefinition field, + final long value, + boolean skipDefault) { + assert field.type() == INT64 || field.type() == UINT64 : "Not an Int64 or UInt64 type " + field; + assert !field.repeated() : "Use writeLongList with repeated types"; + if (skipDefault && !field.oneOf() && value == 0L) return 0; + int bytesWritten = 0; + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_VARINT_OR_ZIGZAG); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, value); + return bytesWritten; + } + + /** + * Write an SInt64 to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param value the int value to write + * @param skipDefault default value results in no-op for non-oneOf + * @return the number of bytes written + */ + public static int writeSInt64( + @NonNull final byte[] output, + final int offset, + @NonNull final FieldDefinition field, + final long value, + boolean skipDefault) { + assert field.type() == SINT64 : "Not a SInt64 type " + field; + assert !field.repeated() : "Use writeLongList with repeated types"; + if (skipDefault && !field.oneOf() && value == 0L) return 0; + int bytesWritten = 0; + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_VARINT_OR_ZIGZAG); + bytesWritten += writeSignedVarInt(output, offset + bytesWritten, value); + return bytesWritten; + } + + /** + * Write a SFixed64 or Fixed64 to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param value the int value to write + * @param skipDefault default value results in no-op for non-oneOf + * @return the number of bytes written + */ + public static int writeFixed64( + @NonNull final byte[] output, + final int offset, + @NonNull final FieldDefinition field, + final long value, + boolean skipDefault) { + assert field.type() == FIXED64 || field.type() == SFIXED64 : "Not a Fixed64 or SFixed64 type " + field; + assert !field.repeated() : "Use writeLongList with repeated types"; + if (skipDefault && !field.oneOf() && value == 0L) return 0; + int bytesWritten = 0; + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_FIXED_32_BIT); + LONG_LITTLE_ENDIAN.set(output, offset + bytesWritten, value); + bytesWritten += Long.BYTES; + return bytesWritten; + } + + /** + * Write an optional signed or unsinged long to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param value the optional long value to write + * @return the number of bytes written + */ + public static int writeOptionalInt64Value( + @NonNull final byte[] output, final int offset, FieldDefinition field, @Nullable Long value) { + int bytesWritten = 0; + if (value != null) { + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + final var newField = field.type().optionalFieldDefinition; + bytesWritten += writeUnsignedVarInt( + output, + offset + bytesWritten, + sizeOfTag(field, WIRE_TYPE_VARINT_OR_ZIGZAG) + sizeOfUnsignedVarInt64(value)); + bytesWritten += writeInt64(output, offset + bytesWritten, newField, value, true); + } + return bytesWritten; + } + + /** + * Write a float to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param value the float value to write + * @return the number of bytes written + */ + public static int writeFloat(@NonNull final byte[] output, final int offset, FieldDefinition field, float value) { + assert field.type() == FieldType.FLOAT : "Not a float type " + field; + assert !field.repeated() : "Use writeFloatList with repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && value == 0) { + return 0; + } + int bytesWritten = 0; + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_FIXED_32_BIT); + INTEGER_LITTLE_ENDIAN.set(output, offset + bytesWritten, Float.floatToIntBits(value)); + bytesWritten += Integer.BYTES; + return bytesWritten; + } + + /** + * Write a double to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param value the double value to write + * @return the number of bytes written + */ + public static int writeDouble(@NonNull final byte[] output, final int offset, FieldDefinition field, double value) { + assert field.type() == FieldType.DOUBLE : "Not a double type " + field; + assert !field.repeated() : "Use writeDoubleList with repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && value == 0) { + return 0; + } + int bytesWritten = 0; + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_FIXED_64_BIT); + LONG_LITTLE_ENDIAN.set(output, offset + bytesWritten, Double.doubleToLongBits(value)); + bytesWritten += Long.BYTES; + return bytesWritten; + } + + /** + * Write an optional float to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param value the optional long value to write + * @return the number of bytes written + */ + public static int writeOptionalFloat( + @NonNull final byte[] output, final int offset, FieldDefinition field, @Nullable Float value) { + int bytesWritten = 0; + if (value != null) { + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + final var newField = field.type().optionalFieldDefinition; + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, sizeOfFloat(newField, value)); + bytesWritten += writeFixed32(output, offset + bytesWritten, newField, Float.floatToIntBits(value), true); + } + return bytesWritten; + } + + /** + * Write an optional double to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param value the optional long value to write + * @return the number of bytes written + */ + public static int writeOptionalDouble( + @NonNull final byte[] output, final int offset, FieldDefinition field, @Nullable Double value) { + int bytesWritten = 0; + if (value != null) { + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + final var newField = field.type().optionalFieldDefinition; + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, sizeOfDouble(newField, value)); + bytesWritten += writeFixed64(output, offset + bytesWritten, newField, Double.doubleToLongBits(value), true); + } + return bytesWritten; + } + + /** + * Write an optional bytes to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param value the optional long value to write + * @return the number of bytes written + */ + public static int writeOptionalBytes( + @NonNull final byte[] output, final int offset, FieldDefinition field, @Nullable Bytes value) { + int bytesWritten = 0; + if (value != null) { + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + final var newField = field.type().optionalFieldDefinition; + final int size = sizeOfBytes(newField, value); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, size); + if (size > 0) { + bytesWritten += writeTag(output, offset + bytesWritten, field, WIRE_TYPE_DELIMITED); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, size); + bytesWritten += value.writeTo(output, offset + bytesWritten); + } + } + return bytesWritten; + } + + /** + * Write an enum to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param enumValue the enum value to write + * @return the number of bytes written + */ + public static int writeEnum( + @NonNull final byte[] output, + final int offset, + @NonNull final FieldDefinition field, + @Nullable final EnumWithProtoMetadata enumValue) { + assert field.type() == FieldType.ENUM : "Not an enum type " + field; + assert !field.repeated() : "Use writeEnumList with repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && (enumValue == null || enumValue.protoOrdinal() == 0)) { + return 0; + } + int bytesWritten = 0; + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_VARINT_OR_ZIGZAG); + bytesWritten += writeUnsignedVarInt(output, offset, enumValue == null ? 0 : enumValue.protoOrdinal()); + return bytesWritten; + } + + /** + * Write a message to data output, assuming the corresponding field is non-repeated. + * + * @param type of message + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing, the field must not be repeated + * @param message the message to write + * @param codec the codec for the given message type + * @return the number of bytes written + */ + public static int writeMessage( + @NonNull final byte[] output, + final int offset, + @NonNull final FieldDefinition field, + final T message, + final Codec codec) { + assert field.type() == FieldType.MESSAGE : "Not a message type " + field; + assert !field.repeated() : "Use writeMessageList with repeated types"; + return writeMessageNoChecks(output, offset, field, message, codec); + } + + /** + * Write a message to data output - no validation checks. + * + * @param type of message + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param message the message to write + * @param codec the codec for the given message type + * @return the number of bytes written + */ + private static int writeMessageNoChecks( + @NonNull final byte[] output, + final int offset, + @NonNull final FieldDefinition field, + final T message, + final Codec codec) { + // When not a oneOf don't write default value + int bytesWritten = 0; + if (field.oneOf() && message == null) { + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, 0); + } else if (message != null) { + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + final int size = codec.measureRecord(message); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, size); + if (size > 0) { + bytesWritten += codec.write(message, output, offset + bytesWritten); + } + } + return bytesWritten; + } + + /** + * Write a bytes to data output, assuming the corresponding field is non-repeated, and field type + * is any delimited: bytes, string, or message. + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing, the field must not be repeated + * @param value the bytes value to write + * @param skipDefault default value results in no-op for non-oneOf + * @return the number of bytes written + */ + public static int writeBytes( + @NonNull final byte[] output, + final int offset, + @NonNull final FieldDefinition field, + final Bytes value, + boolean skipDefault) { + assert field.type() == FieldType.BYTES : "Not a byte[] type " + field; + assert !field.repeated() : "Use writeBytesList with repeated types"; + return writeBytesNoChecks(output, offset, field, value, skipDefault); + } + + /** + * Write a bytes to data output - no validation checks. + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param value the bytes value to write + * @param skipZeroLength this is true for normal single bytes and false for repeated lists + * @return the number of bytes written + */ + private static int writeBytesNoChecks( + @NonNull final byte[] output, + final int offset, + @NonNull final FieldDefinition field, + final Bytes value, + final boolean skipZeroLength) { + // When not a oneOf don't write default value + if (!field.oneOf() && (skipZeroLength && (value.length() == 0))) { + return 0; + } + int bytesWritten = 0; + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, value.length()); + bytesWritten += value.writeTo(output, offset + bytesWritten); + return bytesWritten; + } + + // ================================================================================================================ + // LIST VERSIONS OF WRITE METHODS + + /** + * Write a list of integers to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param list the list of integers value to write + * @return the number of bytes written + */ + public static int writeInt32List( + @NonNull final byte[] output, final int offset, FieldDefinition field, List list) { + assert field.type() != INT32 : "Not a long type " + field; + assert field.repeated() : "Use writeInteger with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) return 0; + final int listSize = list.size(); + int size = 0; + for (int i = 0; i < listSize; i++) { + final int val = list.get(i); + size += sizeOfVarInt32(val); + } + int bytesWritten = 0; + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, size); + for (int i = 0; i < listSize; i++) { + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, list.get(i)); + } + return bytesWritten; + } + + /** + * Write a list of integers to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param list the list of integers value to write + * @return the number of bytes written + */ + public static int writeUInt32List( + @NonNull final byte[] output, final int offset, FieldDefinition field, List list) { + assert field.type() != UINT32 : "Not a long type " + field; + assert field.repeated() : "Use writeInteger with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) return 0; + final int listSize = list.size(); + int size = 0; + for (int i = 0; i < listSize; i++) { + final int val = list.get(i); + size += sizeOfUnsignedVarInt64(Integer.toUnsignedLong(val)); + } + int bytesWritten = 0; + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, size); + for (int i = 0; i < listSize; i++) { + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, Integer.toUnsignedLong(list.get(i))); + } + return bytesWritten; + } + + /** + * Write a list of integers to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param list the list of integers value to write + * @return the number of bytes written + */ + public static int writeSInt32List( + @NonNull final byte[] output, final int offset, FieldDefinition field, List list) { + assert field.type() != SINT32 : "Not a long type " + field; + assert field.repeated() : "Use writeInteger with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) return 0; + final int listSize = list.size(); + int size = 0; + for (int i = 0; i < listSize; i++) { + final int val = list.get(i); + size += sizeOfUnsignedVarInt64(((long) val << 1) ^ ((long) val >> 63)); + } + int bytesWritten = 0; + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, size); + for (int i = 0; i < listSize; i++) { + bytesWritten += writeSignedVarInt(output, offset + bytesWritten, Integer.toUnsignedLong(list.get(i))); + } + return bytesWritten; + } + + /** + * Write a list of integers to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param list the list of integers value to write + * @return the number of bytes written + */ + public static int writeFixed32List( + @NonNull final byte[] output, final int offset, FieldDefinition field, List list) { + assert field.type() != FIXED32 && field.type() != SFIXED32 : "Not a long type " + field; + assert field.repeated() : "Use writeInteger with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) return 0; + final int listSize = list.size(); + int bytesWritten = 0; + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, (long) list.size() * FIXED32_SIZE); + for (int i = 0; i < listSize; i++) { + INTEGER_LITTLE_ENDIAN.set(output, offset + bytesWritten, list.get(i)); + bytesWritten += Integer.BYTES; + } + return bytesWritten; + } + + /** + * Write a list of longs to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param list the list of longs value to write + * @return the number of bytes written + */ + public static int writeInt64List( + @NonNull final byte[] output, final int offset, FieldDefinition field, List list) { + assert field.type() != INT64 && field.type() != UINT64 : "Not a long type " + field; + assert field.repeated() : "Use writeLong with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) return 0; + final int listSize = list.size(); + int size = 0; + for (int i = 0; i < listSize; i++) { + final long val = list.get(i); + size += sizeOfUnsignedVarInt64(val); + } + int bytesWritten = 0; + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, size); + for (int i = 0; i < listSize; i++) { + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, list.get(i)); + } + return bytesWritten; + } + + /** + * Write a list of longs to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param list the list of longs value to write + * @return the number of bytes written + */ + public static int writeSInt64List( + @NonNull final byte[] output, final int offset, FieldDefinition field, List list) { + assert field.type() != SINT64 : "Not a SINT64 type " + field; + assert field.repeated() : "Use writeLong with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) return 0; + final int listSize = list.size(); + int size = 0; + for (int i = 0; i < listSize; i++) { + final long val = list.get(i); + size += sizeOfUnsignedVarInt64(val); + } + int bytesWritten = 0; + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, size); + for (int i = 0; i < listSize; i++) { + bytesWritten += writeSignedVarInt(output, offset + bytesWritten, list.get(i)); + } + return bytesWritten; + } + + /** + * Write a list of longs to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param list the list of longs value to write + * @return the number of bytes written + */ + public static int writeFixed64List( + @NonNull final byte[] output, final int offset, FieldDefinition field, List list) { + assert field.type() != FIXED64 && field.type() != SFIXED64 : "Not a fixed long type " + field; + assert field.repeated() : "Use writeLong with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) return 0; + final int listSize = list.size(); + int bytesWritten = 0; + bytesWritten += writeTag(output, offset, field, WIRE_TYPE_DELIMITED); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, (long) list.size() * FIXED64_SIZE); + for (int i = 0; i < listSize; i++) { + LONG_LITTLE_ENDIAN.set(output, offset + bytesWritten, list.get(i)); + bytesWritten += Long.BYTES; + } + return bytesWritten; + } + + /** + * Write a list of floats to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param list the list of floats value to write + * @return the number of bytes written + */ + public static int writeFloatList( + @NonNull final byte[] output, final int offset, FieldDefinition field, List list) { + assert field.type() == FieldType.FLOAT : "Not a float type " + field; + assert field.repeated() : "Use writeFloat with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) { + return 0; + } + final int size = list.size() * FIXED32_SIZE; + int bytesWritten = 0; + bytesWritten += writeTag(output, offset + bytesWritten, field, WIRE_TYPE_DELIMITED); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, size); + final int listSize = list.size(); + for (int i = 0; i < listSize; i++) { + INTEGER_LITTLE_ENDIAN.set(output, offset + bytesWritten, Float.floatToRawIntBits(list.get(i))); + bytesWritten += Integer.BYTES; + } + return bytesWritten; + } + + /** + * Write a list of doubles to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param list the list of doubles value to write + * @return the number of bytes written + */ + public static int writeDoubleList( + @NonNull final byte[] output, final int offset, FieldDefinition field, List list) { + assert field.type() == FieldType.DOUBLE : "Not a double type " + field; + assert field.repeated() : "Use writeDouble with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) { + return 0; + } + final int size = list.size() * FIXED64_SIZE; + int bytesWritten = 0; + bytesWritten += writeTag(output, offset + bytesWritten, field, WIRE_TYPE_DELIMITED); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, size); + final int listSize = list.size(); + for (int i = 0; i < listSize; i++) { + LONG_LITTLE_ENDIAN.set(output, offset + bytesWritten, Double.doubleToLongBits(list.get(i))); + bytesWritten += Long.BYTES; + } + return bytesWritten; + } + + /** + * Write a list of booleans to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param list the list of booleans value to write + * @return the number of bytes written + */ + public static int writeBooleanList( + @NonNull final byte[] output, final int offset, FieldDefinition field, List list) { + assert field.type() == FieldType.BOOL : "Not a boolean type " + field; + assert field.repeated() : "Use writeBoolean with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) { + return 0; + } + // write + int bytesWritten = 0; + bytesWritten += writeTag(output, offset + bytesWritten, field, WIRE_TYPE_DELIMITED); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, list.size()); + final int listSize = list.size(); + for (int i = 0; i < listSize; i++) { + final boolean b = list.get(i); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, b ? 1 : 0); + } + return bytesWritten; + } + + /** + * Write a list of enums to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param list the list of enums value to write + * @return the number of bytes written + */ + public static int writeEnumList( + @NonNull final byte[] output, + final int offset, + FieldDefinition field, + List list) { + assert field.type() == FieldType.ENUM : "Not an enum type " + field; + assert field.repeated() : "Use writeEnum with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) { + return 0; + } + final int listSize = list.size(); + int size = 0; + for (int i = 0; i < listSize; i++) { + size += sizeOfUnsignedVarInt32(list.get(i).protoOrdinal()); + } + int bytesWritten = 0; + bytesWritten += writeTag(output, offset + bytesWritten, field, WIRE_TYPE_DELIMITED); + bytesWritten += writeUnsignedVarInt(output, offset + bytesWritten, size); + for (int i = 0; i < listSize; i++) { + bytesWritten += writeUnsignedVarInt( + output, offset + bytesWritten, list.get(i).protoOrdinal()); + } + return bytesWritten; + } + + /** + * Write a list of strings to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param list the list of strings value to write + * @return the number of bytes written + */ + public static int writeStringList( + @NonNull final byte[] output, final int offset, FieldDefinition field, List list) { + assert field.type() == FieldType.STRING : "Not a string type " + field; + assert field.repeated() : "Use writeString with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) { + return 0; + } + int curOffset = offset; + final int listSize = list.size(); + for (int i = 0; i < listSize; i++) { + final String value = list.get(i); + curOffset += writeTag(output, curOffset, field, WIRE_TYPE_DELIMITED); + curOffset += writeUnsignedVarInt(output, curOffset, sizeOfStringNoTag(value)); + curOffset += Utf8Tools.encodeUtf8(output, curOffset, value); + } + return curOffset - offset; + } + + /** + * Write a list of messages to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param list the list of messages value to write + * @param codec the codec for the message type + * @return the number of bytes written + * @param type of message + */ + public static int writeMessageList( + @NonNull final byte[] output, final int offset, FieldDefinition field, List list, Codec codec) { + assert field.type() == FieldType.MESSAGE : "Not a message type " + field; + assert field.repeated() : "Use writeMessage with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) { + return 0; + } + final int listSize = list.size(); + int bytesWritten = 0; + for (int i = 0; i < listSize; i++) { + bytesWritten += writeMessageNoChecks(output, offset + bytesWritten, field, list.get(i), codec); + } + return bytesWritten; + } + + /** + * Write a list of bytes objects to data output + * + * @param output the byte array to write to + * @param offset the offset to start writing at + * @param field the descriptor for the field we are writing + * @param list the list of bytes objects value to write + * @return the number of bytes written + */ + public static int writeBytesList( + @NonNull final byte[] output, final int offset, FieldDefinition field, List list) { + assert field.type() == FieldType.BYTES : "Not a message type " + field; + assert field.repeated() : "Use writeBytes with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) { + return 0; + } + final int listSize = list.size(); + int bytesWritten = 0; + for (int i = 0; i < listSize; i++) { + bytesWritten += writeBytesNoChecks(output, offset + bytesWritten, field, list.get(i), false); + } + return bytesWritten; + } +} diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ProtoWriterTools.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ProtoWriterTools.java index 8a0152dd..c0f049b1 100644 --- a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ProtoWriterTools.java +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ProtoWriterTools.java @@ -980,10 +980,10 @@ public static void writeDelimited( // SIZE OF METHODS /** Size of a fixed length 32 bit value in bytes */ - private static final int FIXED32_SIZE = 4; + static final int FIXED32_SIZE = 4; /** Size of a fixed length 64 bit value in bytes */ - private static final int FIXED64_SIZE = 8; + static final int FIXED64_SIZE = 8; /** Size of a max length varint value in bytes */ private static final int MAX_VARINT_SIZE = 10; @@ -1038,7 +1038,7 @@ public static int sizeOfUnsignedVarInt32(final int value) { * @param value The int value to get encoded size for * @return the number of bytes for encoded value */ - private static int sizeOfUnsignedVarInt64(long value) { + static int sizeOfUnsignedVarInt64(long value) { // handle two popular special cases up front ... if ((value & (~0L << 7)) == 0L) return 1; if (value < 0L) return 10; @@ -1211,8 +1211,9 @@ public static int sizeOfInteger(FieldDefinition field, int value, boolean skipDe return switch (field.type()) { case INT32 -> sizeOfTag(field, WIRE_TYPE_VARINT_OR_ZIGZAG) + sizeOfVarInt32(value); case UINT32 -> sizeOfTag(field, WIRE_TYPE_VARINT_OR_ZIGZAG) + sizeOfUnsignedVarInt32(value); - case SINT32 -> sizeOfTag(field, WIRE_TYPE_VARINT_OR_ZIGZAG) - + sizeOfUnsignedVarInt64(((long) value << 1) ^ ((long) value >> 63)); + case SINT32 -> + sizeOfTag(field, WIRE_TYPE_VARINT_OR_ZIGZAG) + + sizeOfUnsignedVarInt64(((long) value << 1) ^ ((long) value >> 63)); case SFIXED32, FIXED32 -> sizeOfTag(field, WIRE_TYPE_VARINT_OR_ZIGZAG) + FIXED32_SIZE; default -> throw unsupported(); }; @@ -1241,8 +1242,8 @@ public static int sizeOfLong(FieldDefinition field, long value, boolean skipDefa if (skipDefault && !field.oneOf() && value == 0) return 0; return switch (field.type()) { case INT64, UINT64 -> sizeOfTag(field, WIRE_TYPE_VARINT_OR_ZIGZAG) + sizeOfUnsignedVarInt64(value); - case SINT64 -> sizeOfTag(field, WIRE_TYPE_VARINT_OR_ZIGZAG) - + sizeOfUnsignedVarInt64((value << 1) ^ (value >> 63)); + case SINT64 -> + sizeOfTag(field, WIRE_TYPE_VARINT_OR_ZIGZAG) + sizeOfUnsignedVarInt64((value << 1) ^ (value >> 63)); case SFIXED64, FIXED64 -> sizeOfTag(field, WIRE_TYPE_VARINT_OR_ZIGZAG) + FIXED64_SIZE; default -> throw unsupported(); }; @@ -1342,7 +1343,7 @@ public static int sizeOfString(FieldDefinition field, String value, boolean skip * @param value string value to get encoded size for * @return the number of bytes for encoded value */ - private static int sizeOfStringNoTag(String value) { + static int sizeOfStringNoTag(String value) { // When not a oneOf don't write default value if ((value == null || value.isEmpty())) { return 0; diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/Utf8Tools.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/Utf8Tools.java index 5461c0b2..c757c892 100644 --- a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/Utf8Tools.java +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/Utf8Tools.java @@ -4,6 +4,7 @@ import static java.lang.Character.*; import com.hedera.pbj.runtime.io.WritableSequentialData; +import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; /** @@ -113,4 +114,56 @@ static void encodeUtf8(final CharSequence in, final WritableSequentialData out) } } } + + /** + * Encodes the input character sequence to a byte array using the same algorithm as protoc, so we are byte for + * byte the same. Returns the number of bytes written. + * + * @param out The byte array to write to + * @param offset The offset in the byte array to start writing at + * @param in The input character sequence to encode + * @return The number of bytes written + * @throws MalformedUtf8Exception if the input contains unpaired surrogates + */ + public static int encodeUtf8(@NonNull final byte[] out, final int offset, final String in) { + int utf16Length = in.length(); + int i = 0; + int j = offset; + // Designed to take advantage of + // https://wiki.openjdk.org/display/HotSpot/RangeCheckElimination + for (char c; i < utf16Length && (c = in.charAt(i)) < 0x80; i++) { + out[j + i] = (byte) c; + } + if (i == utf16Length) { + return j + utf16Length - offset; + } + j += i; + for (char c; i < utf16Length; i++) { + c = in.charAt(i); + if (c < 0x80) { + out[j++] = (byte) c; + } else if (c < 0x800) { // 11 bits, two UTF-8 bytes + out[j++] = (byte) ((0xF << 6) | (c >>> 6)); + out[j++] = (byte) (0x80 | (0x3F & c)); + } else if ((c < Character.MIN_SURROGATE || Character.MAX_SURROGATE < c)) { + // Maximum single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes + out[j++] = (byte) ((0xF << 5) | (c >>> 12)); + out[j++] = (byte) (0x80 | (0x3F & (c >>> 6))); + out[j++] = (byte) (0x80 | (0x3F & c)); + } else { + // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, + // four UTF-8 bytes + final char low; + if (i + 1 == in.length() || !Character.isSurrogatePair(c, (low = in.charAt(++i)))) { + throw new MalformedUtf8Exception("Unpaired surrogate at index " + (i - 1) + " of " + utf16Length); + } + int codePoint = Character.toCodePoint(c, low); + out[j++] = (byte) ((0xF << 4) | (codePoint >>> 18)); + out[j++] = (byte) (0x80 | (0x3F & (codePoint >>> 12))); + out[j++] = (byte) (0x80 | (0x3F & (codePoint >>> 6))); + out[j++] = (byte) (0x80 | (0x3F & codePoint)); + } + } + return j - offset; + } } diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/buffer/Bytes.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/buffer/Bytes.java index bc248ccc..e1b4fdd9 100644 --- a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/buffer/Bytes.java +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/buffer/Bytes.java @@ -309,6 +309,18 @@ public void writeTo(@NonNull final ByteBuffer dstBuffer, final int offset, final dstBuffer.put(buffer, Math.toIntExact(start + offset), length); } + /** + * A helper method for efficient copy of our data into a byte array. + * + * @param dst the byte array to copy into + * @param dstOffset the offset into the destination array to start copying into + * @return the number of bytes copied + */ + public int writeTo(@NonNull final byte[] dst, final int dstOffset) { + System.arraycopy(buffer, start, dst, dstOffset, length); + return length; + } + /** * {@inheritDoc} */ diff --git a/pbj-integration-tests/src/jmh/java/com/hedera/pbj/integration/jmh/ProtobufObjectBench.java b/pbj-integration-tests/src/jmh/java/com/hedera/pbj/integration/jmh/ProtobufObjectBench.java index fed0e1a1..e0cf43b5 100644 --- a/pbj-integration-tests/src/jmh/java/com/hedera/pbj/integration/jmh/ProtobufObjectBench.java +++ b/pbj-integration-tests/src/jmh/java/com/hedera/pbj/integration/jmh/ProtobufObjectBench.java @@ -67,6 +67,7 @@ public static class BenchmarkState { private BufferedData outDataBufferDirect; private ByteBuffer bbout; private ByteBuffer bboutDirect; + private byte[] outArray; public void configure( P pbjModelObject, @@ -100,6 +101,7 @@ public void configure( // output buffers this.bout = new NonSynchronizedByteArrayOutputStream(); WritableStreamingData dout = new WritableStreamingData(this.bout); + this.outArray = new byte[this.protobuf.length * 2]; // make sure big enough this.outDataBuffer = BufferedData.allocate(this.protobuf.length); this.outDataBufferDirect = BufferedData.allocateOffHeap(this.protobuf.length); this.bbout = ByteBuffer.allocate(this.protobuf.length); @@ -191,9 +193,8 @@ public void parseProtoCInputStream(BenchmarkState benchmarkState, Blackhol @OperationsPerInvocation(OPERATION_COUNT) public void writePbjByteArray(BenchmarkState benchmarkState, Blackhole blackhole) throws IOException { for (int i = 0; i < OPERATION_COUNT; i++) { - benchmarkState.outDataBuffer.reset(); - benchmarkState.pbjCodec.write(benchmarkState.pbjModelObject, benchmarkState.outDataBuffer); - blackhole.consume(benchmarkState.outDataBuffer); + benchmarkState.pbjCodec.write(benchmarkState.pbjModelObject, benchmarkState.outArray, 0); + blackhole.consume(benchmarkState.outArray); } }