diff --git a/.gitmodules b/.gitmodules index 6947977ff2..c8e3c1dbcb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1404,6 +1404,9 @@ [submodule "vendor/grammars/vscode-fastly-vcl"] path = vendor/grammars/vscode-fastly-vcl url = https://github.com/fastly/vscode-fastly-vcl.git +[submodule "vendor/grammars/vscode-flatbuffers"] + path = vendor/grammars/vscode-flatbuffers + url = https://github.com/floxay/vscode-flatbuffers.git [submodule "vendor/grammars/vscode-fluent"] path = vendor/grammars/vscode-fluent url = https://github.com/macabeus/vscode-fluent diff --git a/grammars.yml b/grammars.yml index dba0be30d3..154600b865 100644 --- a/grammars.yml +++ b/grammars.yml @@ -1249,6 +1249,8 @@ vendor/grammars/vscode-euphoria: - source.euphoria vendor/grammars/vscode-fastly-vcl: - source.vcl +vendor/grammars/vscode-flatbuffers: +- source.flatbuffers vendor/grammars/vscode-fluent: - source.ftl vendor/grammars/vscode-gcode-syntax: diff --git a/lib/linguist/languages.yml b/lib/linguist/languages.yml index 108105d56e..c49e557a99 100644 --- a/lib/linguist/languages.yml +++ b/lib/linguist/languages.yml @@ -2224,6 +2224,14 @@ Filterscript: tm_scope: none ace_mode: text language_id: 112 +FlatBuffers: + type: data + color: "#ed284a" + extensions: + - ".fbs" + tm_scope: source.flatbuffers + ace_mode: text + language_id: 577640576 Flix: type: programming color: "#d44a45" diff --git a/samples/FlatBuffers/Message.fbs b/samples/FlatBuffers/Message.fbs new file mode 100644 index 0000000000..6361a38245 --- /dev/null +++ b/samples/FlatBuffers/Message.fbs @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +include "Schema.fbs"; +include "SparseTensor.fbs"; +include "Tensor.fbs"; + +namespace org.apache.arrow.flatbuf; + +/// ---------------------------------------------------------------------- +/// Data structures for describing a table row batch (a collection of +/// equal-length Arrow arrays) + +/// Metadata about a field at some level of a nested type tree (but not +/// its children). +/// +/// For example, a List with values `[[1, 2, 3], null, [4], [5, 6], null]` +/// would have {length: 5, null_count: 2} for its List node, and {length: 6, +/// null_count: 0} for its Int16 node, as separate FieldNode structs +struct FieldNode { + /// The number of value slots in the Arrow array at this level of a nested + /// tree + length: long; + + /// The number of observed nulls. Fields with null_count == 0 may choose not + /// to write their physical validity bitmap out as a materialized buffer, + /// instead setting the length of the bitmap buffer to 0. + null_count: long; +} + +enum CompressionType: byte { + // LZ4 frame format, for portability, as provided by lz4frame.h or wrappers + // thereof. Not to be confused with "raw" (also called "block") format + // provided by lz4.h + LZ4_FRAME, + + // Zstandard + ZSTD +} + +/// Provided for forward compatibility in case we need to support different +/// strategies for compressing the IPC message body (like whole-body +/// compression rather than buffer-level) in the future +enum BodyCompressionMethod: byte { + /// Each constituent buffer is first compressed with the indicated + /// compressor, and then written with the uncompressed length in the first 8 + /// bytes as a 64-bit little-endian signed integer followed by the compressed + /// buffer bytes (and then padding as required by the protocol). The + /// uncompressed length may be set to -1 to indicate that the data that + /// follows is not compressed, which can be useful for cases where + /// compression does not yield appreciable savings. + /// Also, empty buffers can optionally be written out as 0-byte compressed + /// buffers, thereby omitting the 8-bytes length header. + BUFFER +} + +/// Optional compression for the memory buffers constituting IPC message +/// bodies. Intended for use with RecordBatch but could be used for other +/// message types +table BodyCompression { + /// Compressor library. + /// For LZ4_FRAME, each compressed buffer must consist of a single frame. + codec: CompressionType = LZ4_FRAME; + + /// Indicates the way the record batch body was compressed + method: BodyCompressionMethod = BUFFER; +} + +/// A data header describing the shared memory layout of a "record" or "row" +/// batch. Some systems call this a "row batch" internally and others a "record +/// batch". +table RecordBatch { + /// number of records / rows. The arrays in the batch should all have this + /// length + length: long; + + /// Nodes correspond to the pre-ordered flattened logical schema + nodes: [FieldNode]; + + /// Buffers correspond to the pre-ordered flattened buffer tree + /// + /// The number of buffers appended to this list depends on the schema. For + /// example, most primitive arrays will have 2 buffers, 1 for the validity + /// bitmap and 1 for the values. For struct arrays, there will only be a + /// single buffer for the validity (nulls) bitmap + buffers: [Buffer]; + + /// Optional compression of the message body + compression: BodyCompression; + + /// Some types such as Utf8View are represented using a variable number of buffers. + /// For each such Field in the pre-ordered flattened logical schema, there will be + /// an entry in variadicBufferCounts to indicate the number of variadic + /// buffers which belong to that Field in the current RecordBatch. + /// + /// For example, the schema + /// col1: Struct + /// col2: Utf8View + /// contains two Fields with variadic buffers so variadicBufferCounts will have + /// two entries, the first counting the variadic buffers of `col1.beta` and the + /// second counting `col2`'s. + /// + /// This field may be omitted if and only if the schema contains no Fields with + /// a variable number of buffers, such as BinaryView and Utf8View. + variadicBufferCounts: [long]; +} + +/// For sending dictionary encoding information. Any Field can be +/// dictionary-encoded, but in this case none of its children may be +/// dictionary-encoded. +/// There is one vector / column per dictionary, but that vector / column +/// may be spread across multiple dictionary batches by using the isDelta +/// flag + +table DictionaryBatch { + id: long; + data: RecordBatch; + + /// If isDelta is true the values in the dictionary are to be appended to a + /// dictionary with the indicated id. If isDelta is false this dictionary + /// should replace the existing dictionary. + isDelta: bool = false; +} + +/// ---------------------------------------------------------------------- +/// The root Message type + +/// This union enables us to easily send different message types without +/// redundant storage, and in the future we can easily add new message types. +/// +/// Arrow implementations do not need to implement all of the message types, +/// which may include experimental metadata types. For maximum compatibility, +/// it is best to send data using RecordBatch +union MessageHeader { + Schema, DictionaryBatch, RecordBatch, Tensor, SparseTensor +} + +table Message { + version: org.apache.arrow.flatbuf.MetadataVersion; + header: MessageHeader; + bodyLength: long; + custom_metadata: [ KeyValue ]; +} + +root_type Message; diff --git a/samples/FlatBuffers/reflection.fbs b/samples/FlatBuffers/reflection.fbs new file mode 100644 index 0000000000..fb50e09f28 --- /dev/null +++ b/samples/FlatBuffers/reflection.fbs @@ -0,0 +1,156 @@ +// This schema defines objects that represent a parsed schema, like +// the binary version of a .fbs file. +// This could be used to operate on unknown FlatBuffers at runtime. +// It can even ... represent itself (!) + +namespace reflection; + +// These must correspond to the enum in idl.h. +enum BaseType : byte { + None, + UType, + Bool, + Byte, + UByte, + Short, + UShort, + Int, + UInt, + Long, + ULong, + Float, + Double, + String, + Vector, + Obj, // Used for tables & structs. + Union, + Array, + Vector64, + + // Add any new type above this value. + MaxBaseType +} + +table Type { + base_type:BaseType; + element:BaseType = None; // Only if base_type == Vector + // or base_type == Array. + index:int = -1; // If base_type == Object, index into "objects" below. + // If base_type == Union, UnionType, or integral derived + // from an enum, index into "enums" below. + // If base_type == Vector && element == Union or UnionType. + fixed_length:uint16 = 0; // Only if base_type == Array. + /// The size (octets) of the `base_type` field. + base_size:uint = 4; // 4 Is a common size due to offsets being that size. + /// The size (octets) of the `element` field, if present. + element_size:uint = 0; +} + +table KeyValue { + key:string (required, key); + value:string; +} + +table EnumVal { + name:string (required); + value:long (key); + object:Object (deprecated); + union_type:Type; + documentation:[string]; + attributes:[KeyValue]; +} + +table Enum { + name:string (required, key); + values:[EnumVal] (required); // In order of their values. + is_union:bool = false; + underlying_type:Type (required); + attributes:[KeyValue]; + documentation:[string]; + /// File that this Enum is declared in. + declaration_file: string; +} + +table Field { + name:string (required, key); + type:Type (required); + id:ushort; + offset:ushort; // Offset into the vtable for tables, or into the struct. + default_integer:long = 0; + default_real:double = 0.0; + deprecated:bool = false; + required:bool = false; + key:bool = false; + attributes:[KeyValue]; + documentation:[string]; + optional:bool = false; + /// Number of padding octets to always add after this field. Structs only. + padding:uint16 = 0; + /// If the field uses 64-bit offsets. + offset64:bool = false; +} + +table Object { // Used for both tables and structs. + name:string (required, key); + fields:[Field] (required); // Sorted. + is_struct:bool = false; + minalign:int; + bytesize:int; // For structs. + attributes:[KeyValue]; + documentation:[string]; + /// File that this Object is declared in. + declaration_file: string; +} + +table RPCCall { + name:string (required, key); + request:Object (required); // must be a table (not a struct) + response:Object (required); // must be a table (not a struct) + attributes:[KeyValue]; + documentation:[string]; +} + +table Service { + name:string (required, key); + calls:[RPCCall]; + attributes:[KeyValue]; + documentation:[string]; + /// File that this Service is declared in. + declaration_file: string; +} + +/// New schema language features that are not supported by old code generators. +enum AdvancedFeatures : ulong (bit_flags) { + AdvancedArrayFeatures, + AdvancedUnionFeatures, + OptionalScalars, + DefaultVectorsAndStrings, +} + +/// File specific information. +/// Symbols declared within a file may be recovered by iterating over all +/// symbols and examining the `declaration_file` field. +table SchemaFile { + /// Filename, relative to project root. + filename:string (required, key); + /// Names of included files, relative to project root. + included_filenames:[string]; +} + +table Schema { + objects:[Object] (required); // Sorted. + enums:[Enum] (required); // Sorted. + file_ident:string; + file_ext:string; + root_table:Object; + services:[Service]; // Sorted. + advanced_features:AdvancedFeatures; + /// All the files used in this compilation. Files are relative to where + /// flatc was invoked. + fbs_files:[SchemaFile]; // Sorted. +} + +root_type Schema; + +file_identifier "BFBS"; +file_extension "bfbs"; diff --git a/vendor/README.md b/vendor/README.md index 7747c21845..75a67de265 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -194,6 +194,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting - **Fantom:** [rkoeninger/sublime-fantom](https://github.com/rkoeninger/sublime-fantom) - **Faust:** [jpcima/language-faust](https://github.com/jpcima/language-faust) - **Fennel:** [kongeor/vsc-fennel](https://github.com/kongeor/vsc-fennel) +- **FlatBuffers:** [floxay/vscode-flatbuffers](https://github.com/floxay/vscode-flatbuffers) - **Flix:** [flix/textmate](https://github.com/flix/textmate) - **Fluent:** [macabeus/vscode-fluent](https://github.com/macabeus/vscode-fluent) - **Forth:** [textmate/forth.tmbundle](https://github.com/textmate/forth.tmbundle) diff --git a/vendor/grammars/vscode-flatbuffers b/vendor/grammars/vscode-flatbuffers new file mode 160000 index 0000000000..10b032b0a5 --- /dev/null +++ b/vendor/grammars/vscode-flatbuffers @@ -0,0 +1 @@ +Subproject commit 10b032b0a5fcd004afb4f551210e76d7e801c10f diff --git a/vendor/licenses/git_submodule/vscode-flatbuffers.dep.yml b/vendor/licenses/git_submodule/vscode-flatbuffers.dep.yml new file mode 100644 index 0000000000..ccb541c5ce --- /dev/null +++ b/vendor/licenses/git_submodule/vscode-flatbuffers.dep.yml @@ -0,0 +1,31 @@ +--- +name: vscode-flatbuffers +version: 10b032b0a5fcd004afb4f551210e76d7e801c10f +type: git_submodule +homepage: https://github.com/floxay/vscode-flatbuffers.git +license: mit +licenses: +- sources: LICENSE + text: | + MIT License + + Copyright (c) 2023 Huba Tuba + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +notices: []