Add strongly typed messages in protocol #114

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

DrTon merged 46 commits into v1 from v1-separate-types

Dec 18, 2024

.github/workflows/js.yml

-Original file line number
+Diff line change
@@ Expand Up / @@ -8,6 +8,7 @@ on: @@
     jobs:
       build:
+        if: false  # disable the entire workflow
         runs-on: ubuntu-latest
         steps:
           - uses: actions/checkout@v3
@@ Expand Down @@

Makefile

-Original file line number
+Diff line change
@@ -0,0 +1,22 @@
+    ROOT_DIR ?= $(realpath $(PWD))
+    BUILD_DIR ?= $(ROOT_DIR)/build
+    BUILD_TYPE ?= Debug
+    all: check
+    configure:
+    	cmake -B${BUILD_DIR} -S. -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=1 -DCMAKE_BUILD_TYPE=${BUILD_TYPE}
+    build: configure
+    	cmake --build ${BUILD_DIR}
+    test: build
+    	ctest --test-dir ${BUILD_DIR}/tests/ --output-on-failure
+    	python3 -m pytest .
+    check: test
+    	python3 -m mypy .
+    clean:
+    	rm -rf ${BUILD_DIR}

README.md

-Original file line number
+Diff line change
@@ Expand Up / @@ -17,27 +17,37 @@ Features: @@
     - Supported output formats: C++, JSON, TypeScript
     - Supported output formats TODO: Go, Markdown (documentation)
-    ## Dependencies
+    ## Runtime Dependencies
     - Python 3.X
     On Linux:
-    ```
+    ```bash
     sudo apt install python3
     ```
-    On Windows 10:
+    On Windows:
 . Download https://bootstrap.pypa.io/get-pip.py
 . Execute `python3 get_pip.py`
 . Execute `pip3 install pyyaml`
-    ### Build dependencies
+    ## Build & Test Dependencies
     - libgtest-dev (for testing)
+    - pytest (for testing)
+    - cmake
+    - ninja
+    - mypy
+    ## Testing & Verification
+    ```bash
+    make check
+    ```
-    ## Generate messages
+    ## Generating messages
     All data types should be placed in one directory. Each protocol can be placed in any arbitrary directory.
@@ Expand Down Expand Up @@
     #### Protocol
-    **Protocol** defines the protocol ID and type IDs for structs that will be used as messages.
-    Type ID used during serialization/deserialization to identify the message type.
-    Multiple protocols may be used in one system, e.g. `my_namespace/bootloader` and `my_namespace/application`.
-    Parser can check the protocol by protocol ID, that can be serialized in message header.
+    **Protocol** defines the protocol ID and message IDs for structs that will be used
+    as messages. Message ID used during serialization/deserialization to identify the
+    message type. Multiple protocols may be used in one system, e.g.
+    `my_namespace/bootloader` and `my_namespace/application`. Parser can check the
+    protocol by protocol ID, that can be serialized in message header.
     Example protocol definition (`weather_station.yaml`):
     ```yaml
     comment: "Weather station application protocol"
-    types_map:
-: "heartbeat"
-: "system_status"
-: "system_command"
-: "baro_report"
+    messages:
+: { name: "heartbeat", type: "application/heartbeat", comment: "Heartbeat message" }
+: { name: "system_status", type: "system/status", comment: "System status message" }
+: { name: "system_command", type: "system/command", comment: "System command message" }
+: { name: "baro_report", type: "measurement/baro_report", comment: "Barometer report message" }
     ```

messgen-generate.py

-Original file line number
+Diff line change
@@ -1,6 +1,7 @@
     import argparse
     import os
+    from messgen.validation import validate_protocol, validate_types
     from messgen import generator, yaml_parser
     from pathlib import Path
@@ Expand All / @@ -26,10 +27,14 @@ def generate(args: argparse.Namespace): @@
         if (gen := generator.get_generator(args.lang, opts)) is not None:
             if parsed_protocols and parsed_types:
-                gen.generate(Path(args.outdir), parsed_types, parsed_protocols)
-            elif parsed_types:
+                for proto_def in parsed_protocols.values():
+                    validate_protocol(proto_def, parsed_types)
+            if parsed_types:
+                validate_types(parsed_types)
                 gen.generate_types(Path(args.outdir), parsed_types)
-            elif parsed_protocols:
+            if parsed_protocols:
                 gen.generate_protocols(Path(args.outdir), parsed_protocols)
         else:
@@ Expand Down @@

messgen/cpp_generator.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -9,8 +9,6 @@
  
        Path,

    )

    from .validation import validate_protocol

    from .common import (

        SEPARATOR,

        SIZE_TYPE,

    @@ -58,6 +56,7 @@ def _namespace(name: str, code:list[str]):
  
            if ns_name:

                code.append("")

                code.append(f"}} // namespace {ns_name}")

                code.append("")

    @contextmanager

    @@ -67,6 +66,7 @@ def _struct(name: str, code: list[str]):
  
            yield

        finally:

            code.append("};")

            code.append("")

    def _inline_comment(type_def: FieldType | EnumValue):

    @@ -125,15 +125,6 @@ def __init__(self, options: dict):
  
            self._ctx: dict = {}

            self._types: dict[str, MessgenType] = {}

        def generate(self, out_dir: Path, types: dict[str, MessgenType], protocols: dict[str, Protocol]) -> None:

            self.validate(types, protocols)

            self.generate_types(out_dir, types)

            self.generate_protocols(out_dir, protocols)

        def validate(self, types: dict[str, MessgenType], protocols: dict[str, Protocol]):

            for proto_def in protocols.values():

                validate_protocol(proto_def, types)

        def generate_types(self, out_dir: Path, types: dict[str, MessgenType]) -> None:

            self._types = types

            for type_name, type_def in types.items():

    @@ -169,7 +160,7 @@ def _generate_type_file(self, type_name: str, type_def: MessgenType) -> list:
  
                elif isinstance(type_def, StructType):

                    code.extend(self._generate_type_struct(type_name, type_def))

                    code.extend(self._generate_members_of(type_name, type_def))

                    code.extend(self._generate_type_members_of(type_name, type_def))

            code = self._PREAMBLE_HEADER + self._generate_includes() + code

    @@ -185,82 +176,92 @@ def _generate_proto_file(self, proto_name: str, proto_def: Protocol) -> list[str
  
            self._add_include("messgen/messgen.h")

            namespace_name, class_name = _split_last_name(proto_name)

            print(f"Namespace: {namespace_name}, Class: {class_name}")

            with _namespace(namespace_name, code):

                with _struct(class_name, code):

                    for type_name in proto_def.types.values():

                        self._add_include(type_name + self._EXT_HEADER)

                    for message in proto_def.messages.values():

                        self._add_include(message.type + self._EXT_HEADER)

                    proto_id = proto_def.proto_id

                    if proto_id is not None:

                        code.append(f"    constexpr static int PROTO_ID = {proto_id};")

                        code.append(f"    constexpr static inline int PROTO_ID = {proto_id};")

                        code.append(f"    constexpr static inline uint32_t HASH = {hash(proto_def)};")

                    code.extend(self._generate_type_id_decl(proto_def))

                    code.extend(self._generate_reflect_type_decl())

                    code.extend(self._generate_messages(class_name, proto_def))

                    code.extend(self._generate_reflect_message_decl())

                    code.extend(self._generate_dispatcher_decl())

                code.extend(self._generate_type_ids(class_name, proto_def))

                code.extend(self._generate_reflect_type(class_name, proto_def))

                code.extend(self._generate_protocol_members_of(class_name, proto_def))

                code.extend(self._generate_reflect_message(class_name, proto_def))

                code.extend(self._generate_dispatcher(class_name))

                code.append("")

            return self._PREAMBLE_HEADER + self._generate_includes() + code

        @staticmethod

        def _generate_type_id_decl(proto: Protocol) -> list[str]:

            return textwrap.indent(textwrap.dedent("""

                template <messgen::type Msg>

                constexpr static inline int TYPE_ID = []{

                    static_assert(sizeof(Msg) == 0, \"Provided type is not part of the protocol.\");

                    return 0;

                }();"""), "    ").splitlines()

        @staticmethod

        def _generate_type_ids(class_name: str, proto: Protocol) -> list[str]:

        def _generate_messages(self, class_name: str, proto_def: Protocol):

            self._add_include("tuple")

            code: list[str] = []

            for type_id, type_name in proto.types.items():

                code.append(f"    template <> constexpr inline int {class_name}::TYPE_ID<{_qual_name(type_name)}> = {type_id};")

            for message in proto_def.messages.values():

                code.extend(textwrap.indent(textwrap.dedent(f"""

                struct {message.name} : {_qual_name(message.type)} {{

                    using data_type = {_qual_name(message.type)};

                    using protocol_type = {class_name};

                    constexpr inline static int PROTO_ID = protocol_type::PROTO_ID;

                    constexpr inline static int MESSAGE_ID = {message.message_id};

                }};"""), "    ").splitlines())

            return code

        def _generate_protocol_members_of(self, class_name: str, proto_def: Protocol):

            self._add_include("tuple")

            code: list[str] = []

            code.append(f"[[nodiscard]] consteval auto members_of(::messgen::reflect_t<{class_name}>) noexcept {{")

            code.append("    return std::tuple{")

            for message in proto_def.messages.values():

                code.append(f"        ::messgen::member<{class_name}, {class_name}::{message.name}>{{\"{message.name}\"}},")

            code.append("    };")

            code.append("}")

            code.append("")

            return code

        @staticmethod

        def _generate_reflect_type_decl() -> list[str]:

        def _generate_reflect_message_decl() -> list[str]:

            return textwrap.indent(textwrap.dedent("""

                template <class Fn>

                constexpr static auto reflect_message(int type_id, Fn&& fn);

                constexpr static auto reflect_message(int msg_id, Fn &&fn);

                """), "    ").splitlines()

        @staticmethod

        def _generate_reflect_type(class_name: str, proto: Protocol) -> list[str]:

        def _generate_reflect_message(class_name: str, proto: Protocol) -> list[str]:

            code: list[str] = []

            code.append("    template <class Fn>")

            code.append(f"    constexpr auto {class_name}::reflect_message(int type_id, Fn&& fn) {{")

            code.append("        switch (type_id) {")

            for type_name in proto.types.values():

                qual_name = _qual_name(type_name)

                code.append(f"            case TYPE_ID<{qual_name}>:")

                code.append(f"                std::forward<Fn>(fn)(::messgen::reflect_type<{qual_name}>);")

                code.append(f"                return;")

            code.append("        }")

            code.append("template <class Fn>")

            code.append(f"constexpr auto {class_name}::reflect_message(int msg_id, Fn &&fn) {{")

            code.append("    switch (msg_id) {")

            for message in proto.messages.values():

                msg_type = f"{class_name}::{_unqual_name(message.name)}"

                code.append(f"        case {msg_type}::MESSAGE_ID:")

                code.append(f"            std::forward<Fn>(fn)(::messgen::reflect_type<{msg_type}>);")

                code.append(f"            return;")

            code.append("    }")

            code.append("}")

            return code

        @staticmethod

        def _generate_dispatcher_decl() -> list[str]:

            return textwrap.indent(textwrap.dedent("""

                template <class T>

                static bool dispatch_message(int msg_id, const uint8_t *payload, T handler);

                constexpr static bool dispatch_message(int msg_id, const uint8_t *payload, T handler);

                """), "    ").splitlines()

        @staticmethod

        def _generate_dispatcher(class_name: str) -> list[str]:

            return textwrap.dedent(f"""

                template <class T>

                bool {class_name}::dispatch_message(int msg_id, const uint8_t *payload, T handler) {{

                constexpr bool {class_name}::dispatch_message(int msg_id, const uint8_t *payload, T handler) {{

                    auto result = false;

                    reflect_message(msg_id, [&]<class R>(R) {{

                        using message_type = messgen::splice_t<R>;

                        if constexpr (requires(message_type msg) {{ handler(msg); }}) {{

                            message_type msg;

                            auto msg = message_type{{}};

                            msg.deserialize(payload);

                            handler(std::move(msg));

                            result = true;

    @@ -269,17 +270,6 @@ def _generate_dispatcher(class_name: str) -> list[str]:
  
                    return result;

                }}""").splitlines()

        @staticmethod

        def _generate_traits() -> list[str]:

            return textwrap.dedent("""

                namespace messgen {

                    template <class T>

                    struct reflect_t {};

                    template <class T>

                    struct splice_t {};

                }""").splitlines()

        @staticmethod

        def _generate_comment_type(type_def):

            if not type_def.comment:

    @@ -379,11 +369,12 @@ def _generate_type_struct(self, type_name: str, type_def: StructType):
  
            is_empty = len(groups) == 0

            is_flat = is_empty or (len(groups) == 1 and groups[0].size is not None)

            if is_flat:

                code.append(_indent("static constexpr size_t FLAT_SIZE = %d;" % (0 if is_empty else groups[0].size)))

                code.append(_indent("constexpr static inline size_t FLAT_SIZE = %d;" % (0 if is_empty else groups[0].size)))

                is_flat_str = "true"

            code.append(_indent(f"static constexpr bool IS_FLAT = {is_flat_str};"))

            code.append(_indent(f"static constexpr const char* NAME = \"{_qual_name(type_name)}\";"))

            code.append(_indent(f"static constexpr const char* SCHEMA = R\"_({self._generate_schema(type_def)})_\";"))

            code.append(_indent(f"constexpr static inline bool IS_FLAT = {is_flat_str};"))

            code.append(_indent(f"constexpr static inline uint32_t HASH = {hash(type_def)};"))

            code.append(_indent(f"constexpr static inline const char* NAME = \"{_qual_name(type_name)}\";"))

            code.append(_indent(f"constexpr static inline const char* SCHEMA = R\"_({self._generate_schema(type_def)})_\";"))

            code.append("")

            for field in type_def.fields:

    @@ -483,7 +474,7 @@ def _generate_type_struct(self, type_name: str, type_def: StructType):
  
            if self._get_cpp_standard() >= 20:

                # Operator <=>

                code.append("")

                code.append(_indent("auto operator<=>(const %s&) const = default;" % unqual_name))

                code.append(_indent("auto operator<=>(const %s &) const = default;" % unqual_name))

            code.append("};")

    @@ -527,17 +518,17 @@ def _generate_includes(self):
  
                code.append("")

            return code

        def _generate_members_of(self, type_name: str, type_def: StructType):

        def _generate_type_members_of(self, type_name: str, type_def: StructType):

            self._add_include("tuple")

            unqual_name = _unqual_name(type_name)

            code: list[str] = []

            code.append("")

            code.append(f"[[nodiscard]] inline constexpr auto members_of(::messgen::reflect_t<{unqual_name}>) noexcept {{")

            code.append(f"[[nodiscard]] consteval auto members_of(::messgen::reflect_t<{unqual_name}>) noexcept {{")

            code.append("    return std::tuple{")

            for field in type_def.fields:

                code.append(f"        ::messgen::member{{\"{field.name}\", &{unqual_name}::{field.name}}},")

                code.append(f"        ::messgen::member_variable{{{{\"{field.name}\"}}, &{unqual_name}::{field.name}}},")

            code.append("    };")

            code.append("}")

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add strongly typed messages in protocol #114

Uh oh!

Diff view

Diff view

There are no files selected for viewing

DrTon Dec 18, 2024 •

edited

Loading

Uh oh!

Uh oh!

Uh oh!

Add strongly typed messages in protocol #114

Uh oh!

Add strongly typed messages in protocol #114

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

DrTon Dec 18, 2024 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Uh oh!

DrTon Dec 18, 2024 •

edited

Loading