Merge branch 'master' into feat/faster-cast-in-generated-python

Author: ntamas

.github/workflows/flake8-generated.cfg

@@ -0,0 +1,12 @@
+# These flake8 settings apply only to auto-generated Python code and are
+# different than this repo's .flake8 file settings that apply to handwritten code.
+# E203 whitespace before ':'
+# E302 expected 2 blank lines, found 1
+# E305 expected 2 blank lines after class or function definition, found 1
+# E501 line too long (5519 > 79 characters)
+# E741 ambiguous variable name 'I'
+# F401 'sys' imported but unused
+[flake8]
+ignore = E203,E302,E305,E741,F401
+max-complexity = 37
+max-line-length = 5519

.github/workflows/python-publish.yml

@@ -84,7 +84,7 @@ jobs:
       shell: bash
       run: echo "CIBW_BUILD=cp${PYTHON_VERSION/./}-*" >> $GITHUB_ENV
     - name: Build wheels
-      uses: pypa/[email protected].0
+      uses: pypa/[email protected].1
       env:
         PYMAVLINK_FAST_INDEX: "1"
         CIBW_BUILD: ${{ matrix.cibw_build }}

.github/workflows/test.yml

@@ -41,10 +41,6 @@ jobs:
       # NOTE: we must do all testing on the installed python package, not
       # on the build tree. Otherwise the testing is invalid and may not
       # indicate the code actually works
-      #
-      # Set pythonpath
-
-      # install
       - name: Install mavlink message
         run: |
           git clone https://github.com/ArduPilot/mavlink.git
@@ -56,7 +52,7 @@ jobs:
 
       - name: Lint generated python code
         run: |
-            flake8 --ignore='W503,E203,E501,E741' --statistics dialects/
+            flake8 --config=.github/workflows/flake8-generated.cfg --statistics dialects/
 
       - name: Type check generated python code
         if: matrix.python-version != '3.7'

DFReader.py

@@ -1385,27 +1385,34 @@ def init_arrays_fast(self, progress_callback=None):
                     if 'MultIds' in fmt.colhash:
                         fmt2.set_mult_ids(null_term(elements[fmt.colhash['MultIds']]), self.mult_lookup)
 
-        # Parse the first 100 messages of each type to try to build the
-        # messages dictionary. 100 was chosen as a reasonable heuristic to
-        # catch every index value that might be in that message.
+        # Parse the messages of each type to try to build the messages
+        # dictionary.
         for mtype in range(256):
             if mtype not in self.formats:
                 continue
             fmt = self.formats[mtype]
-            NMSG = 100 if fmt.instance_field is not None else 1
+            omtype = offsets[mtype]
+            num_offsets = len(omtype)
+            if fmt.instance_field:
+                NMSG = num_offsets
+                if not mtype in type_instances:
+                    type_instances[mtype] = set()
+                if fmt.instance_len == 1:
+                    # hack to reduce load cost, only scan first 100 messages
+                    # for single byte instance values. This works nearly all the time, and allows
+                    # for full indexing for things like NVF which is string indexed, while not taking a huge amount of CPU
+                    # on IMU and other bulk data
+                    NMSG = min(NMSG, 100)
+            else:
+                NMSG = min(1, num_offsets)
             for i in range(NMSG):
-                if i >= len(offsets[mtype]):
-                    break
-                ofs = offsets[mtype][i]
-                if self.formats[mtype].name not in self.messages:
+                ofs = omtype[i]
+                if fmt.name not in self.messages:
                     self.offset = ofs
                     self._parse_next()
-                if self.formats[mtype].instance_field is not None:
-                    fmt = self.formats[mtype]
-                    # see if we've has this instance value before
+                if fmt.instance_field is not None:
+                    # see if we've had this instance value before
                     idata = data[ofs+3+fmt.instance_ofs:ofs+3+fmt.instance_ofs+fmt.instance_len]
-                    if not mtype in type_instances:
-                        type_instances[mtype] = set()
                     if not idata in type_instances[mtype]:
                         # its a new one, need to parse it so we have the complete set of instances
                         type_instances[mtype].add(idata)

generator/mavgen.py

@@ -33,7 +33,6 @@
 # Set defaults for generating MAVLink code
 DEFAULT_WIRE_PROTOCOL = mavparse.PROTOCOL_1_0
 DEFAULT_LANGUAGE = 'Python'
-DEFAULT_ERROR_LIMIT = 200
 DEFAULT_VALIDATE = True
 DEFAULT_STRICT_UNITS = False
 
@@ -298,9 +297,8 @@ def mavgen_validate(xmlfile):
 
 # build all the dialects in the dialects subpackage
 class Opts(object):
-    def __init__(self, output, wire_protocol=DEFAULT_WIRE_PROTOCOL, language=DEFAULT_LANGUAGE, validate=DEFAULT_VALIDATE, error_limit=DEFAULT_ERROR_LIMIT, strict_units=DEFAULT_STRICT_UNITS):
+    def __init__(self, output, wire_protocol=DEFAULT_WIRE_PROTOCOL, language=DEFAULT_LANGUAGE, validate=DEFAULT_VALIDATE, strict_units=DEFAULT_STRICT_UNITS):
         self.wire_protocol = wire_protocol
-        self.error_limit = error_limit
         self.language = language
         self.output = output
         self.validate = validate

mavextra.py

@@ -1081,9 +1081,10 @@ def ekf1_pos(EKF1):
   from . import mavutil
   self = mavutil.mavfile_global
   if ekf_origin is None:
-      if not 'ORGN' in self.messages:
+      # Look for the ORGN[0] message explicitly
+      if not 'ORGN[0]' in self.messages:
           return None
-      ekf_origin = self.messages['ORGN']
+      ekf_origin = self.messages['ORGN[0]']
       (ekf_origin.Lat, ekf_origin.Lng) = (ekf_origin.Lat, ekf_origin.Lng)
   (lat,lon) = gps_offset(ekf_origin.Lat, ekf_origin.Lng, EKF1.PE, EKF1.PN)
   alt = ekf_origin.Alt - EKF1.PD

mavutil.py

@@ -2718,7 +2718,20 @@ def dump_message_verbose(f, m):
         timestamp = "%s.%02u: " % (
             time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp)),
             int(timestamp*100.0)%100)
-    f.write("%s%s (id=%u) (link=%s) (signed=%s) (seq=%u) (src=%u/%u)\n" % (timestamp, m.get_type(), m.get_msgId(), str(m.get_link_id()), str(m.get_signed()), m.get_seq(), m.get_srcSystem(), m.get_srcComponent()))
+    if m.get_signed():
+        signed = f"Yes; out-link={str(m.get_link_id())}"
+    else:
+        signed = "No"
+
+    inbound_link = getattr(m, '_link', None)
+    f.write(
+        f"{timestamp}{m.get_type()} (id={m.get_msgId()}) "
+        f"(seq={m.get_seq()}) "
+        f"(src={m.get_srcSystem()}/{m.get_srcComponent()}) "
+        f"(in-link={inbound_link}) "
+        f"(signed={signed})\n"
+    )
+
     for fieldname in m.get_fieldnames():
 
         # format in those most boring way possible:

tools/mavgen.py

@@ -23,7 +23,6 @@
 parser.add_argument("--lang", dest="language", choices=mavgen.supportedLanguages, default=mavgen.DEFAULT_LANGUAGE, help="language of generated code [default: %(default)s]")
 parser.add_argument("--wire-protocol", choices=[mavparse.PROTOCOL_0_9, mavparse.PROTOCOL_1_0, mavparse.PROTOCOL_2_0], default=mavgen.DEFAULT_WIRE_PROTOCOL, help="MAVLink protocol version. [default: %(default)s]")
 parser.add_argument("--no-validate", action="store_false", dest="validate", default=mavgen.DEFAULT_VALIDATE, help="Do not perform XML validation. Can speed up code generation if XML files are known to be correct.")
-parser.add_argument("--error-limit", default=mavgen.DEFAULT_ERROR_LIMIT, help="maximum number of validation errors to display")
 parser.add_argument("--strict-units", action="store_true", dest="strict_units", default=mavgen.DEFAULT_STRICT_UNITS, help="Perform validation of units attributes.")
 parser.add_argument("definitions", metavar="XML", nargs="+", help="MAVLink definitions")
 args = parser.parse_args()

tools/mavlogdump.py

@@ -238,6 +238,9 @@ def match_type(mtype, patterns):
         if (isbin or islog) and m_type == "FMT":
             output.write(m.get_msgbuf())
             continue
+        if (isbin or islog) and m_type in ["FMTU", "MULT", "UNIT"]:
+            output.write(m.get_msgbuf())
+            continue
         if (isbin or islog) and (m_type == "PARM" and args.parms):
             output.write(m.get_msgbuf())
             continue