Merge branch 'develop'

Author: salcock

.gitignore

@@ -25,3 +25,7 @@ src/openliprovisioner
 compile
 src/openlicollector
 src/openlimediator
+/*.yaml
+/*.pcap
+/*.pcap.gz
+/*.sh

README.md

@@ -1,6 +1,6 @@
 OpenLI -- open source ETSI-compliant Lawful Intercept software
 
-Version: 1.1.14
+Version: 1.1.15
 
 ---------------------------------------------------------------------------
 

configure.ac

@@ -1,6 +1,6 @@
 # Super primitive configure script
 
-AC_INIT([openli],[1.1.14],[[email protected]])
+AC_INIT([openli],[1.1.15],[[email protected]])
 
 AM_INIT_AUTOMAKE([subdir-objects])
 AC_CONFIG_SRCDIR(src/collector/collector.c)
@@ -52,6 +52,7 @@ AC_CHECK_LIB([uuid], [uuid_parse],,libuuid_found=0)
 
 if test "x$libzmq_found" = "x1"; then
         COLLECTOR_LIBS="$COLLECTOR_LIBS -lzmq"
+        MEDIATOR_LIBS="$MEDIATOR_LIBS -lzmq"
 fi
 
 if test "x$libssl11_found" = "x1"; then
@@ -121,10 +122,10 @@ if test "x$enable_provisioner" != "xno"; then
 fi
 
 if test "x$enable_collector" != "xno" -o "x$enable_mediator" != "xno"; then
-        AC_CHECK_LIB([wandder], [wandder_decode_integer_value],libwandder_found=1,libwandder_found=0)
+        AC_CHECK_LIB([wandder], [wandder_timeval_to_generalizedts],libwandder_found=1,libwandder_found=0)
 
         if test "$libwandder_found" = 0; then
-                AC_MSG_ERROR(Required library libwandder 2.0.6 or later not found; use LDFLAGS to specify library location)
+                AC_MSG_ERROR(Required library libwandder 2.0.16 or later not found; use LDFLAGS to specify library location)
         fi
 
 

debian/changelog

@@ -1,3 +1,51 @@
+openli (1.1.15-1) unstable; urgency=medium
+
+  * Collector: fix buffer overflow bug in GTP parsing code.
+  * Collector: fix crash if the RabbitMQ server is restarted while the
+    collector is running.
+  * Mediator: enable RabbitMQ publisher confirms and some local buffering
+    to reduce the possibility of ETSI records being lost if RabbitMQ
+    is restarted during an active intercept.
+  * Collector: enable RabbitMQ publisher confirms when using RMQ to send
+    ETSI records to the mediator(s).
+  * Mediator: fix crashes that can occur if the country code for an
+    agency is not configured.
+  * Add option to choose the timestamp format to be used in encoded ETSI
+    PSHeaders (either microsecondTimestamp or generalizedTime).
+  * Collector: fix bug where long running SIP calls would be incorrectly
+    expired due to "inactivity".
+  * Collector: fix crash if the collector is configured to operate with
+    zero SIP worker threads.
+  * REST API: fix bug where the 'lastseen' property for a mediator was
+    being updated even when the mediator had disconnected.
+  * REST API: fix bug where it was not possible to change the destination
+    mediator for an intercept via the REST API.
+  * Add ability to configure a retransmit window for agency
+    handovers (i.e. the amount of handover data that should be retransmitted
+    if the TCP session for that handover fails). Defaults to zero kilobytes.
+  * Add configuration option to specify how many seconds to
+    wait between connection attempts for handovers. Defaults to 10 seconds.
+  * Add experimental support for including Integrity Check PDUs in the
+    handover stream, as per Annex J of ETSI TS 102 232-1. By default,
+    integrity checks are disabled but may be enabled on a per-agency basis.
+  * Provisioner: fix crash that occurred if using the REST API when the running
+    intercept configuration file was not writable.
+  * REST API: add information about the listening X2/X3 endpoints to the
+    response to a 'collectors/' request.
+  * REST API: collector identifiers now include the operator ID,
+    network element ID and intercept point ID -- this replaces the previous
+    identifier which simply used the collector's IP address.
+  * Provisioner: fix crash when a collector reconnects to the provisioner,
+    due to an idle timer for the previous collector instance remaining active
+    upon reconnection.
+  * Collector: fix 100% CPU loop when a forwarding thread failed to connect
+    to a RabbitMQ broker.
+  * Payload encryption is now performed by the mediator(s) rather than by
+    the collectors -- this change was required to support the integrity
+    check feature.
+
+ -- Shane Alcock <[email protected]>  Sun, 12 Oct 2025 18:31:41 +1300
+
 openli (1.1.14-1) unstable; urgency=medium
 
   * Add support for new RabbitMQ header file structure added in

debian/control

@@ -3,7 +3,7 @@ Section: net
 Priority: optional
 Maintainer: Shane Alcock <[email protected]>
 Build-Depends: debhelper-compat (= 12), dh-autoreconf,
- libtrace4-dev (>= 4.0.28), libyaml-dev, uthash-dev, libwandder2-dev (>=2.0.14),
+ libtrace4-dev (>= 4.0.28), libyaml-dev, uthash-dev, libwandder2-dev (>=2.0.16),
  libjudy-dev, libzmq3-dev, libgoogle-perftools-dev, libosip2-dev (>=5.0.0),
  libssl-dev, librabbitmq-dev, libb64-dev, uuid-dev,
  libmicrohttpd-dev, libjson-c-dev, libsqlcipher-dev, zlib1g-dev
@@ -30,7 +30,7 @@ Package: openli-mediator
 Section: net
 Architecture: any
 Depends: ${shlibs:Depends}, ${misc:Depends}, lsb-base, adduser,
-    rabbitmq-server, procps
+    rabbitmq-server, procps, libwandder2 (>=2.0.16)
 Recommends: strongswan
 Description: Mediation daemon for an OpenLI system
  OpenLI is a software suite that allows network operators to conduct
@@ -47,7 +47,7 @@ Package: openli-collector
 Section: net
 Architecture: any
 Depends: ${shlibs:Depends}, ${misc:Depends}, lsb-base, procps,
- rabbitmq-server, adduser, libwandder2 (>=2.0.13)
+ rabbitmq-server, adduser, libwandder2 (>=2.0.16)
 Description: Collector daemon for an OpenLI system
  OpenLI is a software suite that allows network operators to conduct
  lawful interception of Internet traffic that is compliant with the

debian/openli-provisioner.postinst

@@ -36,6 +36,13 @@ case "$1" in
             chmod 0640 /etc/openli/.intercept-encrypt
         fi
 
+        if [ ! -f /etc/openli/integrity-key.pem ]; then
+            openssl ecparam -name prime256v1 -genkey -noout -out /etc/openli/integrity-key.pem
+            openssl ec -in /etc/openli/integrity-key.pem -pubout -out /etc/openli/integrity-public.pem
+
+            chmod 0600 /etc/openli/integrity-key.pem /etc/openli/integrity-public.pem
+        fi
+
         chown -R ${USER}: /etc/openli
         chown -R ${USER}: /var/lib/openli
         chown -R ${USER}: /var/run/openli

debian/openli-provisioner.postrm

@@ -7,6 +7,8 @@ case "$1" in
         rm -f /etc/openli/provauthdb.phrase
         rm -f /etc/openli/.intercept-encrypt
         rm -f /etc/openli/*.yaml
+        rm -f /etc/openli/integrity-key.pem
+        rm -f /etc/openli/integrity-public.pem
 
         ;;
         remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)

doc/ProvisionerDoc.md

@@ -469,6 +469,20 @@ following options:
 * `restauthkey` -- the passphrase needed to decrypt the SQLite3 database
 
 
+If you are enabling the sending of Integrity Check PDUs to any of the
+law enforcement agencies that will be managed by this provisioner, you
+will need to supply a TLS private key for signing some of the hash digests
+included in those PDUs. To do this, you will need to provide the following
+option:
+
+* `integrity-signing-private-key` -- the path to the file containing the
+                                     private key that is to be used to
+                                     sign integrity checks.
+
+See https://github.com/OpenLI-NZ/openli/wiki/Integrity-Checks for more
+information on how to create a key and enable Integrity Checks for a
+particular agency.
+
 ### Intercept Configuration Syntax
 Intercept configuration, i.e. current intercepts, recipient agencies and
 special servers, is stored in a separate YAML file. Ideally, a user would
@@ -507,6 +521,68 @@ key-value elements:
                    connection. Defaults to 30. If set to zero, the mediator
                    will not require a response to keep alives to maintain the
                    handover connections.
+* `connectretrywait` -- the amount of time (in seconds) to wait between attempts
+                        to connect to an agency handover. Defaults to 10
+                        seconds.
+* `resendwindow`  -- the amount of buffered data to retransmit when a handover
+                     reconnects after a disconnection event (in KBs). Defaults
+                     to zero (i.e. don't retransmit anything that was sent
+                     prior to the disconnection).
+* `timestampformat` -- the timestamp field to include in the ETSI PS Header
+                       when encoding records to send to this agency. Allowed
+                       options are "microseconds" to use the
+                       "MicroSecondTimeStamp" field, and "generalized" to use
+                       the "GeneralizedTime" field. The default is
+                       "microseconds".
+* `payloadencryption`     -- Specifies if the CC and IRI contents sent to this
+                             agency should be encrypted and, if so, which
+                             encryption method to use. If set to "none", no
+                             encryption is performed.
+                             The only encryption method supported right now is
+                             "aes-192-cbc".
+                             The default setting is "none".
+* `encryptionkey`         -- The encryption key to use when encrypting CC and
+                             IRI contents. This option is mandatory if
+                             `payloadencryption` is NOT set to "none". The
+                             ideal key length is 24 characters. Shorter keys
+                             will be padded with null bytes, longer keys will be
+                             truncated to 24 characters.
+* `integrity` -- a YAML mapping object that defines whether integrity check
+                 messages should be sent to this agency, and how these messages
+                 should be generated.
+
+If Integrity Check records are required, the configuration parameters for
+these records can be set by specifying fields inside the `integrity` object
+mentioned above. Note that the agency will most likely tell you what values
+it wants configured for these options, so do not worry too much about having
+to decide what to choose for each option.
+
+The available fields are:
+* `enabled`     -- if true, integrity check records will be sent to this agency.
+                   Defaults to false (i.e. no integrity checks).
+* `hashmethod`  -- the algorithm to use when generating message digests from
+                   the intercepted data PDUs. Defaults to `sha-256`, but
+                   `sha-1`, `sha-384` and `sha-512` are also supported.
+* `signedhashmethod`    -- the algorithm to use when generated a digest from
+                           previous message digests that is going to be signed
+                           using a private key. Defaults to `sha-256` but
+                           `sha-1`, `sha-384` and `sha-512` are also supported.
+* `hashtimeout`  -- produce an integrity check containing a message digest hash
+                    within this number of seconds after seeing the oldest
+                    unhashed data PDU. Defaults to 1 second.
+* `datapducount` -- produce an integrity check containing a message digest hash
+                    as soon as this number of unhashed data PDUs have been seen.
+                    Defaults to 1000 PDUs.
+* `signtimeout`  -- produce an integrity check containing a signed digest hash
+                    of the preceding message digests within this number of
+                    seconds after sending the oldest unsigned digest hash.
+                    Defaults to 30 seconds.
+* `hashpducount` -- produce an integrity check containing a signed digest hash
+                    of the preceding message digests as soon as the number of
+                    unsigned digest hashes exceeds this value. Defaults to 15
+                    digest hashes.
+
+---
 
 VOIP, Email and IPintercepts are also expressed as a YAML sequence, with a key
 of `voipintercepts:`, `emailintercepts:`, and `ipintercepts:` respectively.
@@ -670,6 +746,11 @@ All intercept types also support the following optional key-value elements:
                              ideal key length is 24 characters. Shorter keys
                              will be padded with null bytes, longer keys will be
                              truncated to 24 characters.
+
+Note that encryption configuration provided at the intercept level will
+override any encryption configuration that has been set at the agency level
+for the agency that the intercept is destined for.
+
 ---
 
 The default approach for delivering compressed email content to the agencies

doc/exampleconfigs/provisioner-example.yaml

@@ -38,6 +38,13 @@ encrypt-intercept-config-file: no
 # if an LEA reports difficulty with decoding your intercepted RTP streams.
 voip-ignorecomfort: no
 
+# You may be required to send integrity check messages to at least one of
+# the agencies for which you are providing intercepts. If so, some of these
+# messages will need to be signed, and this configuration option allows you
+# to specify where to find the DSA private key that should be used for
+# generating those signatures.
+integrity-signing-private-key: /etc/openli/integrity-key.pem
+
 # Location of the SQLite3 database where credentials and API keys are stored
 # for authorised access to the REST API. If this option is not present,
 # then the REST API can be used without authentication.

doc/exampleconfigs/running-intercept-example.yaml

@@ -71,6 +71,13 @@ agencies:
                                 # (5 minutes), if handover is idle
    keepalivewait: 30            # agency must respond to a keep alive within
                                 # 30 seconds to avoid being disconnected
+   connectretrywait: 30         # wait 30 seconds between attempts to connect
+                                # to either handover
+   resendwindow: 256            # retransmit the last 256KB of data when a
+                                # handover recovers from a disconnection
+   timestampformat: "generalized"   # include the GeneralizedTime timeStamp
+                                    # field in the PS header of all records
+                                    # sent to this agency
 
  - agencyid: "Spooks"           # id must be unique per agency
    agencycountrycode: "NZ"      # 2 letter country code (ISO 3166) matching the
@@ -83,6 +90,11 @@ agencies:
                                 # (2 minutes), if handover is idle
    keepalivewait: 0             # agency does not respond to keep alives, so
                                 # don't disconnect if no response is received
+   connectretrywait: 5          # wait 5 seconds between attempts to connect
+                                # to either handover
+   timestampformat: "microseconds"   # include the microSecondTimeStamp
+                                     # field in the PS header of all records
+                                     # sent to this agency
 
 # List of active IP intercepts.
 # To change intercepts for a running OpenLI process, modify the intercept list