Merge branch 'develop'

salcock · salcock · commit 1e3790af22fc · 2024-08-15T11:01:53.000+12:00
diff --git a/AUTHORS b/AUTHORS
@@ -16,6 +16,9 @@ In no particular order, we would like to thank the following people:
  * Tyler Marriner for adding support for encrypting inter-component
    communications using TLS and adding RabbitMQ support to the collector.
  * Neil Tapp for pointing out many bugs and logging inaccuracies.
+ * Pim van Stam for contributing code to support the "agencycountrycode"
+   configuration option, as well as handling of certain NL-specific
+   requirements.
 
 Apologies to anyone that has made a contribution but we've forgotten to
 mention you here -- if this has happened to you, please get in touch with the
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 OpenLI -- open source ETSI-compliant Lawful Intercept software
 
-Version: 1.1.7
+Version: 1.1.8
 
 ---------------------------------------------------------------------------
 
diff --git a/configure.ac b/configure.ac
@@ -1,6 +1,6 @@
 # Super primitive configure script
 
-AC_INIT([openli],[1.1.7],[shane@alcock.co.nz])
+AC_INIT([openli],[1.1.8],[shane@alcock.co.nz])
 
 AM_INIT_AUTOMAKE([subdir-objects])
 AC_CONFIG_SRCDIR(src/collector/collector.c)
diff --git a/debian/changelog b/debian/changelog
@@ -1,3 +1,22 @@
+openli (1.1.8-1) unstable; urgency=medium
+
+  * Collector: fix crash in sync_voip thread if an invalid SIP packet
+    is encountered.
+  * Collector: add a single zero byte to the list of recognised SIP
+    keep alives.
+  * Collector: fix crash that can occur if an IP is mapped to a
+    RADIUS session more than once.
+  * Add config option to specify the country where an agency has
+    jurisdiction, which allows us to support country-specific
+    requirements for HI1 operations and keep alive messages.
+  * Keep alive messages for NL agencies now conform to the ETSI-IP.nl
+    requirements.
+  * Use '--' instead of 'NA' as the auth and delivery country code for
+    keep alives when we do not know the country code for the receiving
+    agency.
+
+ -- Shane Alcock <salcock@searchlight.nz>  Wed, 14 Aug 2024 10:13:28 +1200
+
 openli (1.1.7-1) unstable; urgency=medium
 
   * Collector: fix file descriptor leak caused by timers in SMS worker
diff --git a/doc/ProvisionerDoc.md b/doc/ProvisionerDoc.md
@@ -439,6 +439,8 @@ Agencies are expressed as a YAML sequence with a key of `agencies:`. Each
 sequence item represents a single agency and must contain the following
 key-value elements:
 * `agencyid`      -- the unique internal identifier for this agency
+* `agencycountrycode`  -- the 2-letter ISO 3166 country code for the country
+                          where the agency is located.
 * `hi2address`    -- the address of the HI2 handover on the agency side
 * `hi2port`       -- the port number for the HI2 handover on the agency side
 * `hi3address`    -- the address of the HI3 handover on the agency side
diff --git a/doc/exampleconfigs/running-intercept-example.yaml b/doc/exampleconfigs/running-intercept-example.yaml
@@ -61,6 +61,8 @@ defaultradiususers:
 agencies:
 
  - agencyid: "Police"           # id must be unique per agency
+   agencycountrycode: "NZ"      # 2 letter country code (ISO 3166) matching the
+                                # agency's jurisdiction
    hi2address: 192.168.200.1    # address of the HI2 service at the agency
    hi2port: 35530               # port number of the HI2 service at the agency
    hi3address: 192.168.200.1    # address of the HI3 service at the agency
@@ -71,6 +73,8 @@ agencies:
                                 # 30 seconds to avoid being disconnected
 
  - agencyid: "Spooks"           # id must be unique per agency
+   agencycountrycode: "NZ"      # 2 letter country code (ISO 3166) matching the
+                                # agency's jurisdiction
    hi2address: 10.10.1.1        # address of the HI2 service at the agency
    hi2port: 7001                # port number of the HI2 service at the agency
    hi3address: 10.10.1.2        # address of the HI3 service at the agency
diff --git a/rpm/openli.spec b/rpm/openli.spec
@@ -1,5 +1,5 @@
 Name:           openli
-Version:        1.1.7
+Version:        1.1.8
 Release:        1%{?dist}
 Summary:        Software for performing ETSI-compliant lawful intercept
 
@@ -283,6 +283,9 @@ fi
 
 
 %changelog
+* Thu Jul 25 2024 Shane Alcock <salcock@searchlight.nz> - 1.1.8-1
+- Updated for 1.1.8 release
+
 * Tue Jul 23 2024 Shane Alcock <salcock@searchlight.nz> - 1.1.7-1
 - Updated for 1.1.7 release
 
diff --git a/src/agency.c b/src/agency.c
@@ -43,6 +43,9 @@ void free_liagency(liagency_t *lea) {
 	if (lea->agencyid) {
 		free(lea->agencyid);
 	}
+    if (lea->agencycc) {
+        free(lea->agencycc);
+    }
 	free(lea);
 }
 
diff --git a/src/agency.h b/src/agency.h
@@ -39,6 +39,7 @@ typedef struct liagency {
     char *hi3_ipstr;
     char *hi3_portstr;
     char *agencyid;
+    char *agencycc;
     uint32_t keepalivefreq;
     uint32_t keepalivewait;
 } liagency_t;
@@ -48,7 +49,10 @@ typedef struct liagency {
      (strcmp(a->hi2_portstr, b->hi2_portstr) == 0) && \
      (strcmp(a->hi3_ipstr, b->hi3_ipstr) == 0) && \
      (strcmp(a->hi3_portstr, b->hi3_portstr) == 0) && \
-     (strcmp(a->agencyid, b->agencyid) == 0))
+     (strcmp(a->agencyid, b->agencyid) == 0) && \
+     ((a->agencycc == NULL && b->agencycc == NULL) || \
+        (a->agencycc != NULL && b->agencycc != NULL && \
+         strcmp(a->agencycc, b->agencycc) == 0)))
 
 #endif
 
diff --git a/src/collector/collector_sync.c b/src/collector/collector_sync.c
@@ -2000,6 +2000,7 @@ static inline int report_silent_logoffs(collector_sync_t *sync,
         if (remove_session_ip(prev->session[i], &(prev->ip)) == 1) {
             HASH_DELETE(hh, prev->owner[i]->sessions, prev->session[i]);
             free_single_session(prev->session[i]);
+            prev->session[i] = NULL;
         }
     }
     HASH_DELETE(hh, sync->activeips, prev);
@@ -2012,7 +2013,7 @@ static inline int report_silent_logoffs(collector_sync_t *sync,
 static int add_ip_to_session_mapping(collector_sync_t *sync,
         access_session_t *sess, internet_user_t *iuser) {
 
-    int i, replaced = 0;
+    int i, j, replaced = 0;
     ip_to_session_t *prev;
 
     prev = NULL;
@@ -2028,6 +2029,21 @@ static int add_ip_to_session_mapping(collector_sync_t *sync,
                 sizeof(internetaccess_ip_t), prev);
 
         if (prev && prev->cin == sess->cin) {
+            int already = 0;
+            for (j = 0; j < prev->sessioncount; j++) {
+                if (prev->session[j] == sess) {
+                    already = 1;
+                    break;
+                }
+            }
+
+            /* This IP->session mapping is already known (somehow?),
+             * don't insert it twice because that can cause issues
+             * if we have to do a silent-logoff later on */
+            if (already) {
+                continue;
+            }
+
             prev->session = realloc(prev->session,
                     (prev->sessioncount + 1) * sizeof(access_session_t *));
             prev->owner = realloc(prev->owner,
diff --git a/src/collector/collector_sync_voip.c b/src/collector/collector_sync_voip.c
@@ -1953,18 +1953,15 @@ static void sip_update_fast_path(collector_sync_voip_t *sync,
             &(baseirimsg.data.ipmmiri.ipfamily)) != 0) {
         handle_bad_sip_update(sync, &recvdpkt, 1,
                 SIP_PROCESSING_EXTRACTING_IPS);
-        trace_destroy_packet(recvdpkt);
         return;
     }
 
     ret = parse_next_sip_message(sync->sipparser, NULL, NULL);
     if (ret == 0) {
-        trace_destroy_packet(recvdpkt);
         return;
     }
     if (ret < 0) {
         handle_bad_sip_update(sync, &recvdpkt, 1, SIP_PROCESSING_PARSING);
-        trace_destroy_packet(recvdpkt);
         return;
 
     }
@@ -1997,7 +1994,6 @@ static void sip_update_slow_path(collector_sync_voip_t *sync,
             &(baseirimsg.data.ipmmiri.ipfamily)) != 0) {
         handle_bad_sip_update(sync, &recvdpkt, 1,
                 SIP_PROCESSING_EXTRACTING_IPS);
-        trace_destroy_packet(recvdpkt);
         return;
     }
 
diff --git a/src/collector/sipparsing.c b/src/collector/sipparsing.c
@@ -69,6 +69,11 @@ static int parse_tcp_sip_packet(openli_sip_parser_t *p, libtrace_packet_t *pkt,
         return -1;
     }
 
+    /* Yet another keep alive pattern */
+    if (tcprem == 1 && memcmp(payload, "\x00", 1) == 0) {
+        return -1;
+    }
+
     ret = update_tcp_reassemble_stream(stream, (uint8_t *)payload, tcprem,
             ntohl(tcp->seq), pkt);
 
@@ -98,6 +103,11 @@ static int parse_udp_sip_packet(libtrace_udp_t *udp, uint32_t udprem) {
         return -1;
     }
 
+    /* Yet another keep alive pattern */
+    if (udprem == 1 && memcmp(payload, "\x00", 1) == 0) {
+        return -1;
+    }
+
     /* eXosip keep alive */
     if (udprem >= 4 && memcmp(payload, "\x6a\x61\x4b\x00", 4) == 0) {
         return -1;
diff --git a/src/configparser.c b/src/configparser.c
@@ -595,8 +595,9 @@ static int parse_agency_list(prov_intercept_conf_t *state, yaml_document_t *doc,
         newag->hi3_ipstr = NULL;
         newag->hi3_portstr = NULL;
         newag->agencyid = NULL;
+        newag->agencycc = NULL;
         newag->keepalivefreq = DEFAULT_AGENCY_KEEPALIVE_FREQ;
-        newag->keepalivewait = DEFAULT_AGENCY_KEEPALIVE_WAIT;
+        newag->keepalivewait = 0;
 
         for (pair = node->data.mapping.pairs.start;
                 pair < node->data.mapping.pairs.top; pair ++) {
@@ -640,6 +641,13 @@ static int parse_agency_list(prov_intercept_conf_t *state, yaml_document_t *doc,
                 SET_CONFIG_STRING_OPTION(newag->agencyid, value);
             }
 
+            if (key->type == YAML_SCALAR_NODE &&
+                    value->type == YAML_SCALAR_NODE &&
+                    strcmp((char *)key->data.scalar.value,
+                            "agencycountrycode") == 0) {
+                SET_CONFIG_STRING_OPTION(newag->agencycc, value);
+            }
+
             if (key->type == YAML_SCALAR_NODE &&
                     value->type == YAML_SCALAR_NODE &&
                     strcmp((char *)key->data.scalar.value,
@@ -664,6 +672,9 @@ static int parse_agency_list(prov_intercept_conf_t *state, yaml_document_t *doc,
             logger(LOG_INFO,
                     "OpenLI: 'pcapdisk' is a reserved agencyid, please rename to something else.");
             free(newag->agencyid);
+            if (newag->agencycc) {
+                free(newag->agencycc);
+            }
             newag->agencyid = NULL;
         }
 
@@ -684,6 +695,12 @@ static int parse_agency_list(prov_intercept_conf_t *state, yaml_document_t *doc,
                     strlen(prov_ag->ag->agencyid), prov_ag);
 
         } else {
+            if (newag->agencyid) {
+                free(newag->agencyid);
+            }
+            if (newag->agencycc) {
+                free(newag->agencycc);
+            }
             free(newag);
             logger(LOG_INFO, "OpenLI: LEA configuration was incomplete -- skipping.");
         }
diff --git a/src/mediator/handover.c b/src/mediator/handover.c
@@ -209,7 +209,7 @@ void trigger_handover_ka_failure(handover_t *ho) {
  *  @return -1 if an error occurs, 0 otherwise
  */
 int trigger_handover_keepalive(handover_t *ho, uint32_t mediator_id,
-        char *operator_id) {
+        char *operator_id, char *agency_cc) {
 
     wandder_encoded_result_t *kamsg;
     wandder_etsipshdr_data_t hdrdata;
@@ -237,17 +237,30 @@ int trigger_handover_keepalive(handover_t *ho, uint32_t mediator_id,
         /* Include the OpenLI version in the LIID field, so the LEAs can
          * identify which version of the software is being used by the
          * sender.
+         *
+         * PACKAGE_NAME and PACKAGE_VERSION come from config.h
          */
-        /* PACKAGE_NAME and PACKAGE_VERSION come from config.h */
-        snprintf(liidstring, 24, "%s-%s", PACKAGE_NAME, PACKAGE_VERSION);
+        if (agency_cc && strlen(agency_cc) == 2) {
+            hdrdata.delivcc = agency_cc;
+            hdrdata.authcc = agency_cc;
+        } else {
+            hdrdata.delivcc = "--";
+            hdrdata.authcc = "--";
+        }
+        hdrdata.delivcc_len = strlen(hdrdata.delivcc);
+        hdrdata.authcc_len = strlen(hdrdata.authcc);
+
+        /* Netherlands has a specific rule regarding the content of the
+         * LIID within keepalives
+         */
+        if (agency_cc && strcmp(agency_cc, "NL")==0) {
+        	snprintf(liidstring, 2, "-");
+        } else {
+        	snprintf(liidstring, 24, "%s-%s", PACKAGE_NAME, PACKAGE_VERSION);
+        }
         hdrdata.liid = liidstring;
         hdrdata.liid_len = strlen(hdrdata.liid);
 
-        hdrdata.authcc = "NA";
-        hdrdata.authcc_len = strlen(hdrdata.authcc);
-        hdrdata.delivcc = "NA";
-        hdrdata.delivcc_len = strlen(hdrdata.delivcc);
-
         if (operator_id) {
             hdrdata.operatorid = operator_id;
         } else {
@@ -256,7 +269,11 @@ int trigger_handover_keepalive(handover_t *ho, uint32_t mediator_id,
         hdrdata.operatorid_len = strlen(hdrdata.operatorid);
 
         /* Stupid 16 character limit... */
-        snprintf(elemstring, 16, "med-%u", mediator_id);
+        if (agency_cc && strcmp(agency_cc, "NL")==0) {
+        	snprintf(elemstring, 16, "%u", mediator_id);
+        } else {
+        	snprintf(elemstring, 16, "med-%u", mediator_id);
+        }
         hdrdata.networkelemid = elemstring;
         hdrdata.networkelemid_len = strlen(hdrdata.networkelemid);
 
diff --git a/src/mediator/handover.h b/src/mediator/handover.h
@@ -87,6 +87,7 @@ typedef struct handover {
 
 typedef struct mediator_agency {
     char *agencyid;
+    char *agencycc;
     int awaitingconfirm;
     int disabled;
     int disabled_msg;
@@ -135,7 +136,7 @@ void trigger_handover_ka_failure(handover_t *ho);
  *  @return -1 if an error occurs, 0 otherwise
  */
 int trigger_handover_keepalive(handover_t *ho, uint32_t mediator_id,
-        char *operator_id);
+        char *operator_id, char *agency_cc);
 
 /** Disconnects a single mediator handover connection to an LEA.
  *
diff --git a/src/mediator/lea_send_thread.c b/src/mediator/lea_send_thread.c
@@ -66,6 +66,11 @@ static void init_mediator_agency(mediator_agency_t *agency,
 
     agency->awaitingconfirm = 0;
     agency->agencyid = strdup(fromprov->agencyid);
+    if (fromprov->agencycc) {
+        agency->agencycc = strdup(fromprov->agencycc);
+    } else {
+        agency->agencycc = NULL;
+    }
     agency->disabled = 0;
     agency->disabled_msg = 0;
     agency->hi2 = create_new_handover(epollfd, fromprov->hi2_ipstr,
@@ -207,6 +212,12 @@ static void update_agency_handovers(mediator_agency_t *currag,
 
     }
 
+    if (currag->agencycc) {
+        free(currag->agencycc);
+    }
+    currag->agencycc = newag->agencycc;
+    newag->agencycc = NULL;
+
     if (currag->hi3 == NULL || currag->hi3->ipstr == NULL ||
             currag->hi3->portstr == NULL) {
         currag->hi3 = create_new_handover(epollfd, newag->hi3_ipstr,
@@ -494,7 +505,7 @@ static int agency_thread_epoll_event(lea_thread_state_t *state,
             /* we are due to send a keep alive */
             ho = (handover_t *)(mev->state);
             trigger_handover_keepalive(ho, state->mediator_id,
-                    state->operator_id);
+                    state->operator_id, state->agency.agencycc);
             ret = 0;
             break;
         case MED_EPOLL_KA_RESPONSE_TIMER:
@@ -1111,6 +1122,7 @@ void destroy_agency_thread_state(lea_thread_state_t *state) {
     }
     purge_liid_map(&(state->active_liids));
     free(state->agencyid);
+
     close(state->epoll_fd);
 }
 
diff --git a/src/netcomms.c b/src/netcomms.c
diff --git a/src/netcomms.h b/src/netcomms.h
diff --git a/src/provisioner/configwriter.c b/src/provisioner/configwriter.c
diff --git a/src/provisioner/hup_reload.c b/src/provisioner/hup_reload.c
diff --git a/src/provisioner/updateserver_jsoncreation.c b/src/provisioner/updateserver_jsoncreation.c
diff --git a/src/provisioner/updateserver_jsonparsing.c b/src/provisioner/updateserver_jsonparsing.c

-Original file line number
+Diff line change
@@ @@ -1,6 +1,6 @@ @@
 # Super primitive configure script
 -AC_INIT([openli],[1.1.7],[[email protected]])
 +AC_INIT([openli],[1.1.8],[[email protected]])
 AM_INIT_AUTOMAKE([subdir-objects])
 AC_CONFIG_SRCDIR(src/collector/collector.c)
Original file line number	Diff line number	Diff line change
`@@ -43,6 +43,9 @@ void free_liagency(liagency_t *lea) {`
`43`	`43`	`if (lea->agencyid) {`
`44`	`44`	`free(lea->agencyid);`
`45`	`45`	`}`
	`46`	`+ if (lea->agencycc) {`
	`47`	`+ free(lea->agencycc);`
	`48`	`+ }`
`46`	`49`	`free(lea);`
`47`	`50`	`}`
`48`	`51`