Merge branch 'select-timestamp-fmt' into develop

Author: salcock

src/agency.h

@@ -40,6 +40,7 @@
 #define DEFAULT_DIGEST_HASH_PDULIMIT 1000
 #define DEFAULT_DIGEST_SIGN_HASHLIMIT 15
 #define DEFAULT_DIGEST_HASH_METHOD OPENLI_DIGEST_HASH_ALGO_SHA256
+#define DEFAULT_AGENCY_TIMESTAMP_FORMAT OPENLI_ENCODED_TIMESTAMP_MICROSECONDS
 
 typedef enum {
     OPENLI_DIGEST_HASH_ALGO_SHA1 = 1,
@@ -60,6 +61,7 @@ typedef struct liagency {
     uint32_t keepalivewait;
     uint16_t handover_retry;
     uint32_t resend_window_kbs;
+    openli_timestamp_encoding_fmt_t time_fmt;
 
     openli_integrity_hash_method_t digest_hash_method;
     openli_integrity_hash_method_t digest_sign_method;
@@ -84,6 +86,7 @@ typedef struct liagency {
      a->handover_retry == b->handover_retry && \
      a->resend_window_kbs == b->resend_window_kbs && \
      a->digest_required == b->digest_required && \
+     a->time_fmt == b->time_fmt && \
          ((!a->digest_required) || ( \
              a->digest_hash_timeout == b->digest_hash_timeout && \
              a->digest_hash_pdulimit == b->digest_hash_pdulimit && \

src/collector/collector_publish.c

@@ -71,6 +71,7 @@ openli_export_recv_t *create_intercept_details_msg(intercept_common_t *common,
     expmsg->data.cept.encryptmethod = common->encrypt;
     expmsg->data.cept.cepttype = cepttype;
     expmsg->data.cept.targetagency = strdup(common->targetagency);
+    expmsg->data.cept.timefmt = common->time_fmt;
 
     if (common->xid_count > 0) {
         expmsg->data.cept.xids = calloc(common->xid_count, sizeof(uuid_t));

src/collector/collector_publish.h

@@ -211,6 +211,7 @@ typedef struct published_intercept_msg {
     int seqtrackerid;
     payload_encryption_method_t encryptmethod;
     char *encryptkey;
+    openli_timestamp_encoding_fmt_t timefmt;
     uuid_t *xids;
     size_t xid_count;
     openli_intercept_types_t cepttype;

src/collector/collector_seqtracker.c

@@ -210,6 +210,7 @@ static void track_new_intercept(seqtracker_thread_data_t *seqdata,
         intstate->details.authcc_len = strlen(cept->authcc);
         intstate->details.delivcc_len = strlen(cept->delivcc);
         intstate->details.encryptmethod = cept->encryptmethod;
+        intstate->details.timefmt = cept->timefmt;
         intstate->version ++;
 
     } else {
@@ -223,6 +224,7 @@ static void track_new_intercept(seqtracker_thread_data_t *seqdata,
         intstate->details.authcc_len = strlen(cept->authcc);
         intstate->details.delivcc_len = strlen(cept->delivcc);
         intstate->details.encryptmethod = cept->encryptmethod;
+        intstate->details.timefmt = cept->timefmt;
         intstate->cinsequencing = NULL;
         intstate->version = 0;
 
@@ -282,7 +284,7 @@ static int modify_tracked_intercept(seqtracker_thread_data_t *seqdata,
     intstate->details.delivcc_len = strlen(msg->delivcc);
 
     intstate->details.encryptmethod = msg->encryptmethod;
-
+    intstate->details.timefmt = msg->timefmt;
 
     remove_preencoded(seqdata, intstate);
     preencode_etsi_fields(seqdata, intstate);
@@ -364,6 +366,7 @@ static int run_encoding_job(seqtracker_thread_data_t *seqdata,
     job.cin = (int64_t)cin;
     job.cept_version = intstate->version;
     job.encryptmethod = intstate->details.encryptmethod;
+    job.timefmt = intstate->details.timefmt;
 
 	if (recvd->type == OPENLI_EXPORT_IPMMCC ||
 			recvd->type == OPENLI_EXPORT_IPCC ||

src/collector/email_worker.c

@@ -735,19 +735,9 @@ static int update_modified_email_intercept(openli_email_worker_t *state,
     }
 
     if (encodingchanged) {
-        expmsg = (openli_export_recv_t *)calloc(1, sizeof(openli_export_recv_t));
+        expmsg = create_intercept_details_msg(&(found->common),
+                OPENLI_INTERCEPT_TYPE_EMAIL);
         expmsg->type = OPENLI_EXPORT_INTERCEPT_CHANGED;
-        expmsg->data.cept.liid = strdup(found->common.liid);
-        expmsg->data.cept.authcc = strdup(found->common.authcc);
-        expmsg->data.cept.delivcc = strdup(found->common.delivcc);
-        expmsg->data.cept.seqtrackerid = found->common.seqtrackerid;
-        expmsg->data.cept.encryptmethod = found->common.encrypt;
-        if (found->common.encryptkey) {
-            expmsg->data.cept.encryptkey = strdup(found->common.encryptkey);
-        } else {
-            expmsg->data.cept.encryptkey = NULL;
-        }
-
         publish_openli_msg(state->zmq_pubsocks[found->common.seqtrackerid],
                 expmsg);
     }

src/collector/encoder_worker.c

@@ -756,21 +756,29 @@ static encoded_header_template_t *encode_templated_psheader(
     if (job->origreq->ts.tv_sec == 0) {
         gettimeofday(&(job->origreq->ts), NULL);
     }
-
     seqlen = DERIVE_INTEGER_LENGTH(job->seqno);
-    tvsec_len = DERIVE_INTEGER_LENGTH(job->origreq->ts.tv_sec);
-    tvusec_len = DERIVE_INTEGER_LENGTH(job->origreq->ts.tv_usec);
 
-    key = (job->cept_version << 24) + (seqlen << 16) +
-            (tvsec_len << 8) + tvusec_len;
+    if (job->timefmt == OPENLI_ENCODED_TIMESTAMP_MICROSECONDS) {
+        tvsec_len = DERIVE_INTEGER_LENGTH(job->origreq->ts.tv_sec);
+        tvusec_len = DERIVE_INTEGER_LENGTH(job->origreq->ts.tv_usec);
+
+        key = (job->cept_version << 24) + (seqlen << 16) +
+                (tvsec_len << 8) + tvusec_len;
+    } else if (job->timefmt == OPENLI_ENCODED_TIMESTAMP_GENERALIZED) {
+        // all libwandder generalized timestamps are encoded with the
+        // same length
+        key = (job->cept_version << 24) + (seqlen << 16);
+    } else {
+        return NULL;
+    }
 
     JLI(pval, t_set->headers, key);
     if (*pval == 0) {
         tplate = calloc(1, sizeof(encoded_header_template_t));
 
         if (etsili_create_header_template(encoder, job->preencoded,
                 (int64_t)job->cin, (int64_t)job->seqno, &(job->origreq->ts),
-                tplate) < 0) {
+                tplate, job->timefmt) < 0) {
             free(tplate);
             return NULL;
         }
@@ -781,7 +789,7 @@ static encoded_header_template_t *encode_templated_psheader(
         tplate = (encoded_header_template_t *)(*pval);
 
         if (etsili_update_header_template(tplate, (int64_t)job->seqno,
-                &(job->origreq->ts)) < 0) {
+                &(job->origreq->ts), job->timefmt) < 0) {
             return NULL;
         }
     }
@@ -798,7 +806,8 @@ static int encode_etsi(openli_encoder_t *enc, openli_encoding_job_t *job,
     saved_encoding_templates_t *t_set = NULL;
     encoded_header_template_t *hdr_tplate = NULL;
 
-    snprintf(keystr, 1000, "%s-%s", job->liid, job->cinstr);
+    snprintf(keystr, 1000, "%s-%s-%u", job->liid, job->cinstr,
+            job->timefmt);
     JSLI(pval, enc->saved_intercept_templates, (const uint8_t *)keystr);
     if ((*pval)) {
         t_set = (saved_encoding_templates_t *)(*pval);
@@ -902,7 +911,6 @@ static int process_job(openli_encoder_t *enc, void *socket) {
         } else if (job.origreq->type == OPENLI_EXPORT_RAW_IRI) {
             encode_rawip(&job, &(result[next]), OPENLI_PROTO_RAWIP_IRI);
         } else {
-
             if ((x = encode_etsi(enc, &job, &(result[next]))) <= 0) {
                 /* What do we do in the event of an error? */
                 if (x < 0) {

src/collector/etsiencoding/etsiencoding.c

@@ -71,7 +71,8 @@ static inline uint8_t encode_pspdu_sequence(uint8_t *space, uint8_t space_len,
 
 void encode_etsili_pshdr(wandder_encoder_t *encoder,
         wandder_etsipshdr_data_t *hdrdata, int64_t cin,
-        int64_t seqno, struct timeval *tv) {
+        int64_t seqno, struct timeval *tv,
+        openli_timestamp_encoding_fmt_t timefmt) {
 
     uint32_t tvclass = 1;       // timeOfInterception
 
@@ -120,26 +121,30 @@ void encode_etsili_pshdr(wandder_encoder_t *encoder,
     wandder_encode_next(encoder, WANDDER_TAG_INTEGER,
             WANDDER_CLASS_CONTEXT_PRIMITIVE, 4, &(seqno),
             sizeof(int64_t));
-    /*
-    wandder_encode_next(encoder, WANDDER_TAG_GENERALTIME,
-            WANDDER_CLASS_CONTEXT_PRIMITIVE, 5, tv,
-            sizeof(struct timeval));
-    */
+
+    if (timefmt == OPENLI_ENCODED_TIMESTAMP_GENERALIZED) {
+        wandder_encode_next(encoder, WANDDER_TAG_GENERALTIME,
+                WANDDER_CLASS_CONTEXT_PRIMITIVE, 5, tv,
+                sizeof(struct timeval));
+    }
 
     if (hdrdata->intpointid) {
         wandder_encode_next(encoder, WANDDER_TAG_PRINTABLE,
                 WANDDER_CLASS_CONTEXT_PRIMITIVE, 6, hdrdata->intpointid,
                 hdrdata->intpointid_len);
     }
 
-    ENC_CSEQUENCE(encoder, 7);
-    wandder_encode_next(encoder, WANDDER_TAG_INTEGER,
-            WANDDER_CLASS_CONTEXT_PRIMITIVE, 0, &(tv->tv_sec),
-            sizeof(tv->tv_sec));
-    wandder_encode_next(encoder, WANDDER_TAG_INTEGER,
-            WANDDER_CLASS_CONTEXT_PRIMITIVE, 1, &(tv->tv_usec),
-            sizeof(tv->tv_usec));
-    wandder_encode_endseq(encoder);
+
+    if (timefmt == OPENLI_ENCODED_TIMESTAMP_MICROSECONDS) {
+        ENC_CSEQUENCE(encoder, 7);
+        wandder_encode_next(encoder, WANDDER_TAG_INTEGER,
+                WANDDER_CLASS_CONTEXT_PRIMITIVE, 0, &(tv->tv_sec),
+                sizeof(tv->tv_sec));
+        wandder_encode_next(encoder, WANDDER_TAG_INTEGER,
+                WANDDER_CLASS_CONTEXT_PRIMITIVE, 1, &(tv->tv_usec),
+                sizeof(tv->tv_usec));
+        wandder_encode_endseq(encoder);
+    }
 
     wandder_encode_next(encoder, WANDDER_TAG_ENUM,
             WANDDER_CLASS_CONTEXT_PRIMITIVE, 8, &tvclass, sizeof(tvclass));

src/collector/etsiencoding/etsiencoding.h

@@ -140,6 +140,7 @@ typedef struct encoder_job {
     char *liid;
     uint8_t cept_version;
     payload_encryption_method_t encryptmethod;
+    openli_timestamp_encoding_fmt_t timefmt;
 } PACKED openli_encoding_job_t;
 
 void encode_ipaddress(wandder_encoder_t *encoder,
@@ -177,22 +178,27 @@ void clear_global_templates(Pvoid_t *saved_templates);
 
 void encode_etsili_pshdr(wandder_encoder_t *encoder,
         wandder_etsipshdr_data_t *hdrdata, int64_t cin,
-        int64_t seqno, struct timeval *tv);
+        int64_t seqno, struct timeval *tv,
+        openli_timestamp_encoding_fmt_t timefmt);
 
 /* defined in tripayload.c */
 wandder_encoded_result_t *encode_etsi_keepalive(wandder_encoder_t *encoder,
-        wandder_etsipshdr_data_t *hdrdata, int64_t seqno);
+        wandder_etsipshdr_data_t *hdrdata, int64_t seqno,
+        openli_timestamp_encoding_fmt_t timefmt);
+
 wandder_encoded_result_t *encode_etsi_integrity_check(
         wandder_encoder_t *encoder, wandder_etsipshdr_data_t *hdrdata,
         int64_t self_seqno, openli_integrity_hash_method_t hashmethod,
         uint32_t datatype, openli_proto_msgtype_t msgtype,
         uint8_t *checkval, unsigned int checkvallen,
-        int64_t *inclseqnos, size_t numseqnos);
+        int64_t *inclseqnos, size_t numseqnos,
+        openli_timestamp_encoding_fmt_t timefmt);
 
 /* defined in hi1notification.c */
 wandder_encoded_result_t *encode_etsi_hi1_notification(
         wandder_encoder_t *encoder, hi1_notify_data_t *not_data,
-        char *operatorid, char *shortopid);
+        char *operatorid, char *shortopid,
+        openli_timestamp_encoding_fmt_t timefmt);
 
 /* defined in encryptcontainer.c */
 int encrypt_aes_192_cbc(EVP_CIPHER_CTX *ctx, uint8_t *buf, uint16_t buflen,

src/collector/etsiencoding/hi1notification.c

@@ -96,7 +96,8 @@ static inline void encode_hi1_notification_body(wandder_encoder_t *encoder,
 
 wandder_encoded_result_t *encode_etsi_hi1_notification(
         wandder_encoder_t *encoder, hi1_notify_data_t *not_data,
-        char *operatorid, char *shortopid) {
+        char *operatorid, char *shortopid,
+        openli_timestamp_encoding_fmt_t timefmt) {
 
     struct timeval tv;
     wandder_etsipshdr_data_t hdrdata;
@@ -115,7 +116,8 @@ wandder_encoded_result_t *encode_etsi_hi1_notification(
     hdrdata.intpointid_len = 0;
 
     gettimeofday(&tv, NULL);
-    encode_etsili_pshdr(encoder, &hdrdata, 0, (int64_t)not_data->seqno, &tv);
+    encode_etsili_pshdr(encoder, &hdrdata, 0, (int64_t)not_data->seqno, &tv,
+            timefmt);
     encode_hi1_notification_body(encoder, not_data, shortopid);
     return wandder_encode_finish(encoder);
 }

src/collector/etsiencoding/tripayload.c

@@ -36,7 +36,8 @@ wandder_encoded_result_t *encode_etsi_integrity_check(
         int64_t self_seqno, openli_integrity_hash_method_t hashmethod,
         uint32_t checktype, openli_proto_msgtype_t msgtype,
         uint8_t *checkval, unsigned int checkvallen,
-        int64_t *inclseqnos, size_t numseqnos) {
+        int64_t *inclseqnos, size_t numseqnos,
+        openli_timestamp_encoding_fmt_t timefmt) {
 
 
     struct timeval tv;
@@ -45,7 +46,7 @@ wandder_encoded_result_t *encode_etsi_integrity_check(
     uint32_t hashalgo = 0;
 
     gettimeofday(&tv, NULL);
-    encode_etsili_pshdr(encoder, hdrdata, 0, self_seqno, &tv);
+    encode_etsili_pshdr(encoder, hdrdata, 0, self_seqno, &tv, timefmt);
     ENC_CSEQUENCE(encoder, 2);          // Payload
     ENC_CSEQUENCE(encoder, 2);          // TRIPayload
     ENC_CSEQUENCE(encoder, 0);          // integrityCheck
@@ -90,12 +91,13 @@ wandder_encoded_result_t *encode_etsi_integrity_check(
 }
 
 wandder_encoded_result_t *encode_etsi_keepalive(wandder_encoder_t *encoder,
-        wandder_etsipshdr_data_t *hdrdata, int64_t seqno) {
+        wandder_etsipshdr_data_t *hdrdata, int64_t seqno,
+        openli_timestamp_encoding_fmt_t timefmt) {
 
     struct timeval tv;
 
     gettimeofday(&tv, NULL);
-    encode_etsili_pshdr(encoder, hdrdata, 0, seqno, &tv);
+    encode_etsili_pshdr(encoder, hdrdata, 0, seqno, &tv, timefmt);
     ENC_CSEQUENCE(encoder, 2);          // Payload
     ENC_CSEQUENCE(encoder, 2);          // TRIPayload
     wandder_encode_next(encoder, WANDDER_TAG_NULL,