diff --git a/packages/zscaler_zpa/changelog.yml b/packages/zscaler_zpa/changelog.yml index 21ac852fb89..4999ad0ee56 100644 --- a/packages/zscaler_zpa/changelog.yml +++ b/packages/zscaler_zpa/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.23.2" + changes: + - description: Remove fields with empty values from incoming JSON document to avoid errors. + type: bugfix + link: https://github.com/elastic/integrations/pull/14493 - version: "1.23.1" changes: - description: Fix `related.host` to be `related.hosts`. diff --git a/packages/zscaler_zpa/data_stream/app_connector_status/elasticsearch/ingest_pipeline/default.yml b/packages/zscaler_zpa/data_stream/app_connector_status/elasticsearch/ingest_pipeline/default.yml index 7a98d1e023a..47f00d76b77 100644 --- a/packages/zscaler_zpa/data_stream/app_connector_status/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zscaler_zpa/data_stream/app_connector_status/elasticsearch/ingest_pipeline/default.yml @@ -13,6 +13,24 @@ processors: field: event.original target_field: json ignore_failure: true + - script: + description: Drops null/empty values in json recursively. + tag: remove_empty_values_from_json + lang: painless + source: | + boolean dropEmptyFields(Object object) { + if (object == null || object == '') { + return true; + } else if (object instanceof Map) { + ((Map) object).values().removeIf(value -> dropEmptyFields(value)); + return (((Map) object).size() == 0); + } else if (object instanceof List) { + ((List) object).removeIf(value -> dropEmptyFields(value)); + return (((List) object).length == 0); + } + return false; + } + dropEmptyFields(ctx.json); - append: field: event.category value: package @@ -28,7 +46,7 @@ processors: - E MMM dd HH:mm:ss yyyy - E MMM d HH:mm:ss yyyy - E MMM d HH:mm:ss yyyy - if: ctx.json?.LogTimestamp != null && ctx.json.LogTimestamp != '' + if: ctx.json?.LogTimestamp != null on_failure: - remove: field: json.LogTimestamp @@ -181,7 +199,7 @@ processors: target_field: zscaler_zpa.app_connector_status.timestamp.authentication formats: - ISO8601 - if: ctx.json?.TimestampAuthentication != null && ctx.json.TimestampAuthentication != '' + if: ctx.json?.TimestampAuthentication != null on_failure: - remove: field: json.TimestampAuthentication @@ -193,7 +211,7 @@ processors: target_field: zscaler_zpa.app_connector_status.timestamp.unauthentication formats: - ISO8601 - if: ctx.json?.TimestampUnAuthentication != null && ctx.json.TimestampUnAuthentication != '' + if: ctx.json?.TimestampUnAuthentication != null on_failure: - remove: field: json.TimestampUnAuthentication @@ -228,7 +246,7 @@ processors: - date: field: json.HostStartTime target_field: zscaler_zpa.app_connector_status.host_start_time - if: ctx.json?.HostStartTime != '0' && ctx.json.HostStartTime != null && ctx.json.HostStartTime != '' + if: ctx.json?.HostStartTime != '0' && ctx.json.HostStartTime != null formats: - UNIX on_failure: @@ -240,7 +258,7 @@ processors: - date: field: json.ConnectorStartTime target_field: zscaler_zpa.app_connector_status.connector_start_time - if: ctx.json?.ConnectorStartTime != '0' && ctx.json.ConnectorStartTime != null && ctx.json.ConnectorStartTime != '' + if: ctx.json?.ConnectorStartTime != '0' && ctx.json.ConnectorStartTime != null formats: - UNIX on_failure: @@ -287,6 +305,7 @@ processors: ignore_missing: true - script: description: Drops null/empty values recursively. + tag: remove_empty_values_from_event lang: painless source: | boolean dropEmptyFields(Object object) { diff --git a/packages/zscaler_zpa/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/zscaler_zpa/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 2135042703c..4c08cfd6958 100644 --- a/packages/zscaler_zpa/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zscaler_zpa/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -13,11 +13,29 @@ processors: field: event.original target_field: json ignore_failure: true + - script: + description: Drops null/empty values in json recursively. + tag: remove_empty_values_from_json + lang: painless + source: | + boolean dropEmptyFields(Object object) { + if (object == null || object == '') { + return true; + } else if (object instanceof Map) { + ((Map) object).values().removeIf(value -> dropEmptyFields(value)); + return (((Map) object).size() == 0); + } else if (object instanceof List) { + ((List) object).removeIf(value -> dropEmptyFields(value)); + return (((List) object).length == 0); + } + return false; + } + dropEmptyFields(ctx.json); - date: field: json.ModifiedTime formats: - ISO8601 - if: ctx.json?.ModifiedTime != null && ctx.json.ModifiedTime != '' + if: ctx.json?.ModifiedTime != null on_failure: - remove: field: json.ModifiedTime @@ -26,7 +44,7 @@ processors: value: '{{{_ingest.on_failure_message}}}' - date: field: json.CreationTime - if: ctx.json?.ModifiedTime == '' && ctx.json.CreationTime != null && ctx.json.CreationTime != '' + if: ctx.json?.ModifiedTime == null && ctx.json.CreationTime != null formats: - ISO8601 on_failure: @@ -43,7 +61,7 @@ processors: field: event.kind value: event - script: - if: ctx.json?.AuditOperationType != null && ctx.json.AuditOperationType != '' + if: ctx.json?.AuditOperationType != null lang: painless params: event_classification: @@ -338,6 +356,7 @@ processors: ignore_missing: true - script: description: Drops null/empty values recursively. + tag: remove_empty_values_from_event lang: painless source: | boolean dropEmptyFields(Object object) { diff --git a/packages/zscaler_zpa/data_stream/browser_access/elasticsearch/ingest_pipeline/default.yml b/packages/zscaler_zpa/data_stream/browser_access/elasticsearch/ingest_pipeline/default.yml index 2591194ace2..6942b400399 100644 --- a/packages/zscaler_zpa/data_stream/browser_access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zscaler_zpa/data_stream/browser_access/elasticsearch/ingest_pipeline/default.yml @@ -12,13 +12,31 @@ processors: - json: field: event.original target_field: json + - script: + description: Drops null/empty values in json recursively. + tag: remove_empty_values_from_json + lang: painless + source: | + boolean dropEmptyFields(Object object) { + if (object == null || object == '') { + return true; + } else if (object instanceof Map) { + ((Map) object).values().removeIf(value -> dropEmptyFields(value)); + return (((Map) object).size() == 0); + } else if (object instanceof List) { + ((List) object).removeIf(value -> dropEmptyFields(value)); + return (((List) object).length == 0); + } + return false; + } + dropEmptyFields(ctx.json); - date: field: json.LogTimestamp formats: - E MMM dd HH:mm:ss yyyy - E MMM d HH:mm:ss yyyy - E MMM d HH:mm:ss yyyy - if: ctx.json?.LogTimestamp != null && ctx.json.LogTimestamp != '' + if: ctx.json?.LogTimestamp != null on_failure: - remove: field: json.LogTimestamp @@ -200,7 +218,7 @@ processors: target_field: zscaler_zpa.browser_access.timestamp.request.receive.start formats: - ISO8601 - if: ctx.json?.TimestampRequestReceiveStart != null && ctx.json.TimestampRequestReceiveStart != '' + if: ctx.json?.TimestampRequestReceiveStart != null on_failure: - remove: field: json.TimestampRequestReceiveStart @@ -212,7 +230,7 @@ processors: target_field: zscaler_zpa.browser_access.timestamp.request.receive.header_finish formats: - ISO8601 - if: ctx.json?.TimestampRequestReceiveHeaderFinish != null && ctx.json.TimestampRequestReceiveHeaderFinish != '' + if: ctx.json?.TimestampRequestReceiveHeaderFinish != null on_failure: - remove: field: json.TimestampRequestReceiveHeaderFinish @@ -224,7 +242,7 @@ processors: target_field: zscaler_zpa.browser_access.timestamp.request.receive.finish formats: - ISO8601 - if: ctx.json?.TimestampRequestReceiveFinish != null && ctx.json.TimestampRequestReceiveFinish != '' + if: ctx.json?.TimestampRequestReceiveFinish != null on_failure: - remove: field: json.TimestampRequestReceiveFinish @@ -236,7 +254,7 @@ processors: target_field: zscaler_zpa.browser_access.timestamp.request.transmit.start formats: - ISO8601 - if: ctx.json?.TimestampRequestTransmitStart != null && ctx.json.TimestampRequestTransmitStart != '' + if: ctx.json?.TimestampRequestTransmitStart != null on_failure: - remove: field: json.TimestampRequestTransmitStart @@ -248,7 +266,7 @@ processors: target_field: zscaler_zpa.browser_access.timestamp.request.transmit.finish formats: - ISO8601 - if: ctx.json?.TimestampRequestTransmitFinish != null && ctx.json.TimestampRequestTransmitFinish != '' + if: ctx.json?.TimestampRequestTransmitFinish != null on_failure: - remove: field: json.TimestampRequestTransmitFinish @@ -260,7 +278,7 @@ processors: target_field: zscaler_zpa.browser_access.timestamp.response.receive.start formats: - ISO8601 - if: ctx.json?.TimestampResponseReceiveStart != null && ctx.json.TimestampResponseReceiveStart != '' + if: ctx.json?.TimestampResponseReceiveStart != null on_failure: - remove: field: json.TimestampResponseReceiveStart @@ -272,7 +290,7 @@ processors: target_field: zscaler_zpa.browser_access.timestamp.response.receive.finish formats: - ISO8601 - if: ctx.json?.TimestampResponseReceiveFinish != null && ctx.json.TimestampResponseReceiveFinish != '' + if: ctx.json?.TimestampResponseReceiveFinish != null on_failure: - remove: field: json.TimestampResponseReceiveFinish @@ -284,7 +302,7 @@ processors: target_field: zscaler_zpa.browser_access.timestamp.response.transmit.start formats: - ISO8601 - if: ctx.json?.TimestampResponseTransmitStart != null && ctx.json.TimestampResponseTransmitStart != '' + if: ctx.json?.TimestampResponseTransmitStart != null on_failure: - remove: field: json.TimestampResponseTransmitStart @@ -296,7 +314,7 @@ processors: target_field: zscaler_zpa.browser_access.timestamp.response.transmit.finish formats: - ISO8601 - if: ctx.json?.TimestampResponseTransmitFinish != null && ctx.json.TimestampResponseTransmitFinish != '' + if: ctx.json?.TimestampResponseTransmitFinish != null on_failure: - remove: field: json.TimestampResponseTransmitFinish @@ -358,6 +376,7 @@ processors: ignore_missing: true - script: description: Drops null/empty values recursively. + tag: remove_empty_values_from_event lang: painless source: | boolean dropEmptyFields(Object object) { diff --git a/packages/zscaler_zpa/data_stream/user_activity/elasticsearch/ingest_pipeline/default.yml b/packages/zscaler_zpa/data_stream/user_activity/elasticsearch/ingest_pipeline/default.yml index bda9ddd4cfb..ff170b2d8bf 100644 --- a/packages/zscaler_zpa/data_stream/user_activity/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zscaler_zpa/data_stream/user_activity/elasticsearch/ingest_pipeline/default.yml @@ -12,13 +12,30 @@ processors: - json: field: event.original target_field: json + - script: + description: Drops null/empty values in json recursively. + lang: painless + source: | + boolean dropEmptyFields(Object object) { + if (object == null || object == '') { + return true; + } else if (object instanceof Map) { + ((Map) object).values().removeIf(value -> dropEmptyFields(value)); + return (((Map) object).size() == 0); + } else if (object instanceof List) { + ((List) object).removeIf(value -> dropEmptyFields(value)); + return (((List) object).length == 0); + } + return false; + } + dropEmptyFields(ctx.json); - date: field: json.LogTimestamp formats: - E MMM dd HH:mm:ss yyyy - E MMM d HH:mm:ss yyyy - E MMM d HH:mm:ss yyyy - if: ctx.json?.LogTimestamp != null && ctx.json.LogTimestamp != '' + if: ctx.json?.LogTimestamp != null on_failure: - remove: field: json.LogTimestamp @@ -301,7 +318,7 @@ processors: target_field: zscaler_zpa.user_activity.timestamp.connection.start formats: - ISO8601 - if: ctx.json?.TimestampConnectionStart != null && ctx.json.TimestampConnectionStart != '' + if: ctx.json?.TimestampConnectionStart != null on_failure: - remove: field: json.TimestampConnectionStart @@ -313,7 +330,7 @@ processors: target_field: zscaler_zpa.user_activity.timestamp.connection.end formats: - ISO8601 - if: ctx.json?.TimestampConnectionEnd != null && ctx.json.TimestampConnectionEnd != '' + if: ctx.json?.TimestampConnectionEnd != null on_failure: - remove: field: json.TimestampConnectionEnd @@ -325,7 +342,7 @@ processors: target_field: zscaler_zpa.user_activity.timestamp.ca.tx formats: - ISO8601 - if: ctx.json?.TimestampCATx != null && ctx.json.TimestampCATx != '' + if: ctx.json?.TimestampCATx != null on_failure: - remove: field: json.TimestampCATx @@ -337,7 +354,7 @@ processors: target_field: zscaler_zpa.user_activity.timestamp.ca.rx formats: - ISO8601 - if: ctx.json?.TimestampCARx != null && ctx.json.TimestampCARx != '' + if: ctx.json?.TimestampCARx != null on_failure: - remove: field: json.TimestampCARx @@ -349,7 +366,7 @@ processors: target_field: zscaler_zpa.user_activity.timestamp.app_learn_start formats: - ISO8601 - if: ctx.json?.TimestampAppLearnStart != null && ctx.json.TimestampAppLearnStart != '' + if: ctx.json?.TimestampAppLearnStart != null on_failure: - remove: field: json.TimestampAppLearnStart @@ -361,7 +378,7 @@ processors: target_field: zscaler_zpa.user_activity.timestamp.zen.client.rx.first formats: - ISO8601 - if: ctx.json?.TimestampZENFirstRxClient != null && ctx.json.TimestampZENFirstRxClient != '' + if: ctx.json?.TimestampZENFirstRxClient != null on_failure: - remove: field: json.TimestampZENFirstRxClient @@ -373,7 +390,7 @@ processors: target_field: zscaler_zpa.user_activity.timestamp.zen.client.tx.first formats: - ISO8601 - if: ctx.json?.TimestampZENFirstTxClient != null && ctx.json.TimestampZENFirstTxClient != '' + if: ctx.json?.TimestampZENFirstTxClient != null on_failure: - remove: field: json.TimestampZENFirstTxClient @@ -385,7 +402,7 @@ processors: target_field: zscaler_zpa.user_activity.timestamp.zen.client.rx.last formats: - ISO8601 - if: ctx.json?.TimestampZENLastRxClient != null && ctx.json.TimestampZENLastRxClient != '' + if: ctx.json?.TimestampZENLastRxClient != null on_failure: - remove: field: json.TimestampZENLastRxClient @@ -397,7 +414,7 @@ processors: target_field: zscaler_zpa.user_activity.timestamp.zen.client.tx.last formats: - ISO8601 - if: ctx.json?.TimestampZENLastTxClient != null && ctx.json.TimestampZENLastTxClient != '' + if: ctx.json?.TimestampZENLastTxClient != null on_failure: - remove: field: json.TimestampZENLastTxClient @@ -409,7 +426,7 @@ processors: target_field: zscaler_zpa.user_activity.timestamp.connector_zen.setup_complete formats: - ISO8601 - if: ctx.json?.TimestampConnectorZENSetupComplete != null && ctx.json.TimestampConnectorZENSetupComplete != '' + if: ctx.json?.TimestampConnectorZENSetupComplete != null on_failure: - remove: field: json.TimestampConnectorZENSetupComplete @@ -421,7 +438,7 @@ processors: target_field: zscaler_zpa.user_activity.timestamp.zen.connector.rx.first formats: - ISO8601 - if: ctx.json?.TimestampZENFirstRxConnector != null && ctx.json.TimestampZENFirstRxConnector != '' + if: ctx.json?.TimestampZENFirstRxConnector != null on_failure: - remove: field: json.TimestampZENFirstRxConnector @@ -433,7 +450,7 @@ processors: target_field: zscaler_zpa.user_activity.timestamp.zen.connector.tx.first formats: - ISO8601 - if: ctx.json?.TimestampZENFirstTxConnector != null && ctx.json.TimestampZENFirstTxConnector != '' + if: ctx.json?.TimestampZENFirstTxConnector != null on_failure: - remove: field: json.TimestampZENFirstTxConnector @@ -445,7 +462,7 @@ processors: target_field: zscaler_zpa.user_activity.timestamp.zen.connector.rx.last formats: - ISO8601 - if: ctx.json?.TimestampZENLastRxConnector != null && ctx.json.TimestampZENLastRxConnector != '' + if: ctx.json?.TimestampZENLastRxConnector != null on_failure: - remove: field: json.TimestampZENLastRxConnector @@ -457,7 +474,7 @@ processors: target_field: zscaler_zpa.user_activity.timestamp.zen.connector.tx.last formats: - ISO8601 - if: ctx.json?.TimestampZENLastTxConnector != null && ctx.json.TimestampZENLastTxConnector != '' + if: ctx.json?.TimestampZENLastTxConnector != null on_failure: - remove: field: json.TimestampZENLastTxConnector diff --git a/packages/zscaler_zpa/data_stream/user_status/elasticsearch/ingest_pipeline/default.yml b/packages/zscaler_zpa/data_stream/user_status/elasticsearch/ingest_pipeline/default.yml index d95717abdd3..276c2bec171 100644 --- a/packages/zscaler_zpa/data_stream/user_status/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zscaler_zpa/data_stream/user_status/elasticsearch/ingest_pipeline/default.yml @@ -13,13 +13,31 @@ processors: field: event.original target_field: json ignore_failure: true + - script: + description: Drops null/empty values in json recursively. + tag: remove_empty_values_from_json + lang: painless + source: | + boolean dropEmptyFields(Object object) { + if (object == null || object == '') { + return true; + } else if (object instanceof Map) { + ((Map) object).values().removeIf(value -> dropEmptyFields(value)); + return (((Map) object).size() == 0); + } else if (object instanceof List) { + ((List) object).removeIf(value -> dropEmptyFields(value)); + return (((List) object).length == 0); + } + return false; + } + dropEmptyFields(ctx.json); - date: field: json.LogTimestamp formats: - E MMM dd HH:mm:ss yyyy - E MMM d HH:mm:ss yyyy - E MMM d HH:mm:ss yyyy - if: ctx.json?.LogTimestamp != null && ctx.json.LogTimestamp != '' + if: ctx.json?.LogTimestamp != null on_failure: - remove: field: json.LogTimestamp @@ -152,7 +170,7 @@ processors: target_field: zscaler_zpa.user_status.timestamp.authentication formats: - ISO8601 - if: ctx.json?.TimestampAuthentication != null && ctx.json.TimestampAuthentication != '' + if: ctx.json?.TimestampAuthentication != null on_failure: - remove: field: json.TimestampAuthentication @@ -164,7 +182,7 @@ processors: target_field: zscaler_zpa.user_status.timestamp.unauthentication formats: - ISO8601 - if: ctx.json?.TimestampAuthentication != null && ctx.json.TimestampUnAuthentication != '' + if: ctx.json?.TimestampUnAuthentication != null on_failure: - remove: field: json.TimestampUnAuthentication @@ -232,6 +250,7 @@ processors: if: ctx.json?.PosturesMiss instanceof List - script: description: Drops null/empty values recursively. + tag: remove_empty_values_from_event lang: painless source: | boolean dropEmptyFields(Object object) { diff --git a/packages/zscaler_zpa/manifest.yml b/packages/zscaler_zpa/manifest.yml index e716ee1a0b7..13212dc0aa8 100644 --- a/packages/zscaler_zpa/manifest.yml +++ b/packages/zscaler_zpa/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: zscaler_zpa title: Zscaler Private Access -version: "1.23.1" +version: "1.23.2" source: license: Elastic-2.0 description: Collect logs from Zscaler Private Access (ZPA) with Elastic Agent.