Skip to content

[zscaler_zpa] Cleanup empty field from json to avoid errors #14493

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
+135 −37
Open

[zscaler_zpa] Cleanup empty field from json to avoid errors #14493

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
bcbdc06
[zscaler_zpa] Cleanup empty field from json to avoid errors
SimonKoetting Jul 10, 2025
bfccd95
[zscaler_zpa] Cleanup empty field from json to avoid errors
SimonKoetting Jul 10, 2025
fa77642
Merge branch 'main' into zscaler_zpa_remove_empty_json_values
SimonKoetting Jul 10, 2025
e90b963
address review comments
SimonKoetting Jul 10, 2025
1a1a7d9
Merge branch 'zscaler_zpa_remove_empty_json_values' of github.com:Sim…
SimonKoetting Jul 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions packages/zscaler_zpa/changelog.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "1.23.2"
changes:
- description: Remove fields with empty values from incoming JSON document to avoid errors.
type: bugfix
link: https://github.com/elastic/integrations/pull/14493
- version: "1.23.1"
changes:
- description: Fix `related.host` to be `related.hosts`.
Expand Down
29 changes: 24 additions & 5 deletions ...es/zscaler_zpa/data_stream/app_connector_status/elasticsearch/ingest_pipeline/default.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,24 @@ processors:
field: event.original
target_field: json
ignore_failure: true
- script:
description: Drops null/empty values in json recursively.
tag: remove_empty_values_from_json
lang: painless
source: |
boolean dropEmptyFields(Object object) {
if (object == null || object == '') {
return true;
} else if (object instanceof Map) {
((Map) object).values().removeIf(value -> dropEmptyFields(value));
return (((Map) object).size() == 0);
} else if (object instanceof List) {
((List) object).removeIf(value -> dropEmptyFields(value));
return (((List) object).length == 0);
}
return false;
}
dropEmptyFields(ctx.json);
- append:
field: event.category
value: package
Expand All @@ -28,7 +46,7 @@ processors:
- E MMM dd HH:mm:ss yyyy
- E MMM d HH:mm:ss yyyy
- E MMM d HH:mm:ss yyyy
if: ctx.json?.LogTimestamp != null && ctx.json.LogTimestamp != ''
if: ctx.json?.LogTimestamp != null
on_failure:
- remove:
field: json.LogTimestamp
Expand Down Expand Up @@ -181,7 +199,7 @@ processors:
target_field: zscaler_zpa.app_connector_status.timestamp.authentication
formats:
- ISO8601
if: ctx.json?.TimestampAuthentication != null && ctx.json.TimestampAuthentication != ''
if: ctx.json?.TimestampAuthentication != null
on_failure:
- remove:
field: json.TimestampAuthentication
Expand All @@ -193,7 +211,7 @@ processors:
target_field: zscaler_zpa.app_connector_status.timestamp.unauthentication
formats:
- ISO8601
if: ctx.json?.TimestampUnAuthentication != null && ctx.json.TimestampUnAuthentication != ''
if: ctx.json?.TimestampUnAuthentication != null
on_failure:
- remove:
field: json.TimestampUnAuthentication
Expand Down Expand Up @@ -228,7 +246,7 @@ processors:
- date:
field: json.HostStartTime
target_field: zscaler_zpa.app_connector_status.host_start_time
if: ctx.json?.HostStartTime != '0' && ctx.json.HostStartTime != null && ctx.json.HostStartTime != ''
if: ctx.json?.HostStartTime != '0' && ctx.json.HostStartTime != null
formats:
- UNIX
on_failure:
Expand All @@ -240,7 +258,7 @@ processors:
- date:
field: json.ConnectorStartTime
target_field: zscaler_zpa.app_connector_status.connector_start_time
if: ctx.json?.ConnectorStartTime != '0' && ctx.json.ConnectorStartTime != null && ctx.json.ConnectorStartTime != ''
if: ctx.json?.ConnectorStartTime != '0' && ctx.json.ConnectorStartTime != null
formats:
- UNIX
on_failure:
Expand Down Expand Up @@ -287,6 +305,7 @@ processors:
ignore_missing: true
- script:
description: Drops null/empty values recursively.
tag: remove_empty_values_from_event
lang: painless
source: |
boolean dropEmptyFields(Object object) {
Expand Down
25 changes: 22 additions & 3 deletions packages/zscaler_zpa/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,29 @@ processors:
field: event.original
target_field: json
ignore_failure: true
- script:
description: Drops null/empty values in json recursively.
tag: remove_empty_values_from_json
lang: painless
source: |
boolean dropEmptyFields(Object object) {
if (object == null || object == '') {
return true;
} else if (object instanceof Map) {
((Map) object).values().removeIf(value -> dropEmptyFields(value));
return (((Map) object).size() == 0);
} else if (object instanceof List) {
((List) object).removeIf(value -> dropEmptyFields(value));
return (((List) object).length == 0);
}
return false;
}
dropEmptyFields(ctx.json);
- date:
field: json.ModifiedTime
formats:
- ISO8601
if: ctx.json?.ModifiedTime != null && ctx.json.ModifiedTime != ''
if: ctx.json?.ModifiedTime != null
on_failure:
- remove:
field: json.ModifiedTime
Expand All @@ -26,7 +44,7 @@ processors:
value: '{{{_ingest.on_failure_message}}}'
- date:
field: json.CreationTime
if: ctx.json?.ModifiedTime == '' && ctx.json.CreationTime != null && ctx.json.CreationTime != ''
if: ctx.json?.ModifiedTime == null && ctx.json.CreationTime != null
formats:
- ISO8601
on_failure:
Expand All @@ -43,7 +61,7 @@ processors:
field: event.kind
value: event
- script:
if: ctx.json?.AuditOperationType != null && ctx.json.AuditOperationType != ''
if: ctx.json?.AuditOperationType != null
lang: painless
params:
event_classification:
Expand Down Expand Up @@ -338,6 +356,7 @@ processors:
ignore_missing: true
- script:
description: Drops null/empty values recursively.
tag: remove_empty_values_from_event
lang: painless
source: |
boolean dropEmptyFields(Object object) {
Expand Down
39 changes: 29 additions & 10 deletions packages/zscaler_zpa/data_stream/browser_access/elasticsearch/ingest_pipeline/default.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,31 @@ processors:
- json:
field: event.original
target_field: json
- script:
description: Drops null/empty values in json recursively.
tag: remove_empty_values_from_json
lang: painless
source: |
boolean dropEmptyFields(Object object) {
if (object == null || object == '') {
return true;
} else if (object instanceof Map) {
((Map) object).values().removeIf(value -> dropEmptyFields(value));
return (((Map) object).size() == 0);
} else if (object instanceof List) {
((List) object).removeIf(value -> dropEmptyFields(value));
return (((List) object).length == 0);
}
return false;
}
dropEmptyFields(ctx.json);
- date:
field: json.LogTimestamp
formats:
- E MMM dd HH:mm:ss yyyy
- E MMM d HH:mm:ss yyyy
- E MMM d HH:mm:ss yyyy
if: ctx.json?.LogTimestamp != null && ctx.json.LogTimestamp != ''
if: ctx.json?.LogTimestamp != null
on_failure:
- remove:
field: json.LogTimestamp
Expand Down Expand Up @@ -200,7 +218,7 @@ processors:
target_field: zscaler_zpa.browser_access.timestamp.request.receive.start
formats:
- ISO8601
if: ctx.json?.TimestampRequestReceiveStart != null && ctx.json.TimestampRequestReceiveStart != ''
if: ctx.json?.TimestampRequestReceiveStart != null
on_failure:
- remove:
field: json.TimestampRequestReceiveStart
Expand All @@ -212,7 +230,7 @@ processors:
target_field: zscaler_zpa.browser_access.timestamp.request.receive.header_finish
formats:
- ISO8601
if: ctx.json?.TimestampRequestReceiveHeaderFinish != null && ctx.json.TimestampRequestReceiveHeaderFinish != ''
if: ctx.json?.TimestampRequestReceiveHeaderFinish != null
on_failure:
- remove:
field: json.TimestampRequestReceiveHeaderFinish
Expand All @@ -224,7 +242,7 @@ processors:
target_field: zscaler_zpa.browser_access.timestamp.request.receive.finish
formats:
- ISO8601
if: ctx.json?.TimestampRequestReceiveFinish != null && ctx.json.TimestampRequestReceiveFinish != ''
if: ctx.json?.TimestampRequestReceiveFinish != null
on_failure:
- remove:
field: json.TimestampRequestReceiveFinish
Expand All @@ -236,7 +254,7 @@ processors:
target_field: zscaler_zpa.browser_access.timestamp.request.transmit.start
formats:
- ISO8601
if: ctx.json?.TimestampRequestTransmitStart != null && ctx.json.TimestampRequestTransmitStart != ''
if: ctx.json?.TimestampRequestTransmitStart != null
on_failure:
- remove:
field: json.TimestampRequestTransmitStart
Expand All @@ -248,7 +266,7 @@ processors:
target_field: zscaler_zpa.browser_access.timestamp.request.transmit.finish
formats:
- ISO8601
if: ctx.json?.TimestampRequestTransmitFinish != null && ctx.json.TimestampRequestTransmitFinish != ''
if: ctx.json?.TimestampRequestTransmitFinish != null
on_failure:
- remove:
field: json.TimestampRequestTransmitFinish
Expand All @@ -260,7 +278,7 @@ processors:
target_field: zscaler_zpa.browser_access.timestamp.response.receive.start
formats:
- ISO8601
if: ctx.json?.TimestampResponseReceiveStart != null && ctx.json.TimestampResponseReceiveStart != ''
if: ctx.json?.TimestampResponseReceiveStart != null
on_failure:
- remove:
field: json.TimestampResponseReceiveStart
Expand All @@ -272,7 +290,7 @@ processors:
target_field: zscaler_zpa.browser_access.timestamp.response.receive.finish
formats:
- ISO8601
if: ctx.json?.TimestampResponseReceiveFinish != null && ctx.json.TimestampResponseReceiveFinish != ''
if: ctx.json?.TimestampResponseReceiveFinish != null
on_failure:
- remove:
field: json.TimestampResponseReceiveFinish
Expand All @@ -284,7 +302,7 @@ processors:
target_field: zscaler_zpa.browser_access.timestamp.response.transmit.start
formats:
- ISO8601
if: ctx.json?.TimestampResponseTransmitStart != null && ctx.json.TimestampResponseTransmitStart != ''
if: ctx.json?.TimestampResponseTransmitStart != null
on_failure:
- remove:
field: json.TimestampResponseTransmitStart
Expand All @@ -296,7 +314,7 @@ processors:
target_field: zscaler_zpa.browser_access.timestamp.response.transmit.finish
formats:
- ISO8601
if: ctx.json?.TimestampResponseTransmitFinish != null && ctx.json.TimestampResponseTransmitFinish != ''
if: ctx.json?.TimestampResponseTransmitFinish != null
on_failure:
- remove:
field: json.TimestampResponseTransmitFinish
Expand Down Expand Up @@ -358,6 +376,7 @@ processors:
ignore_missing: true
- script:
description: Drops null/empty values recursively.
tag: remove_empty_values_from_event
lang: painless
source: |
boolean dropEmptyFields(Object object) {
Expand Down
Loading