diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ca50105709f..471aa1eb086 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -436,6 +436,7 @@ /packages/ti_cif3 @elastic/security-service-integrations /packages/ti_crowdstrike @elastic/security-service-integrations /packages/ti_cybersixgill @elastic/security-service-integrations +/packages/ti_cyware_threat_intelligence @elastic/security-service-integrations /packages/ti_domaintools @elastic/security-service-integrations /packages/ti_eclecticiq @elastic/security-service-integrations /packages/ti_eset @elastic/security-service-integrations diff --git a/packages/ti_cyware_intel_exchange/_dev/build/build.yml b/packages/ti_cyware_intel_exchange/_dev/build/build.yml new file mode 100644 index 00000000000..97fc8aa10cd --- /dev/null +++ b/packages/ti_cyware_intel_exchange/_dev/build/build.yml @@ -0,0 +1,3 @@ +dependencies: + ecs: + reference: git@v8.17.0 diff --git a/packages/ti_cyware_intel_exchange/_dev/build/docs/README.md b/packages/ti_cyware_intel_exchange/_dev/build/docs/README.md new file mode 100644 index 00000000000..ded23ca8e2a --- /dev/null +++ b/packages/ti_cyware_intel_exchange/_dev/build/docs/README.md @@ -0,0 +1,62 @@ +# Cyware Intel Exchange + +## Overview + +[Cyware Intel Exchange](https://www.cyware.com/products/intel-exchange) is an intelligent client-server exchange that leverages advanced technologies like Artificial Intelligence and Machine Learning to automatically ingest, analyze, correlate and act upon the threat data ingested from multiple external sources and internally deployed security tools. + +## Data streams + +The Cyware Intel Exchange integration collects the following events: +- **[Indicator](https://ctixapiv3.cyware.com/rules/save-result-set/retrieve-saved-result-set-data)** - This fetches all the saved result set data for conditional IOCs present in the application.. + +## Requirements + +### Agentless-enabled integration +Agentless integrations allow you to collect data without having to manage Elastic Agent in your cloud. They make manual agent deployment unnecessary, so you can focus on your data instead of the agent that collects it. For more information, refer to [Agentless integrations](https://www.elastic.co/guide/en/serverless/current/security-agentless-integrations.html) and the [Agentless integrations FAQ](https://www.elastic.co/guide/en/serverless/current/agentless-integration-troubleshooting.html). + +Agentless deployments are only supported in Elastic Serverless and Elastic Cloud environments. This functionality is in beta and is subject to change. Beta features are not subject to the support SLA of official GA features. + +### Agent-based installation +Elastic Agent must be installed. For more details, check the Elastic Agent [installation instructions](docs-content://reference/fleet/install-elastic-agents.md). You can install only one Elastic Agent per host. + +## Compatibility + +For Rest API, this module has been tested against the **[CTIX API v3](https://ctixapiv3.cyware.com/intel-exchange-api-reference)** version. + +## Setup + +**Note** - Before you start the setup, ensure that you have **Create** and **Update** permissions for **CTIX Integrators**. + +### Follow below steps to generate Open API credentials for collecting data from the CTIX API: + +1. Go to **Administration** > **Integration Management**. +2. In **Third Party Developers**, click **CTIX Integrators**. +3. Click **Add New**. Enter the following details: + - **Name**: Enter a unique name for the API credentials in 50 characters. + - **Description**: Enter a description for the credentials within 1000 characters. + - **Expiry Date**: Select an expiry date for open API keys. To apply an expiration date for the credentials, you can select **Expires On** and select the date. To ensure the credentials never expire, you can select **Never Expire**. +4. Click **Add New**. +5. Click **Download** to download the API credentials in CSV format. You can also click **Copy** to copy the endpoint URL, secret key, and access ID. + +For more details, refer to the [Authentication](https://ctixapiv3.cyware.com/authentication) documentation and the guide on how to [Generate Open API Credentials](https://techdocs.cyware.com/en/299670-447852-configure-open-api.html). + +### Enable the integration in Elastic + +1. In Kibana navigate to **Management** > **Integrations**. +2. In the search top bar, type **Cyware Intel Exchange**. +3. Select the **Cyware Intel Exchange** integration afrom the search results. +4. Click on the "Add Cyware Intel Exchange" button to add the integration. +5. Add all the required integration configuration parameters: URL, Access ID and Secret Key. +6. Save the integration. + +## Logs reference + +### Indicator + +This is the `Indicator` dataset. + +#### Example + +{{event "indicator"}} + +{{fields "indicator"}} diff --git a/packages/ti_cyware_intel_exchange/_dev/deploy/docker/docker-compose.yml b/packages/ti_cyware_intel_exchange/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 00000000000..e64e9aa70ff --- /dev/null +++ b/packages/ti_cyware_intel_exchange/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,14 @@ +version: '3.8' +services: + indicator-api: + image: docker.elastic.co/observability/stream:v0.18.0 + ports: + - 8080 + volumes: + - ./files:/files:ro + environment: + PORT: 8080 + command: + - http-server + - --addr=:8080 + - --config=/files/config.yml diff --git a/packages/ti_cyware_intel_exchange/_dev/deploy/docker/files/config.yml b/packages/ti_cyware_intel_exchange/_dev/deploy/docker/files/config.yml new file mode 100644 index 00000000000..7fc56144a6e --- /dev/null +++ b/packages/ti_cyware_intel_exchange/_dev/deploy/docker/files/config.yml @@ -0,0 +1,617 @@ +rules: + - path: /ingestion/rules/save_result_set/ + methods: ["GET"] + query_params: + page_size: 2 + page: 3 + AccessID: "{AccessID:.*}" + version: "{version:.*}" + from_timestamp: "{from_timestamp:.*}" + Expires: "{Expires:.*}" + Signature: "{Signature:.*}" + responses: + - status_code: 200 + body: |- + {{ minify_json ` + { + "next": null, + "previous": "ingestion/rules/save_result_set/?page=3&page_size=2&from_timestamp=1747947825&version=v3", + "page_size": 2, + "total": 8, + "results": [ + { + "id": "82e7f1ce-36ce-41d2-9cca-ba56b2a6febb", + "ctix_created": 1751632663, + "ctix_modified": 1751632663, + "version": "v3", + "ctix_tags": [ + { + "id": "ec16aa3d-617c-468c-99a8-6b21af0a8508", + "name": "(http_inspect) unknown Content-Encoding used (Firepower)", + "created": 1748526014, + "modified": 1748526014, + "tag_type": { + "id": "source", + "name": "Source" + }, + "is_active": true, + "created_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@default.com", + "last_name": "Default", + "first_name": "System" + }, + "colour_code": "#45505E", + "modified_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@default.com", + "last_name": "Default", + "first_name": "System" + } + }, + { + "id": "0bdcf6dc-546f-4021-a292-1f5313ff8d55", + "name": "(smtp) unknown command (Firepower)", + "created": 1748612481, + "modified": 1748612481, + "tag_type": { + "id": "source", + "name": "Source" + }, + "is_active": true, + "created_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@default.com", + "last_name": "Default", + "first_name": "System" + }, + "colour_code": "#45505E", + "modified_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@default.com", + "last_name": "Default", + "first_name": "System" + } + } + ], + "data": [ + { + "id": "f4e91f4e-65f7-40d5-95d2-be7de434c438", + "name": "1.128.0.0", + "tags": [ + "cve-2024-9047", + "Feed", + "vulnerability exploitation", + "china-nexus", + "cve-2024-51378", + "backdoor", + "cobalt strike", + "vshell", + "cve-2024-27199", + "cve-2024-56145", + "cve-2021-22205", + "brute ratel", + "cve-2024-51567", + "bypassboss", + "multi-industry targeting", + "custom tools", + "cve-2024-27198", + ".net", + "sql injection", + "cve-2025-31324", + "apt", + "cve-2017-9805", + "dll sideloading", + "pulsepack" + ], + "country": "Hong Kong", + "created": 1748342106.0, + "sources": [ + { + "tlp": "WHITE", + "name": "Vault", + "score": 100, + "last_seen": null, + "first_seen": 1748342106 + } + ], + "ctix_tlp": null, + "ioc_type": "ipv4-addr", + "modified": 1748354570.904088, + "sdo_name": "1.128.0.0", + "sdo_type": "indicator", + "severity": "UNKNOWN", + "ctix_score": 0, + "is_revoked": false, + "source_tlp": "NONE", + "valid_from": 1748342106.0, + "analyst_tlp": null, + "is_actioned": false, + "is_reviewed": false, + "valid_until": 1750932000.0, + "ctix_created": 1748354583.272796, + "is_whitelist": false, + "analyst_score": null, + "ctix_modified": 1751632658.306212, + "custom_scores": null, + "is_deprecated": true, + "indicator_type": { + "type": "ipv4-addr", + "attribute_field": "value" + }, + "is_false_positive": false + } + ], + "timestamp": 1751632663, + "title": "Package From CTIX" + } + ] + } + `}} + - path: /ingestion/rules/save_result_set/ + methods: ["GET"] + query_params: + page_size: 2 + page: 2 + AccessID: "{AccessID:.*}" + version: "{version:.*}" + from_timestamp: "{from_timestamp:.*}" + Expires: "{Expires:.*}" + Signature: "{Signature:.*}" + responses: + - status_code: 200 + body: |- + {{ minify_json ` + { + "next": "ingestion/rules/save_result_set/?page=3&page_size=2&from_timestamp=1747947825&version=v3", + "previous": "ingestion/rules/save_result_set/?page=1&page_size=2&from_timestamp=1747947825&version=v3", + "page_size": 2, + "total": 8, + "results": [ + { + "id": "2fd17a6b-3939-485c-80ed-84d469da1b93", + "ctix_created": 1751291553, + "ctix_modified": 1751291553, + "version": "v3", + "ctix_tags": [ + { + "id": "ec16aa3d-617c-468c-99a8-6b21af0a8508", + "name": "(http_inspect) unknown Content-Encoding used (Firepower)", + "created": 1748526014, + "modified": 1748526014, + "tag_type": { + "id": "source", + "name": "Source" + }, + "is_active": true, + "created_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@default.com", + "last_name": "Default", + "first_name": "System" + }, + "colour_code": "#45505E", + "modified_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@default.com", + "last_name": "Default", + "first_name": "System" + } + }, + { + "id": "0bdcf6dc-546f-4021-a292-1f5313ff8d55", + "name": "(smtp) unknown command (Firepower)", + "created": 1748612481, + "modified": 1748612481, + "tag_type": { + "id": "source", + "name": "Source" + }, + "is_active": true, + "created_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@default.com", + "last_name": "Default", + "first_name": "System" + }, + "colour_code": "#45505E", + "modified_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@default.com", + "last_name": "Default", + "first_name": "System" + } + } + ], + "data": [ + { + "id": "6b0e2522-979d-4754-84ec-bd3b4ec19cbf", + "name": "digital", + "tags": [ + "URL spoofing", + "domain redirection", + "cryptocurrency", + "brand impersonation" + ], + "country": "India", + "created": 1746812972.0, + "sources": [ + { + "tlp": "WHITE", + "name": "Vault", + "score": 100, + "last_seen": null, + "first_seen": 1746812972 + } + ], + "ctix_tlp": null, + "ioc_type": "domain-name", + "modified": 1748354676.710264, + "sdo_name": "digital", + "sdo_type": "indicator", + "severity": "UNKNOWN", + "ctix_score": 90, + "is_revoked": false, + "source_tlp": "NONE", + "valid_from": 1746812972.0, + "analyst_tlp": null, + "is_actioned": false, + "is_reviewed": false, + "valid_until": null, + "ctix_created": 1748354691.634547, + "is_whitelist": false, + "analyst_score": null, + "ctix_modified": 1751291551.915257, + "custom_scores": null, + "is_deprecated": false, + "indicator_type": { + "type": "domain-name", + "attribute_field": "value" + }, + "is_false_positive": false + } + ], + "timestamp": 1751291553, + "title": "Package From CTIX" + }, + { + "id": "9439c3e8-4bea-4f34-8ab3-1c5c02c2503f", + "ctix_created": 1751301210, + "ctix_modified": 1751301210, + "version": "v3", + "ctix_tags": [ + { + "id": "ec16aa3d-617c-468c-99a8-6b21af0a8508", + "name": "(http_inspect) unknown Content-Encoding used (Firepower)", + "created": 1748526014, + "modified": 1748526014, + "tag_type": { + "id": "source", + "name": "Source" + }, + "is_active": true, + "created_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@default.com", + "last_name": "Default", + "first_name": "System" + }, + "colour_code": "#45505E", + "modified_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@default.com", + "last_name": "Default", + "first_name": "System" + } + }, + { + "id": "0bdcf6dc-546f-4021-a292-1f5313ff8d55", + "name": "(smtp) unknown command (Firepower)", + "created": 1748612481, + "modified": 1748612481, + "tag_type": { + "id": "source", + "name": "Source" + }, + "is_active": true, + "created_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@default.com", + "last_name": "Default", + "first_name": "System" + }, + "colour_code": "#45505E", + "modified_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@default.com", + "last_name": "Default", + "first_name": "System" + } + } + ], + "data": [ + { + "id": "375b30ef-74c8-4ad2-8570-c01d1d90e017", + "name": "digital", + "tags": [ + "cryptocurrency", + "domain redirection", + "URL spoofing", + "brand impersonation" + ], + "country": null, + "created": 1746812972.0, + "sources": [ + { + "tlp": "WHITE", + "name": "Vault", + "score": 100, + "last_seen": null, + "first_seen": 1746812972 + } + ], + "ctix_tlp": null, + "ioc_type": "domain-name", + "modified": 1748354676.709378, + "sdo_name": "digital", + "sdo_type": "indicator", + "severity": "UNKNOWN", + "ctix_score": 90, + "is_revoked": false, + "source_tlp": "NONE", + "valid_from": 1746812972.0, + "analyst_tlp": null, + "is_actioned": false, + "is_reviewed": false, + "valid_until": null, + "ctix_created": 1748354691.631653, + "is_whitelist": false, + "analyst_score": null, + "ctix_modified": 1751301206.388778, + "custom_scores": null, + "is_deprecated": false, + "indicator_type": { + "type": "domain-name", + "attribute_field": "value" + }, + "is_false_positive": false + } + ], + "timestamp": 1751301210, + "title": "Package From CTIX" + } + ] + } + `}} + - path: /ingestion/rules/save_result_set/ + methods: ["GET"] + query_params: + page_size: 2 + page: 1 + AccessID: "{AccessID:.*}" + version: "{version:.*}" + from_timestamp: "{from_timestamp:.*}" + Expires: "{Expires:.*}" + Signature: "{Signature:.*}" + responses: + - status_code: 200 + body: |- + {{ minify_json ` + { + "next": "ingestion/rules/save_result_set/?page=2&page_size=2&from_timestamp=1747947825&version=v3", + "previous": null, + "page_size": 2, + "total": 4, + "results": [ + { + "id": "4e0a8868-f812-4139-b715-7e69850360db", + "ctix_created": 1751279301, + "ctix_modified": 1751279301, + "version": "v3", + "ctix_tags": [ + { + "id": "ec16aa3d-617c-468c-99a8-6b21af0a8508", + "name": "(http_inspect) unknown Content-Encoding used (Firepower)", + "created": 1748526014, + "modified": 1748526014, + "tag_type": { + "id": "source", + "name": "Source" + }, + "is_active": true, + "created_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@example.com", + "last_name": "Default", + "first_name": "System" + }, + "colour_code": "#45505E", + "modified_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@default.com", + "last_name": "Default", + "first_name": "System" + } + }, + { + "id": "0bdcf6dc-546f-4021-a292-1f5313ff8d55", + "name": "(smtp) unknown command (Firepower)", + "created": 1748612481, + "modified": 1748612481, + "tag_type": { + "id": "source", + "name": "Source" + }, + "is_active": true, + "created_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@default.com", + "last_name": "Default", + "first_name": "System" + }, + "colour_code": "#45505E", + "modified_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@default.com", + "last_name": "Default", + "first_name": "System" + } + } + ], + "data": [ + { + "id": "f36749f2-776f-4153-b840-08bad5fb18b1", + "name": "e8c5c5829b630dcf61b55f271ac6c085", + "tags": [ + "brand impersonation", + "cryptocurrency", + "Apple" + ], + "country": null, + "created": 1746812972.0, + "sources": [ + { + "tlp": "WHITE", + "name": "Vault", + "score": 100, + "last_seen": null, + "first_seen": 1746812972 + } + ], + "ctix_tlp": null, + "ioc_type": "file", + "modified": 1748354676.716103, + "sdo_name": "e8c5c5829b630dcf61b55f271ac6c085", + "sdo_type": "indicator", + "severity": "UNKNOWN", + "ctix_score": 90, + "is_revoked": false, + "source_tlp": "NONE", + "valid_from": 1746812972.0, + "analyst_tlp": null, + "is_actioned": false, + "is_reviewed": false, + "valid_until": null, + "ctix_created": 1748354691.651628, + "is_whitelist": false, + "analyst_score": null, + "ctix_modified": 1751279296.273555, + "custom_scores": null, + "is_deprecated": false, + "indicator_type": { + "type": "file", + "attribute_field": "MD5" + }, + "is_false_positive": false + } + ], + "timestamp": 1751279301, + "title": "Package From CTIX" + }, + { + "id": "6db685d1-8ca6-4efc-b5cd-f8dcbf070ab8", + "ctix_created": 1751280122, + "ctix_modified": 1751280122, + "version": "v3", + "ctix_tags": [ + { + "id": "ec16aa3d-617c-468c-99a8-6b21af0a8508", + "name": "(http_inspect) unknown Content-Encoding used (Firepower)", + "created": 1748526014, + "modified": 1748526014, + "tag_type": { + "id": "source", + "name": "Source" + }, + "is_active": true, + "created_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@default.com", + "last_name": "Default", + "first_name": "System" + }, + "colour_code": "#45505E", + "modified_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@default.com", + "last_name": "Default", + "first_name": "System" + } + }, + { + "id": "0bdcf6dc-546f-4021-a292-1f5313ff8d55", + "name": "(smtp) unknown command (Firepower)", + "created": 1748612481, + "modified": 1748612481, + "tag_type": { + "id": "source", + "name": "Source" + }, + "is_active": true, + "created_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@default.com", + "last_name": "Default", + "first_name": "System" + }, + "colour_code": "#45505E", + "modified_by": { + "id": "c47b363a-2a0a-4bea-942f-27a260d121d1", + "email": "system@default.com", + "last_name": "Default", + "first_name": "System" + } + } + ], + "data": [ + { + "id": "34ea4742-cba4-4b8f-84ed-306ae74d1142", + "name": "example.com", + "tags": [ + "domain redirection", + "Apple" + ], + "country": "Germany", + "created": 1746812972.0, + "sources": [ + { + "tlp": "WHITE", + "name": "Vault", + "score": 100, + "last_seen": null, + "first_seen": 1746812972 + } + ], + "ctix_tlp": null, + "ioc_type": "domain-name", + "modified": 1748354676.712021, + "sdo_name": "space", + "sdo_type": "indicator", + "severity": "UNKNOWN", + "ctix_score": 90, + "is_revoked": false, + "source_tlp": "NONE", + "valid_from": 1746812972.0, + "analyst_tlp": null, + "is_actioned": false, + "is_reviewed": false, + "valid_until": null, + "ctix_created": 1748354691.640348, + "is_whitelist": false, + "analyst_score": null, + "ctix_modified": 1751280121.022215, + "custom_scores": null, + "is_deprecated": false, + "indicator_type": { + "type": "domain-name", + "attribute_field": "value" + }, + "is_false_positive": false + } + ], + "timestamp": 1751280122, + "title": "Package From CTIX" + } + ] + } + `}} diff --git a/packages/ti_cyware_intel_exchange/changelog.yml b/packages/ti_cyware_intel_exchange/changelog.yml new file mode 100644 index 00000000000..ecbe107e5d8 --- /dev/null +++ b/packages/ti_cyware_intel_exchange/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.1.0" + changes: + - description: Initial release. + type: enhancement + link: https://github.com/elastic/integrations/pull/14500 diff --git a/packages/ti_cyware_intel_exchange/data_stream/indicator/_dev/test/pipeline/test-common-config.yml b/packages/ti_cyware_intel_exchange/data_stream/indicator/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..37e8fa225fd --- /dev/null +++ b/packages/ti_cyware_intel_exchange/data_stream/indicator/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,3 @@ +fields: + tags: + - preserve_duplicate_custom_fields diff --git a/packages/ti_cyware_intel_exchange/data_stream/indicator/_dev/test/pipeline/test-indicator.log b/packages/ti_cyware_intel_exchange/data_stream/indicator/_dev/test/pipeline/test-indicator.log new file mode 100644 index 00000000000..c1c39dd8bc7 --- /dev/null +++ b/packages/ti_cyware_intel_exchange/data_stream/indicator/_dev/test/pipeline/test-indicator.log @@ -0,0 +1,3 @@ +{"analyst_score":20,"analyst_tlp":"unknown","country":"Malaysia","created":1748535036,"ctix_created":1748614256.901936,"ctix_modified":1752044704.638542,"ctix_score":0,"ctix_tlp":"unknown","custom_scores":500,"id":"996ba2cd-80de-496d-8ef5-98bc4bcc1f5a","indicator_type":{"attribute_field":"value","type":"ipv4-addr"},"relations":[{"id":"49ba3973-2354-4e25-a347-58b261ca9fd7","created":1747948578.660674,"modified":1747948578.660679,"field_name":"object_refs","source_ref":"26f4d192-e529-433c-ae04-cf35ae86c441","target_ref":"9fbf7925-9b92-4a82-a5a8-e6e67f177ebc","source_name":"Stix2dot1bundle_65263c24-0f6a-4ef8-be02-c7f4972cda01","source_type":"report","target_name":"7XcLwqTP@0jSqp.RUnVs.io","target_type":"indicator","unique_hash":"49ba397323544e25634758b261ca9fd7","source_sub_type":"unknown","target_sub_type":"email-addr","relationship_type":"related-to"}],"ioc_type":"ipv4-addr","is_actioned":false,"is_deprecated":true,"is_false_positive":false,"is_reviewed":false,"is_revoked":false,"is_whitelist":false,"modified":1749132501.868055,"name":"216.160.83.56","sdo_name":"216.160.83.56","sdo_type":"indicator","severity":"UNKNOWN","source_tlp":"NONE","sources":[{"first_seen":1748535036,"last_seen":1748535045,"name":"Alien Vault","score":100,"tlp":"WHITE"}],"tags":["stealth","ssh","ssh access","backdoor","asus routers","operational relay box","cve-2023-39780","nvram","Feed",".net","apt","persistence","botnet","authentication bypass"],"valid_from":1748535036,"valid_until":1751659200} +{"analyst_score":null,"analyst_tlp":null,"country":null,"created":1746812972,"ctix_created":1748354691.651628,"ctix_modified":1751279296.273555,"ctix_score":90,"ctix_tlp":null,"custom_scores":null,"id":"f36749f2-776f-4153-b840-08bad5fb18b1","indicator_type":{"attribute_field":"MD5","type":"file"},"ioc_type":"file","is_actioned":false,"is_deprecated":false,"is_false_positive":false,"is_reviewed":false,"is_revoked":false,"is_whitelist":false,"modified":1748354676.716103,"name":"e8c5c5829b630dcf61b55f271ac6c085","sdo_name":"e8c5c5829b630dcf61b55f271ac6c085","sdo_type":"indicator","severity":"UNKNOWN","source_tlp":"NONE","sources":[{"first_seen":1746812972,"last_seen":null,"name":"Alien Vault","score":100,"tlp":"WHITE"}],"tags":["domain redirection","brand impersonation","cryptocurrency","Apple","URL spoofing","X\/Twitter"],"valid_from":1746812972,"valid_until":null} +{"external_references":[{"external_references-1":"value_reference"}],"analyst_description":"analyst_description-1","custom_attributes":{"key1":"value1"},"report_types":["report1","report2"],"tlp_value":"CLEAR","source_description":"SAVE result SET v3","analyst_score":null,"analyst_tlp":null,"country":"Malaysia","created":1748535036,"ctix_created":1748614256.901936,"ctix_modified":1752044704.638542,"ctix_score":0,"ctix_tlp":null,"custom_scores":null,"id":"z36ba2cd-80de-496d-8ef5-98bc4bcc1f5a","indicator_type":{"attribute_field":"value","type":"ipv4-addr"},"relations":[{"id":"49ba3973-2354-4e25-a347-58b261ca9fd7","created":1747948578.660674,"modified":1747948578.660679,"field_name":"object_refs","source_ref":"26f4d192-e529-433c-ae04-cf35ae86c441","target_ref":"9fbf7925-9b92-4a82-a5a8-e6e67f177ebc","source_name":"Stix2dot1bundle_65263c24-0f6a-4ef8-be02-c7f4972cda01","source_type":"report","target_name":"7XcLwqTP@0jSqp.RUnVs.io","target_type":"indicator","unique_hash":"49ba397323544e25634758b261ca9fd7","source_sub_type":"source_type","target_sub_type":"email-addr","relationship_type":"related-to"}],"ioc_type":"ipv4-addr","is_actioned":true,"is_deprecated":true,"is_false_positive":false,"is_reviewed":true,"is_revoked":true,"is_whitelist":false,"modified":1749132501.868055,"name":"0.0.0.0","sdo_name":"0.0.0.0","sdo_type":"indicator","severity":"UNKNOWN","source_tlp":"NONE","sources":[{"first_seen":1748535036,"last_seen":null,"name":"Alien Vault","score":100,"tlp":"WHITE"}],"tags":["stealth","ssh","ssh access","backdoor","asus routers","operational relay box","cve-2023-39780","nvram","Feed",".net","apt","persistence","botnet","authentication bypass"],"valid_from":1748535036,"valid_until":1751659200} diff --git a/packages/ti_cyware_intel_exchange/data_stream/indicator/_dev/test/pipeline/test-indicator.log-expected.json b/packages/ti_cyware_intel_exchange/data_stream/indicator/_dev/test/pipeline/test-indicator.log-expected.json new file mode 100644 index 00000000000..2e368c47ef6 --- /dev/null +++ b/packages/ti_cyware_intel_exchange/data_stream/indicator/_dev/test/pipeline/test-indicator.log-expected.json @@ -0,0 +1,362 @@ +{ + "expected": [ + { + "@timestamp": "2025-07-09T07:05:04.638Z", + "ecs": { + "version": "8.17.0" + }, + "event": { + "category": [ + "threat" + ], + "created": "2025-05-30T14:10:56.901Z", + "id": "996ba2cd-80de-496d-8ef5-98bc4bcc1f5a", + "kind": "enrichment", + "original": "{\"analyst_score\":20,\"analyst_tlp\":\"unknown\",\"country\":\"Malaysia\",\"created\":1748535036,\"ctix_created\":1748614256.901936,\"ctix_modified\":1752044704.638542,\"ctix_score\":0,\"ctix_tlp\":\"unknown\",\"custom_scores\":500,\"id\":\"996ba2cd-80de-496d-8ef5-98bc4bcc1f5a\",\"indicator_type\":{\"attribute_field\":\"value\",\"type\":\"ipv4-addr\"},\"relations\":[{\"id\":\"49ba3973-2354-4e25-a347-58b261ca9fd7\",\"created\":1747948578.660674,\"modified\":1747948578.660679,\"field_name\":\"object_refs\",\"source_ref\":\"26f4d192-e529-433c-ae04-cf35ae86c441\",\"target_ref\":\"9fbf7925-9b92-4a82-a5a8-e6e67f177ebc\",\"source_name\":\"Stix2dot1bundle_65263c24-0f6a-4ef8-be02-c7f4972cda01\",\"source_type\":\"report\",\"target_name\":\"7XcLwqTP@0jSqp.RUnVs.io\",\"target_type\":\"indicator\",\"unique_hash\":\"49ba397323544e25634758b261ca9fd7\",\"source_sub_type\":\"unknown\",\"target_sub_type\":\"email-addr\",\"relationship_type\":\"related-to\"}],\"ioc_type\":\"ipv4-addr\",\"is_actioned\":false,\"is_deprecated\":true,\"is_false_positive\":false,\"is_reviewed\":false,\"is_revoked\":false,\"is_whitelist\":false,\"modified\":1749132501.868055,\"name\":\"216.160.83.56\",\"sdo_name\":\"216.160.83.56\",\"sdo_type\":\"indicator\",\"severity\":\"UNKNOWN\",\"source_tlp\":\"NONE\",\"sources\":[{\"first_seen\":1748535036,\"last_seen\":1748535045,\"name\":\"Alien Vault\",\"score\":100,\"tlp\":\"WHITE\"}],\"tags\":[\"stealth\",\"ssh\",\"ssh access\",\"backdoor\",\"asus routers\",\"operational relay box\",\"cve-2023-39780\",\"nvram\",\"Feed\",\".net\",\"apt\",\"persistence\",\"botnet\",\"authentication bypass\"],\"valid_from\":1748535036,\"valid_until\":1751659200}", + "severity": 21, + "type": [ + "indicator" + ] + }, + "observer": { + "product": "Threat Intelligence Management", + "vendor": "Cyware" + }, + "related": { + "ip": [ + "216.160.83.56" + ] + }, + "tags": [ + "preserve_duplicate_custom_fields", + "stealth", + "ssh", + "ssh access", + "backdoor", + "asus routers", + "operational relay box", + "cve-2023-39780", + "nvram", + "Feed", + ".net", + "apt", + "persistence", + "botnet", + "authentication bypass" + ], + "threat": { + "indicator": { + "first_seen": [ + "2025-05-29T16:10:36.000Z" + ], + "geo": { + "country_name": "Malaysia" + }, + "ip": [ + "216.160.83.56" + ], + "last_seen": [ + "2025-05-29T16:10:45.000Z" + ], + "marking": { + "tlp": [ + "WHITE" + ] + }, + "modified_at": "2025-06-05T14:08:21.868Z", + "provider": [ + "Alien Vault" + ], + "type": "ipv4-addr" + } + }, + "ti_cyware_intel_exchange": { + "indicator": { + "analyst_score": 20, + "analyst_tlp": "unknown", + "country": "Malaysia", + "created": "2025-05-29T16:10:36.000Z", + "ctix_created": "2025-05-30T14:10:56.901Z", + "ctix_modified": "2025-07-09T07:05:04.638Z", + "ctix_score": 0, + "ctix_tlp": "unknown", + "custom_scores": 500, + "id": "996ba2cd-80de-496d-8ef5-98bc4bcc1f5a", + "indicator_type": { + "attribute_field": "value", + "type": "ipv4-addr" + }, + "ioc_type": "ipv4-addr", + "ip": "216.160.83.56", + "is_actioned": false, + "is_deprecated": true, + "is_false_positive": false, + "is_reviewed": false, + "is_revoked": false, + "is_whitelist": false, + "modified": "2025-06-05T14:08:21.868Z", + "sdo_ip": "216.160.83.56", + "sdo_type": "indicator", + "source_tlp": "NONE", + "sources": [ + { + "first_seen": "2025-05-29T16:10:36.000Z", + "last_seen": "2025-05-29T16:10:45.000Z", + "name": "Alien Vault", + "score": 100, + "tlp": "WHITE" + } + ], + "tags_list": [ + "stealth", + "ssh", + "ssh access", + "backdoor", + "asus routers", + "operational relay box", + "cve-2023-39780", + "nvram", + "Feed", + ".net", + "apt", + "persistence", + "botnet", + "authentication bypass" + ], + "valid_from": "2025-05-29T16:10:36.000Z", + "valid_until": "2025-07-04T20:00:00.000Z" + } + } + }, + { + "@timestamp": "2025-06-30T10:28:16.273Z", + "ecs": { + "version": "8.17.0" + }, + "event": { + "category": [ + "threat" + ], + "created": "2025-05-27T14:04:51.651Z", + "id": "f36749f2-776f-4153-b840-08bad5fb18b1", + "kind": "enrichment", + "original": "{\"analyst_score\":null,\"analyst_tlp\":null,\"country\":null,\"created\":1746812972,\"ctix_created\":1748354691.651628,\"ctix_modified\":1751279296.273555,\"ctix_score\":90,\"ctix_tlp\":null,\"custom_scores\":null,\"id\":\"f36749f2-776f-4153-b840-08bad5fb18b1\",\"indicator_type\":{\"attribute_field\":\"MD5\",\"type\":\"file\"},\"ioc_type\":\"file\",\"is_actioned\":false,\"is_deprecated\":false,\"is_false_positive\":false,\"is_reviewed\":false,\"is_revoked\":false,\"is_whitelist\":false,\"modified\":1748354676.716103,\"name\":\"e8c5c5829b630dcf61b55f271ac6c085\",\"sdo_name\":\"e8c5c5829b630dcf61b55f271ac6c085\",\"sdo_type\":\"indicator\",\"severity\":\"UNKNOWN\",\"source_tlp\":\"NONE\",\"sources\":[{\"first_seen\":1746812972,\"last_seen\":null,\"name\":\"Alien Vault\",\"score\":100,\"tlp\":\"WHITE\"}],\"tags\":[\"domain redirection\",\"brand impersonation\",\"cryptocurrency\",\"Apple\",\"URL spoofing\",\"X\\/Twitter\"],\"valid_from\":1746812972,\"valid_until\":null}", + "severity": 99, + "type": [ + "indicator" + ] + }, + "observer": { + "product": "Threat Intelligence Management", + "vendor": "Cyware" + }, + "tags": [ + "preserve_duplicate_custom_fields", + "domain redirection", + "brand impersonation", + "cryptocurrency", + "Apple", + "URL spoofing", + "X/Twitter" + ], + "threat": { + "indicator": { + "first_seen": [ + "2025-05-09T17:49:32.000Z" + ], + "marking": { + "tlp": [ + "WHITE" + ] + }, + "modified_at": "2025-05-27T14:04:36.716Z", + "name": [ + "e8c5c5829b630dcf61b55f271ac6c085" + ], + "provider": [ + "Alien Vault" + ], + "type": "file" + } + }, + "ti_cyware_intel_exchange": { + "indicator": { + "created": "2025-05-09T17:49:32.000Z", + "ctix_created": "2025-05-27T14:04:51.651Z", + "ctix_modified": "2025-06-30T10:28:16.273Z", + "ctix_score": 90, + "id": "f36749f2-776f-4153-b840-08bad5fb18b1", + "indicator_type": { + "attribute_field": "MD5", + "type": "file" + }, + "ioc_type": "file", + "is_actioned": false, + "is_deprecated": false, + "is_false_positive": false, + "is_reviewed": false, + "is_revoked": false, + "is_whitelist": false, + "modified": "2025-05-27T14:04:36.716Z", + "name": "e8c5c5829b630dcf61b55f271ac6c085", + "sdo_name": "e8c5c5829b630dcf61b55f271ac6c085", + "sdo_type": "indicator", + "source_tlp": "NONE", + "sources": [ + { + "first_seen": "2025-05-09T17:49:32.000Z", + "name": "Alien Vault", + "score": 100, + "tlp": "WHITE" + } + ], + "tags_list": [ + "domain redirection", + "brand impersonation", + "cryptocurrency", + "Apple", + "URL spoofing", + "X/Twitter" + ], + "valid_from": "2025-05-09T17:49:32.000Z" + } + } + }, + { + "@timestamp": "2025-07-09T07:05:04.638Z", + "ecs": { + "version": "8.17.0" + }, + "event": { + "category": [ + "threat" + ], + "created": "2025-05-30T14:10:56.901Z", + "id": "z36ba2cd-80de-496d-8ef5-98bc4bcc1f5a", + "kind": "enrichment", + "original": "{\"external_references\":[{\"external_references-1\":\"value_reference\"}],\"analyst_description\":\"analyst_description-1\",\"custom_attributes\":{\"key1\":\"value1\"},\"report_types\":[\"report1\",\"report2\"],\"tlp_value\":\"CLEAR\",\"source_description\":\"SAVE result SET v3\",\"analyst_score\":null,\"analyst_tlp\":null,\"country\":\"Malaysia\",\"created\":1748535036,\"ctix_created\":1748614256.901936,\"ctix_modified\":1752044704.638542,\"ctix_score\":0,\"ctix_tlp\":null,\"custom_scores\":null,\"id\":\"z36ba2cd-80de-496d-8ef5-98bc4bcc1f5a\",\"indicator_type\":{\"attribute_field\":\"value\",\"type\":\"ipv4-addr\"},\"relations\":[{\"id\":\"49ba3973-2354-4e25-a347-58b261ca9fd7\",\"created\":1747948578.660674,\"modified\":1747948578.660679,\"field_name\":\"object_refs\",\"source_ref\":\"26f4d192-e529-433c-ae04-cf35ae86c441\",\"target_ref\":\"9fbf7925-9b92-4a82-a5a8-e6e67f177ebc\",\"source_name\":\"Stix2dot1bundle_65263c24-0f6a-4ef8-be02-c7f4972cda01\",\"source_type\":\"report\",\"target_name\":\"7XcLwqTP@0jSqp.RUnVs.io\",\"target_type\":\"indicator\",\"unique_hash\":\"49ba397323544e25634758b261ca9fd7\",\"source_sub_type\":\"source_type\",\"target_sub_type\":\"email-addr\",\"relationship_type\":\"related-to\"}],\"ioc_type\":\"ipv4-addr\",\"is_actioned\":true,\"is_deprecated\":true,\"is_false_positive\":false,\"is_reviewed\":true,\"is_revoked\":true,\"is_whitelist\":false,\"modified\":1749132501.868055,\"name\":\"0.0.0.0\",\"sdo_name\":\"0.0.0.0\",\"sdo_type\":\"indicator\",\"severity\":\"UNKNOWN\",\"source_tlp\":\"NONE\",\"sources\":[{\"first_seen\":1748535036,\"last_seen\":null,\"name\":\"Alien Vault\",\"score\":100,\"tlp\":\"WHITE\"}],\"tags\":[\"stealth\",\"ssh\",\"ssh access\",\"backdoor\",\"asus routers\",\"operational relay box\",\"cve-2023-39780\",\"nvram\",\"Feed\",\".net\",\"apt\",\"persistence\",\"botnet\",\"authentication bypass\"],\"valid_from\":1748535036,\"valid_until\":1751659200}", + "severity": 21, + "type": [ + "indicator" + ] + }, + "observer": { + "product": "Threat Intelligence Management", + "vendor": "Cyware" + }, + "related": { + "ip": [ + "0.0.0.0" + ] + }, + "tags": [ + "preserve_duplicate_custom_fields", + "stealth", + "ssh", + "ssh access", + "backdoor", + "asus routers", + "operational relay box", + "cve-2023-39780", + "nvram", + "Feed", + ".net", + "apt", + "persistence", + "botnet", + "authentication bypass" + ], + "threat": { + "indicator": { + "description": "SAVE result SET v3", + "first_seen": [ + "2025-05-29T16:10:36.000Z" + ], + "geo": { + "country_name": "Malaysia" + }, + "ip": [ + "0.0.0.0" + ], + "marking": { + "tlp": [ + "WHITE" + ] + }, + "modified_at": "2025-06-05T14:08:21.868Z", + "provider": [ + "Alien Vault" + ], + "type": "ipv4-addr" + } + }, + "ti_cyware_intel_exchange": { + "indicator": { + "analyst_description": "analyst_description-1", + "country": "Malaysia", + "created": "2025-05-29T16:10:36.000Z", + "ctix_created": "2025-05-30T14:10:56.901Z", + "ctix_modified": "2025-07-09T07:05:04.638Z", + "ctix_score": 0, + "custom_attributes": { + "key1": "value1" + }, + "external_references": [ + { + "external_references-1": "value_reference" + } + ], + "id": "z36ba2cd-80de-496d-8ef5-98bc4bcc1f5a", + "indicator_type": { + "attribute_field": "value", + "type": "ipv4-addr" + }, + "ioc_type": "ipv4-addr", + "ip": "0.0.0.0", + "is_actioned": true, + "is_deprecated": true, + "is_false_positive": false, + "is_reviewed": true, + "is_revoked": true, + "is_whitelist": false, + "modified": "2025-06-05T14:08:21.868Z", + "report_types": [ + "report1", + "report2" + ], + "sdo_ip": "0.0.0.0", + "sdo_type": "indicator", + "source_description": "SAVE result SET v3", + "source_tlp": "NONE", + "sources": [ + { + "first_seen": "2025-05-29T16:10:36.000Z", + "name": "Alien Vault", + "score": 100, + "tlp": "WHITE" + } + ], + "tags_list": [ + "stealth", + "ssh", + "ssh access", + "backdoor", + "asus routers", + "operational relay box", + "cve-2023-39780", + "nvram", + "Feed", + ".net", + "apt", + "persistence", + "botnet", + "authentication bypass" + ], + "tlp_value": "CLEAR", + "valid_from": "2025-05-29T16:10:36.000Z", + "valid_until": "2025-07-04T20:00:00.000Z" + } + } + } + ] +} diff --git a/packages/ti_cyware_intel_exchange/data_stream/indicator/_dev/test/system/test-default-config.yml b/packages/ti_cyware_intel_exchange/data_stream/indicator/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..7fd4066909a --- /dev/null +++ b/packages/ti_cyware_intel_exchange/data_stream/indicator/_dev/test/system/test-default-config.yml @@ -0,0 +1,11 @@ +input: cel +service: indicator-api +data_stream: + vars: + access_id: test_username + secret_key: test_secret_key + url: http://{{Hostname}}:{{Port}} + batch_size: 2 + preserve_original_event: true +assert: + hit_count: 5 \ No newline at end of file diff --git a/packages/ti_cyware_intel_exchange/data_stream/indicator/agent/stream/cel.yml.hbs b/packages/ti_cyware_intel_exchange/data_stream/indicator/agent/stream/cel.yml.hbs new file mode 100644 index 00000000000..7ec8b1f1683 --- /dev/null +++ b/packages/ti_cyware_intel_exchange/data_stream/indicator/agent/stream/cel.yml.hbs @@ -0,0 +1,126 @@ +config_version: 2 +interval: {{interval}} +resource.tracer: + enabled: {{enable_request_tracer}} + filename: "../../logs/cel/http-request-trace-*.ndjson" + maxbackups: 5 +{{#if proxy_url}} +resource.proxy_url: {{proxy_url}} +{{/if}} +{{#if ssl}} +resource.ssl: {{ssl}} +{{/if}} +{{#if http_client_timeout}} +resource.timeout: {{http_client_timeout}} +{{/if}} +resource.url: {{url}} +state: + access_id: {{access_id}} + secret_key: {{secret_key}} + page_size: {{batch_size}} + initial_interval: {{initial_interval}} +redact: + fields: + - access_id + - secret_key +program: |- + ( + state.?want_more.orValue(false) ? + state.with( + { + "expires": int(now() + duration("28s")), + } + ) + : + state.with( + { + "expires": int(now() + duration("28s")), + "from_time": state.?cursor.modified_timestamp.orValue(int(now - duration(state.initial_interval))), + } + ) + ).as(state, + request( + "GET", + state.url.trim_right("/") + "/ingestion/rules/save_result_set/?" + { + "page_size": [string(state.page_size)], + "page": [string(state.?page_number.orValue(1))], + "version": ["v3"], + "from_timestamp": [string(state.from_time)], + "AccessID": [state.access_id], + "Expires": [string(state.expires)], + "Signature": [( + [ + state.access_id, + string(state.expires) + ].join("\n") + .hmac("sha1", bytes(state.secret_key)) + .base64())], + }.format_query() + ).do_request().as(resp, resp.StatusCode == 200 ? + resp.Body.decode_json().as(body, { + "events": body.results.map(r, + r.data.map(result, { + "message": result.encode_json() + }) + ).flatten(), + "want_more": has(body.next) && body.next != null, + "page_number": has(body.next) && body.next != null ? + int(state.?page_number.orValue(1)) + 1 + : + 1, + "access_id": state.access_id, + "page_size": state.page_size, + "secret_key": state.secret_key, + "from_time": string(state.from_time), + "cursor": { + ?"modified_timestamp": body.results.size() > 0 ? + optional.of(body.results.map(e, e.ctix_modified).max().as(last, + has(state.?cursor.modified_timestamp) && last < state.cursor.modified_timestamp ? + state.cursor.modified_timestamp + : + last + )) + : + state.?cursor.modified_timestamp, + }, + }) + : + { + "events": { + "error": { + "code": string(resp.StatusCode), + "id": string(resp.Status), + "message": "GET:" + state.url.trim_right("/") + "/ingestion/rules/save_result_set/? " + + ( + size(resp.Body) != 0 ? + string(resp.Body) + : + string(resp.Status) + " (" + string(resp.StatusCode) + ")" + ), + }, + }, + "want_more": false, + "access_id": state.access_id, + "page_size": state.page_size, + "secret_key": state.secret_key, + "initial_interval": state.initial_interval, + } + ) + ) +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#if preserve_duplicate_custom_fields}} + - preserve_duplicate_custom_fields +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/ti_cyware_intel_exchange/data_stream/indicator/elasticsearch/ilm/default_policy.json b/packages/ti_cyware_intel_exchange/data_stream/indicator/elasticsearch/ilm/default_policy.json new file mode 100644 index 00000000000..be17d7f650e --- /dev/null +++ b/packages/ti_cyware_intel_exchange/data_stream/indicator/elasticsearch/ilm/default_policy.json @@ -0,0 +1,24 @@ + +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_age": "2d", + "max_size": "50gb" + }, + "set_priority": { + "priority": 100 + } + } + }, + "delete": { + "min_age": "3d", + "actions": { + "delete": {} + } + } + } + } +} diff --git a/packages/ti_cyware_intel_exchange/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml b/packages/ti_cyware_intel_exchange/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..92b256ab2ac --- /dev/null +++ b/packages/ti_cyware_intel_exchange/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,583 @@ +--- +description: Pipeline for processing Indicator logs. +processors: + - set: + field: ecs.version + tag: set_ecs_version + value: 8.17.0 + - terminate: + tag: data_collection_error + if: ctx.error?.message != null && ctx.message == null && ctx.event?.original == null + - rename: + field: message + tag: rename_message_to_event_original + target_field: event.original + ignore_missing: true + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. + if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null + - json: + field: event.original + tag: json_event_original + target_field: json + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: observer.vendor + tag: set_observer_vendor + value: Cyware + - set: + field: observer.product + tag: set_observer_product + value: Threat Intelligence Management + - set: + field: event.kind + tag: set_event_kind + value: enrichment + - append: + field: event.category + tag: append_threat_into_event_category + value: threat + - append: + field: event.type + tag: append_indicator_into_event_type + value: indicator + - fingerprint: + fields: + - json.id + - json.ctix_modified + tag: fingerprint_indicator + target_field: _id + - rename: + field: json.analyst_description + tag: rename_analyst_description + target_field: ti_cyware_intel_exchange.indicator.analyst_description + ignore_missing: true + - convert: + field: json.analyst_score + tag: convert_analyst_score_to_long + target_field: ti_cyware_intel_exchange.indicator.analyst_score + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - rename: + field: json.analyst_tlp + tag: rename_analyst_tlp + target_field: ti_cyware_intel_exchange.indicator.analyst_tlp + ignore_missing: true + - rename: + field: json.country + tag: rename_country + target_field: ti_cyware_intel_exchange.indicator.country + ignore_missing: true + - set: + field: threat.indicator.geo.country_name + tag: set_threat_indicator_geo_country_name_from_indicator_country + copy_from: ti_cyware_intel_exchange.indicator.country + ignore_empty_value: true + - date: + field: json.created + tag: date_created + target_field: ti_cyware_intel_exchange.indicator.created + formats: + - UNIX + if: ctx.json?.created != null && ctx.json.created != '' + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - date: + field: json.ctix_created + tag: date_ctix_created + target_field: ti_cyware_intel_exchange.indicator.ctix_created + formats: + - UNIX + if: ctx.json?.ctix_created != null && ctx.json.ctix_created != '' + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: event.created + tag: set_event_created_from_indicator_ctix_created + copy_from: ti_cyware_intel_exchange.indicator.ctix_created + ignore_empty_value: true + - date: + field: json.ctix_modified + tag: date_ctix_modified + target_field: ti_cyware_intel_exchange.indicator.ctix_modified + formats: + - UNIX + if: ctx.json?.ctix_modified != null && ctx.json.ctix_modified != '' + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: '@timestamp' + tag: set_@timestamp_from_indicator_ctix_modified + copy_from: ti_cyware_intel_exchange.indicator.ctix_modified + ignore_empty_value: true + - convert: + field: json.ctix_score + tag: convert_ctix_score_to_long + target_field: ti_cyware_intel_exchange.indicator.ctix_score + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - script: + lang: painless + description: Script to set event_severity. + tag: set_event_severity + if: ctx.ti_cyware_intel_exchange?.indicator?.ctix_score != null + source: |- + ctx.event = ctx.event ?: [:]; + if (ctx.ti_cyware_intel_exchange.indicator.ctix_score <= 21 ) { + ctx.event.severity = 21; + } else if (ctx.ti_cyware_intel_exchange.indicator.ctix_score >= 22 && ctx.ti_cyware_intel_exchange.indicator.ctix_score <= 47) { + ctx.event.severity = 47; + } else if (ctx.ti_cyware_intel_exchange.indicator.ctix_score >= 48 && ctx.ti_cyware_intel_exchange.indicator.ctix_score <= 73) { + ctx.event.severity = 73; + } else if (ctx.ti_cyware_intel_exchange.indicator.ctix_score >= 74 && ctx.ti_cyware_intel_exchange.indicator.ctix_score <= 100) { + ctx.event.severity = 99; + } + - rename: + field: json.ctix_tlp + tag: rename_ctix_tlp + target_field: ti_cyware_intel_exchange.indicator.ctix_tlp + ignore_missing: true + - rename: + field: json.custom_attributes + tag: rename_custom_attributes + target_field: ti_cyware_intel_exchange.indicator.custom_attributes + ignore_missing: true + - convert: + field: json.custom_scores + tag: convert_custom_scores_to_long + target_field: ti_cyware_intel_exchange.indicator.custom_scores + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - rename: + field: json.external_references + tag: rename_external_references + target_field: ti_cyware_intel_exchange.indicator.external_references + ignore_missing: true + - rename: + field: json.id + tag: rename_id + target_field: ti_cyware_intel_exchange.indicator.id + ignore_missing: true + - set: + field: event.id + tag: set_event_id_from_indicator_id + copy_from: ti_cyware_intel_exchange.indicator.id + ignore_empty_value: true + - rename: + field: json.indicator_type.attribute_field + tag: rename_indicator_type_attribute_field + target_field: ti_cyware_intel_exchange.indicator.indicator_type.attribute_field + ignore_missing: true + - rename: + field: json.indicator_type.type + tag: rename_indicator_type_type + target_field: ti_cyware_intel_exchange.indicator.indicator_type.type + ignore_missing: true + - rename: + field: json.ioc_type + tag: rename_ioc_type + target_field: ti_cyware_intel_exchange.indicator.ioc_type + ignore_missing: true + - set: + field: threat.indicator.type + tag: set_threat_indicator_type_from_indicator_ioc_type + copy_from: ti_cyware_intel_exchange.indicator.ioc_type + ignore_empty_value: true + - convert: + field: json.name + tag: convert_name_to_ip + target_field: ti_cyware_intel_exchange.indicator.ip + type: ip + ignore_missing: true + if: ctx.json?.name != '' + on_failure: + - rename: + field: json.name + target_field: ti_cyware_intel_exchange.indicator.name + ignore_missing: true + - append: + field: related.ip + tag: append_indicator_ip_into_related_ip + value: '{{{ti_cyware_intel_exchange.indicator.ip}}}' + allow_duplicates: false + if: ctx.ti_cyware_intel_exchange?.indicator?.ip != null + - convert: + field: json.is_actioned + tag: convert_is_actioned_to_boolean + target_field: ti_cyware_intel_exchange.indicator.is_actioned + type: boolean + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: json.is_deprecated + tag: convert_is_deprecated_to_boolean + target_field: ti_cyware_intel_exchange.indicator.is_deprecated + type: boolean + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: json.is_false_positive + tag: convert_is_false_positive_to_boolean + target_field: ti_cyware_intel_exchange.indicator.is_false_positive + type: boolean + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: json.is_reviewed + tag: convert_is_reviewed_to_boolean + target_field: ti_cyware_intel_exchange.indicator.is_reviewed + type: boolean + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: json.is_revoked + tag: convert_is_revoked_to_boolean + target_field: ti_cyware_intel_exchange.indicator.is_revoked + type: boolean + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: json.is_whitelist + tag: convert_is_whitelist_to_boolean + target_field: ti_cyware_intel_exchange.indicator.is_whitelist + type: boolean + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - date: + field: json.modified + tag: date_modified + target_field: ti_cyware_intel_exchange.indicator.modified + formats: + - UNIX + if: ctx.json?.modified != null && ctx.json.modified != '' + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: threat.indicator.modified_at + tag: set_threat_indicator_modified_at_from_indicator_modified + copy_from: ti_cyware_intel_exchange.indicator.modified + ignore_empty_value: true + - rename: + field: json.report_types + tag: rename_report_types + target_field: ti_cyware_intel_exchange.indicator.report_types + ignore_missing: true + - convert: + field: json.sdo_name + tag: convert_sdo_name_to_ip + target_field: ti_cyware_intel_exchange.indicator.sdo_ip + type: ip + ignore_missing: true + if: ctx.json?.sdo_name != '' + on_failure: + - rename: + field: json.sdo_name + target_field: ti_cyware_intel_exchange.indicator.sdo_name + ignore_missing: true + - append: + field: threat.indicator.name + tag: append_ti_cyware_intel_exchange_indicator_sdo_name_into_threat_indicator_name + value: '{{{ti_cyware_intel_exchange.indicator.sdo_name}}}' + allow_duplicates: false + if: ctx.ti_cyware_intel_exchange?.indicator?.sdo_name != null + - append: + field: threat.indicator.ip + tag: append_ti_cyware_intel_exchange_indicator_sdo_ip_into_threat_indicator_ip + value: '{{{ti_cyware_intel_exchange.indicator.sdo_ip}}}' + allow_duplicates: false + if: ctx.ti_cyware_intel_exchange?.indicator?.sdo_ip != null + - append: + field: related.ip + tag: append_indicator_sdo_ip_into_related_ip + value: '{{{ti_cyware_intel_exchange.indicator.sdo_ip}}}' + allow_duplicates: false + if: ctx.ti_cyware_intel_exchange?.indicator?.sdo_ip != null + - rename: + field: json.sdo_type + tag: rename_sdo_type + target_field: ti_cyware_intel_exchange.indicator.sdo_type + ignore_missing: true + - rename: + field: json.source_description + tag: rename_source_description + target_field: ti_cyware_intel_exchange.indicator.source_description + ignore_missing: true + - set: + field: threat.indicator.description + tag: set_threat_indicator_description_from_indicator_source_description + copy_from: ti_cyware_intel_exchange.indicator.source_description + ignore_empty_value: true + - rename: + field: json.source_tlp + tag: rename_source_tlp + target_field: ti_cyware_intel_exchange.indicator.source_tlp + ignore_missing: true + - foreach: + field: json.sources + tag: foreach_sources_first_seen + if: ctx.json?.sources instanceof List + processor: + date: + field: _ingest._value.first_seen + tag: date_sources_first_seen + target_field: _ingest._value.first_seen + formats: + - UNIX + on_failure: + - remove: + field: _ingest._value.first_seen + ignore_missing: true + - foreach: + field: json.sources + tag: foreach_sources_first_seen + if: ctx.json?.sources instanceof List + processor: + append: + field: threat.indicator.first_seen + tag: append_sources_first_seen_into_threat_indicator_first_seen + value: '{{{_ingest._value.first_seen}}}' + allow_duplicates: false + - foreach: + field: json.sources + tag: foreach_sources_last_seen + if: ctx.json?.sources instanceof List + processor: + date: + field: _ingest._value.last_seen + tag: date_sources_last_seen + target_field: _ingest._value.last_seen + formats: + - UNIX + on_failure: + - remove: + field: _ingest._value.last_seen + ignore_missing: true + - foreach: + field: json.sources + tag: foreach_sources_last_seen + if: ctx.json?.sources instanceof List + processor: + append: + field: threat.indicator.last_seen + tag: append_sources_last_seen_into_threat_indicator_last_seen + value: '{{{_ingest._value.last_seen}}}' + allow_duplicates: false + - foreach: + field: json.sources + tag: foreach_sources_name + if: ctx.json?.sources instanceof List + processor: + append: + field: threat.indicator.provider + tag: append_sources_name_into_threat_indicator_provider + value: '{{{_ingest._value.name}}}' + allow_duplicates: false + - foreach: + field: json.sources + tag: foreach_sources_score + if: ctx.json?.sources instanceof List + processor: + convert: + field: _ingest._value.score + tag: convert_sources_score_to_long + type: long + ignore_missing: true + on_failure: + - remove: + field: _ingest._value.score + ignore_missing: true + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - foreach: + field: json.sources + tag: foreach_sources_tlp + if: ctx.json?.sources instanceof List + processor: + append: + field: threat.indicator.marking.tlp + tag: append_sources_tlp_into_threat_indicator_marking_tlp + value: '{{{_ingest._value.tlp}}}' + allow_duplicates: false + - rename: + field: json.sources + tag: rename_sources + target_field: ti_cyware_intel_exchange.indicator.sources + ignore_missing: true + - rename: + field: json.tags + tag: rename_tags_to_tags_list + target_field: ti_cyware_intel_exchange.indicator.tags_list + if: ctx.json?.tags instanceof List + ignore_missing: true + - rename: + field: json.tags + tag: rename_tags_to_tags_object + target_field: ti_cyware_intel_exchange.indicator.tags_object + ignore_missing: true + if: ctx.json?.tags instanceof Map + - foreach: + field: ti_cyware_intel_exchange.indicator.tags_list + tag: foreach_ti_cyware_intel_exchange_indicator_tags_list + if: ctx.ti_cyware_intel_exchange?.indicator?.tags_list instanceof List + processor: + append: + field: tags + tag: append_tags_list_into_tags + value: '{{{_ingest._value}}}' + allow_duplicates: false + - rename: + field: json.tlp_value + tag: rename_tlp_value + target_field: ti_cyware_intel_exchange.indicator.tlp_value + ignore_missing: true + - date: + field: json.valid_from + tag: date_valid_from + target_field: ti_cyware_intel_exchange.indicator.valid_from + formats: + - UNIX + if: ctx.json?.valid_from != null && ctx.json.valid_from != '' + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - date: + field: json.valid_until + tag: date_valid_until + target_field: ti_cyware_intel_exchange.indicator.valid_until + formats: + - UNIX + if: ctx.json?.valid_until != null && ctx.json.valid_until != '' + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - foreach: + field: ti_cyware_intel_exchange.indicator.sources + tag: foreach_ti_cyware_intel_exchange_indicator_sources_/ + if: ctx.ti_cyware_intel_exchange?.indicator?.sources instanceof List + processor: + remove: + field: + - _ingest._value.tlp + - _ingest._value.name + - _ingest._value.last_seen + - _ingest._value.first_seen + tag: remove_custom_duplicate_fields_from_ti_cyware_intel_exchange_indicator_sources + ignore_missing: true + if: ctx.tags == null || !ctx.tags.contains('preserve_duplicate_custom_fields') + - remove: + field: + - ti_cyware_intel_exchange.indicator.country + - ti_cyware_intel_exchange.indicator.ctix_created + - ti_cyware_intel_exchange.indicator.ctix_modified + - ti_cyware_intel_exchange.indicator.external_references + - ti_cyware_intel_exchange.indicator.id + - ti_cyware_intel_exchange.indicator.ioc_type + - ti_cyware_intel_exchange.indicator.modified + - ti_cyware_intel_exchange.indicator.source_description + tag: remove_custom_duplicate_fields + ignore_missing: true + if: ctx.tags == null || !ctx.tags.contains('preserve_duplicate_custom_fields') + - remove: + field: json + tag: remove_json + ignore_missing: true + - script: + tag: script_to_drop_null_values + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: |- + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null +on_failure: + - append: + field: error.message + value: |- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/ti_cyware_intel_exchange/data_stream/indicator/fields/base-fields.yml b/packages/ti_cyware_intel_exchange/data_stream/indicator/fields/base-fields.yml new file mode 100644 index 00000000000..80a48b71c4b --- /dev/null +++ b/packages/ti_cyware_intel_exchange/data_stream/indicator/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module. + value: ti_cyware_intel_exchange +- name: event.dataset + type: constant_keyword + description: Event dataset. + value: ti_cyware_intel_exchange.indicator +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/ti_cyware_intel_exchange/data_stream/indicator/fields/beats.yml b/packages/ti_cyware_intel_exchange/data_stream/indicator/fields/beats.yml new file mode 100644 index 00000000000..d5fd38748ba --- /dev/null +++ b/packages/ti_cyware_intel_exchange/data_stream/indicator/fields/beats.yml @@ -0,0 +1,6 @@ +- name: input.type + type: keyword + description: Type of Filebeat input. +- name: log.offset + type: long + description: Log offset. diff --git a/packages/ti_cyware_intel_exchange/data_stream/indicator/fields/ecs.yml b/packages/ti_cyware_intel_exchange/data_stream/indicator/fields/ecs.yml new file mode 100644 index 00000000000..8442725f2f4 --- /dev/null +++ b/packages/ti_cyware_intel_exchange/data_stream/indicator/fields/ecs.yml @@ -0,0 +1,7 @@ +# Define ECS constant fields as constant_keyword +- name: observer.vendor + type: constant_keyword + value: Cyware +- name: observer.product + type: constant_keyword + value: Threat Intelligence Management diff --git a/packages/ti_cyware_intel_exchange/data_stream/indicator/fields/fields.yml b/packages/ti_cyware_intel_exchange/data_stream/indicator/fields/fields.yml new file mode 100644 index 00000000000..9896386fd39 --- /dev/null +++ b/packages/ti_cyware_intel_exchange/data_stream/indicator/fields/fields.yml @@ -0,0 +1,100 @@ +- name: ti_cyware_intel_exchange + type: group + fields: + - name: indicator + type: group + fields: + - name: analyst_description + type: keyword + - name: analyst_score + type: long + - name: analyst_tlp + type: keyword + - name: country + type: keyword + - name: created + type: date + - name: ctix_created + type: date + - name: ctix_modified + type: date + - name: ctix_score + type: long + - name: ctix_tlp + type: keyword + - name: custom_attributes + type: flattened + - name: custom_scores + type: long + - name: external_references + type: flattened + - name: id + type: keyword + - name: indicator_type + type: group + fields: + - name: attribute_field + type: keyword + - name: type + type: keyword + - name: ioc_type + type: keyword + - name: ip + type: ip + - name: is_actioned + type: boolean + - name: is_deprecated + type: boolean + description: Returns a value to indicate if the threat data object is deprecated. + - name: is_false_positive + type: boolean + description: Returns a value to indicate if the object is false positive. + - name: is_reviewed + type: boolean + - name: is_revoked + type: boolean + - name: is_whitelist + type: boolean + description: Returns a value to indicate if the threat data object is whitelisted. + - name: modified + type: date + - name: name + type: keyword + - name: report_types + type: keyword + - name: sdo_ip + type: ip + - name: sdo_name + type: keyword + - name: sdo_type + type: keyword + - name: severity + type: keyword + - name: source_description + type: keyword + - name: source_tlp + type: keyword + - name: sources + type: group + fields: + - name: first_seen + type: date + - name: last_seen + type: date + - name: name + type: keyword + - name: score + type: long + - name: tlp + type: keyword + - name: tags_list + type: keyword + - name: tags_object + type: flattened + - name: tlp_value + type: keyword + description: Returns the TLP value associated with the threat data object. + - name: valid_from + type: date + - name: valid_until + type: date diff --git a/packages/ti_cyware_intel_exchange/data_stream/indicator/fields/is-transform-source-true.yml b/packages/ti_cyware_intel_exchange/data_stream/indicator/fields/is-transform-source-true.yml new file mode 100644 index 00000000000..fd4766eacd5 --- /dev/null +++ b/packages/ti_cyware_intel_exchange/data_stream/indicator/fields/is-transform-source-true.yml @@ -0,0 +1,4 @@ +- name: labels.is_transform_source + type: constant_keyword + description: Distinguishes between documents that are a source for a transform and documents that are an output of a transform, to facilitate easier filtering. + value: "true" diff --git a/packages/ti_cyware_intel_exchange/data_stream/indicator/lifecycle.yml b/packages/ti_cyware_intel_exchange/data_stream/indicator/lifecycle.yml new file mode 100644 index 00000000000..5a4af9095b7 --- /dev/null +++ b/packages/ti_cyware_intel_exchange/data_stream/indicator/lifecycle.yml @@ -0,0 +1 @@ +data_retention: "5d" diff --git a/packages/ti_cyware_intel_exchange/data_stream/indicator/manifest.yml b/packages/ti_cyware_intel_exchange/data_stream/indicator/manifest.yml new file mode 100644 index 00000000000..ca854d36cee --- /dev/null +++ b/packages/ti_cyware_intel_exchange/data_stream/indicator/manifest.yml @@ -0,0 +1,143 @@ +title: Collect Indicator logs from Cyware Intel Exchange +type: logs +streams: + - input: cel + title: Indicator + description: Collect Indicator logs from Cyware Intel Exchange. + template_path: cel.yml.hbs + enabled: true + vars: + - name: url + type: text + title: URL + description: Base URL. The url must be in the format- https://.cyware.com/ctixapi + required: true + show_user: true + - name: access_id + type: text + title: Access ID + description: Access ID of the Cyware Intel Exchange API. + required: true + show_user: true + secret: true + - name: secret_key + type: password + title: Secret Key + description: Secret Key of the Cyware Intel Exchange API. + required: true + show_user: true + secret: true + - name: initial_interval + type: text + title: Initial Interval + description: How far back to pull the Indicator logs from Cyware Intel Exchange API. Supported units for this parameter are h/m/s. + default: 24h + multi: false + required: true + show_user: true + - name: interval + type: text + title: Interval + description: Duration between requests to the Cyware Intel Exchange API. Supported units for this parameter are h/m/s. + default: 5m + multi: false + required: true + show_user: true + - name: batch_size + type: integer + title: Batch Size + description: Batch size for the response of the Cyware Intel Exchange API. + default: 500 + multi: false + required: true + show_user: false + - name: http_client_timeout + type: text + title: HTTP Client Timeout + description: Duration before declaring that the HTTP client connection has timed out. Supported time units are ns, us, ms, s, m, h. + multi: false + required: true + show_user: false + default: 30s + - name: enable_request_tracer + type: bool + title: Enable request tracing + multi: false + required: false + show_user: false + description: >- + The request tracer logs requests and responses to the agent's local file-system for debugging configurations. + Enabling this request tracing compromises security and should only be used for debugging. Disabling the request + tracer will delete any stored traces. + See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-cel.html#_resource_tracer_enable) + for details. + default: false + - name: preserve_original_event + required: false + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original`. + type: bool + multi: false + default: false + - name: preserve_duplicate_custom_fields + required: false + show_user: false + title: Preserve duplicate custom fields + description: Preserve ti_cyware_intel_exchange.indicator fields that were copied to Elastic Common Schema (ECS) fields. + type: bool + multi: false + default: false + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - ti_cyware_intel_exchange-indicator + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. + - name: proxy_url + type: text + title: Proxy URL + multi: false + required: false + show_user: false + description: URL to proxy connections in the form of http[s]://:@:. Please ensure your username and password are in URL encoded format. + - name: ssl + type: yaml + title: SSL Configuration + description: SSL configuration options. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/configuration-ssl.html#ssl-common-config) for details. + multi: false + required: false + show_user: false + default: | + #certificate_authorities: + # - | + # -----BEGIN CERTIFICATE----- + # MIIDCjCCAfKgAwIBAgITJ706Mu2wJlKckpIvkWxEHvEyijANBgkqhkiG9w0BAQsF + # ADAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwIBcNMTkwNzIyMTkyOTA0WhgPMjExOTA2 + # MjgxOTI5MDRaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB + # BQADggEPADCCAQoCggEBANce58Y/JykI58iyOXpxGfw0/gMvF0hUQAcUrSMxEO6n + # fZRA49b4OV4SwWmA3395uL2eB2NB8y8qdQ9muXUdPBWE4l9rMZ6gmfu90N5B5uEl + # 94NcfBfYOKi1fJQ9i7WKhTjlRkMCgBkWPkUokvBZFRt8RtF7zI77BSEorHGQCk9t + # /D7BS0GJyfVEhftbWcFEAG3VRcoMhF7kUzYwp+qESoriFRYLeDWv68ZOvG7eoWnP + # PsvZStEVEimjvK5NSESEQa9xWyJOmlOKXhkdymtcUd/nXnx6UTCFgnkgzSdTWV41 + # CI6B6aJ9svCTI2QuoIq2HxX/ix7OvW1huVmcyHVxyUECAwEAAaNTMFEwHQYDVR0O + # BBYEFPwN1OceFGm9v6ux8G+DZ3TUDYxqMB8GA1UdIwQYMBaAFPwN1OceFGm9v6ux + # 8G+DZ3TUDYxqMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAG5D + # 874A4YI7YUwOVsVAdbWtgp1d0zKcPRR+r2OdSbTAV5/gcS3jgBJ3i1BN34JuDVFw + # 3DeJSYT3nxy2Y56lLnxDeF8CUTUtVQx3CuGkRg1ouGAHpO/6OqOhwLLorEmxi7tA + # H2O8mtT0poX5AnOAhzVy7QW0D/k4WaoLyckM5hUa6RtvgvLxOwA0U+VGurCDoctu + # 8F4QOgTAWyh8EZIwaKCliFRSynDpv3JTUwtfZkxo6K6nce1RhCWFAsMvDZL8Dgc0 + # yvgJ38BRsFOtkRuAGSf6ZUwTO8JJRRIFnpUzXflAnGivK9M13D5GEQMmIl6U9Pvk + # sxSmbIUfc2SGJGCJD4I= + # -----END CERTIFICATE----- diff --git a/packages/ti_cyware_intel_exchange/data_stream/indicator/sample_event.json b/packages/ti_cyware_intel_exchange/data_stream/indicator/sample_event.json new file mode 100644 index 00000000000..c5becae5805 --- /dev/null +++ b/packages/ti_cyware_intel_exchange/data_stream/indicator/sample_event.json @@ -0,0 +1,105 @@ +{ + "@timestamp": "2025-06-30T10:28:16.273Z", + "agent": { + "ephemeral_id": "751c79ce-98e4-4341-ba93-91f131896885", + "id": "ff30387c-91f0-4c7d-b81b-496c6f23179a", + "name": "elastic-agent-78627", + "type": "filebeat", + "version": "8.18.0" + }, + "data_stream": { + "dataset": "ti_cyware_intel_exchange.indicator", + "namespace": "10992", + "type": "logs" + }, + "ecs": { + "version": "8.17.0" + }, + "elastic_agent": { + "id": "ff30387c-91f0-4c7d-b81b-496c6f23179a", + "snapshot": false, + "version": "8.18.0" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "threat" + ], + "created": "2025-05-27T14:04:51.651Z", + "dataset": "ti_cyware_intel_exchange.indicator", + "id": "f36749f2-776f-4153-b840-08bad5fb18b1", + "ingested": "2025-07-18T08:58:04Z", + "kind": "enrichment", + "original": "{\"analyst_score\":null,\"analyst_tlp\":null,\"country\":null,\"created\":1746812972,\"ctix_created\":1748354691.651628,\"ctix_modified\":1751279296.273555,\"ctix_score\":90,\"ctix_tlp\":null,\"custom_scores\":null,\"id\":\"f36749f2-776f-4153-b840-08bad5fb18b1\",\"indicator_type\":{\"attribute_field\":\"MD5\",\"type\":\"file\"},\"ioc_type\":\"file\",\"is_actioned\":false,\"is_deprecated\":false,\"is_false_positive\":false,\"is_reviewed\":false,\"is_revoked\":false,\"is_whitelist\":false,\"modified\":1748354676.716103,\"name\":\"e8c5c5829b630dcf61b55f271ac6c085\",\"sdo_name\":\"e8c5c5829b630dcf61b55f271ac6c085\",\"sdo_type\":\"indicator\",\"severity\":\"UNKNOWN\",\"source_tlp\":\"NONE\",\"sources\":[{\"first_seen\":1746812972,\"last_seen\":null,\"name\":\"Vault\",\"score\":100,\"tlp\":\"WHITE\"}],\"tags\":[\"brand impersonation\",\"cryptocurrency\",\"Apple\"],\"valid_from\":1746812972,\"valid_until\":null}", + "severity": 99, + "type": [ + "indicator" + ] + }, + "input": { + "type": "cel" + }, + "observer": { + "product": "Threat Intelligence Management", + "vendor": "Cyware" + }, + "tags": [ + "preserve_original_event", + "forwarded", + "ti_cyware_intel_exchange-indicator", + "brand impersonation", + "cryptocurrency", + "Apple" + ], + "threat": { + "indicator": { + "first_seen": [ + "2025-05-09T17:49:32.000Z" + ], + "marking": { + "tlp": [ + "WHITE" + ] + }, + "modified_at": "2025-05-27T14:04:36.716Z", + "name": [ + "e8c5c5829b630dcf61b55f271ac6c085" + ], + "provider": [ + "Vault" + ], + "type": "file" + } + }, + "ti_cyware_intel_exchange": { + "indicator": { + "created": "2025-05-09T17:49:32.000Z", + "ctix_score": 90, + "indicator_type": { + "attribute_field": "MD5", + "type": "file" + }, + "is_actioned": false, + "is_deprecated": false, + "is_false_positive": false, + "is_reviewed": false, + "is_revoked": false, + "is_whitelist": false, + "name": "e8c5c5829b630dcf61b55f271ac6c085", + "sdo_name": "e8c5c5829b630dcf61b55f271ac6c085", + "sdo_type": "indicator", + "source_tlp": "NONE", + "sources": [ + { + "score": 100 + } + ], + "tags_list": [ + "brand impersonation", + "cryptocurrency", + "Apple" + ], + "valid_from": "2025-05-09T17:49:32.000Z" + } + } +} diff --git a/packages/ti_cyware_intel_exchange/docs/README.md b/packages/ti_cyware_intel_exchange/docs/README.md new file mode 100644 index 00000000000..5c264c93462 --- /dev/null +++ b/packages/ti_cyware_intel_exchange/docs/README.md @@ -0,0 +1,227 @@ +# Cyware Intel Exchange + +## Overview + +[Cyware Intel Exchange](https://www.cyware.com/products/intel-exchange) is an intelligent client-server exchange that leverages advanced technologies like Artificial Intelligence and Machine Learning to automatically ingest, analyze, correlate and act upon the threat data ingested from multiple external sources and internally deployed security tools. + +## Data streams + +The Cyware Intel Exchange integration collects the following events: +- **[Indicator](https://ctixapiv3.cyware.com/rules/save-result-set/retrieve-saved-result-set-data)** - This fetches all the saved result set data for conditional IOCs present in the application.. + +## Requirements + +### Agentless-enabled integration +Agentless integrations allow you to collect data without having to manage Elastic Agent in your cloud. They make manual agent deployment unnecessary, so you can focus on your data instead of the agent that collects it. For more information, refer to [Agentless integrations](https://www.elastic.co/guide/en/serverless/current/security-agentless-integrations.html) and the [Agentless integrations FAQ](https://www.elastic.co/guide/en/serverless/current/agentless-integration-troubleshooting.html). + +Agentless deployments are only supported in Elastic Serverless and Elastic Cloud environments. This functionality is in beta and is subject to change. Beta features are not subject to the support SLA of official GA features. + +### Agent-based installation +Elastic Agent must be installed. For more details, check the Elastic Agent [installation instructions](docs-content://reference/fleet/install-elastic-agents.md). You can install only one Elastic Agent per host. + +## Compatibility + +For Rest API, this module has been tested against the **[CTIX API v3](https://ctixapiv3.cyware.com/intel-exchange-api-reference)** version. + +## Setup + +**Note** - Before you start the setup, ensure that you have **Create** and **Update** permissions for **CTIX Integrators**. + +### Follow below steps to generate Open API credentials for collecting data from the CTIX API: + +1. Go to **Administration** > **Integration Management**. +2. In **Third Party Developers**, click **CTIX Integrators**. +3. Click **Add New**. Enter the following details: + - **Name**: Enter a unique name for the API credentials in 50 characters. + - **Description**: Enter a description for the credentials within 1000 characters. + - **Expiry Date**: Select an expiry date for open API keys. To apply an expiration date for the credentials, you can select **Expires On** and select the date. To ensure the credentials never expire, you can select **Never Expire**. +4. Click **Add New**. +5. Click **Download** to download the API credentials in CSV format. You can also click **Copy** to copy the endpoint URL, secret key, and access ID. + +For more details, refer to the [Authentication](https://ctixapiv3.cyware.com/authentication) documentation and the guide on how to [Generate Open API Credentials](https://techdocs.cyware.com/en/299670-447852-configure-open-api.html). + +### Enable the integration in Elastic + +1. In Kibana navigate to **Management** > **Integrations**. +2. In the search top bar, type **Cyware Intel Exchange**. +3. Select the **Cyware Intel Exchange** integration afrom the search results. +4. Click on the "Add Cyware Intel Exchange" button to add the integration. +5. Add all the required integration configuration parameters: URL, Access ID and Secret Key. +6. Save the integration. + +## Logs reference + +### Indicator + +This is the `Indicator` dataset. + +#### Example + +An example event for `indicator` looks as following: + +```json +{ + "@timestamp": "2025-06-30T10:28:16.273Z", + "agent": { + "ephemeral_id": "751c79ce-98e4-4341-ba93-91f131896885", + "id": "ff30387c-91f0-4c7d-b81b-496c6f23179a", + "name": "elastic-agent-78627", + "type": "filebeat", + "version": "8.18.0" + }, + "data_stream": { + "dataset": "ti_cyware_intel_exchange.indicator", + "namespace": "10992", + "type": "logs" + }, + "ecs": { + "version": "8.17.0" + }, + "elastic_agent": { + "id": "ff30387c-91f0-4c7d-b81b-496c6f23179a", + "snapshot": false, + "version": "8.18.0" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "threat" + ], + "created": "2025-05-27T14:04:51.651Z", + "dataset": "ti_cyware_intel_exchange.indicator", + "id": "f36749f2-776f-4153-b840-08bad5fb18b1", + "ingested": "2025-07-18T08:58:04Z", + "kind": "enrichment", + "original": "{\"analyst_score\":null,\"analyst_tlp\":null,\"country\":null,\"created\":1746812972,\"ctix_created\":1748354691.651628,\"ctix_modified\":1751279296.273555,\"ctix_score\":90,\"ctix_tlp\":null,\"custom_scores\":null,\"id\":\"f36749f2-776f-4153-b840-08bad5fb18b1\",\"indicator_type\":{\"attribute_field\":\"MD5\",\"type\":\"file\"},\"ioc_type\":\"file\",\"is_actioned\":false,\"is_deprecated\":false,\"is_false_positive\":false,\"is_reviewed\":false,\"is_revoked\":false,\"is_whitelist\":false,\"modified\":1748354676.716103,\"name\":\"e8c5c5829b630dcf61b55f271ac6c085\",\"sdo_name\":\"e8c5c5829b630dcf61b55f271ac6c085\",\"sdo_type\":\"indicator\",\"severity\":\"UNKNOWN\",\"source_tlp\":\"NONE\",\"sources\":[{\"first_seen\":1746812972,\"last_seen\":null,\"name\":\"Vault\",\"score\":100,\"tlp\":\"WHITE\"}],\"tags\":[\"brand impersonation\",\"cryptocurrency\",\"Apple\"],\"valid_from\":1746812972,\"valid_until\":null}", + "severity": 99, + "type": [ + "indicator" + ] + }, + "input": { + "type": "cel" + }, + "observer": { + "product": "Threat Intelligence Management", + "vendor": "Cyware" + }, + "tags": [ + "preserve_original_event", + "forwarded", + "ti_cyware_intel_exchange-indicator", + "brand impersonation", + "cryptocurrency", + "Apple" + ], + "threat": { + "indicator": { + "first_seen": [ + "2025-05-09T17:49:32.000Z" + ], + "marking": { + "tlp": [ + "WHITE" + ] + }, + "modified_at": "2025-05-27T14:04:36.716Z", + "name": [ + "e8c5c5829b630dcf61b55f271ac6c085" + ], + "provider": [ + "Vault" + ], + "type": "file" + } + }, + "ti_cyware_intel_exchange": { + "indicator": { + "created": "2025-05-09T17:49:32.000Z", + "ctix_score": 90, + "indicator_type": { + "attribute_field": "MD5", + "type": "file" + }, + "is_actioned": false, + "is_deprecated": false, + "is_false_positive": false, + "is_reviewed": false, + "is_revoked": false, + "is_whitelist": false, + "name": "e8c5c5829b630dcf61b55f271ac6c085", + "sdo_name": "e8c5c5829b630dcf61b55f271ac6c085", + "sdo_type": "indicator", + "source_tlp": "NONE", + "sources": [ + { + "score": 100 + } + ], + "tags_list": [ + "brand impersonation", + "cryptocurrency", + "Apple" + ], + "valid_from": "2025-05-09T17:49:32.000Z" + } + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| event.dataset | Event dataset. | constant_keyword | +| event.module | Event module. | constant_keyword | +| input.type | Type of Filebeat input. | keyword | +| labels.is_transform_source | Distinguishes between documents that are a source for a transform and documents that are an output of a transform, to facilitate easier filtering. | constant_keyword | +| log.offset | Log offset. | long | +| observer.product | | constant_keyword | +| observer.vendor | | constant_keyword | +| ti_cyware_intel_exchange.indicator.analyst_description | | keyword | +| ti_cyware_intel_exchange.indicator.analyst_score | | long | +| ti_cyware_intel_exchange.indicator.analyst_tlp | | keyword | +| ti_cyware_intel_exchange.indicator.country | | keyword | +| ti_cyware_intel_exchange.indicator.created | | date | +| ti_cyware_intel_exchange.indicator.ctix_created | | date | +| ti_cyware_intel_exchange.indicator.ctix_modified | | date | +| ti_cyware_intel_exchange.indicator.ctix_score | | long | +| ti_cyware_intel_exchange.indicator.ctix_tlp | | keyword | +| ti_cyware_intel_exchange.indicator.custom_attributes | | flattened | +| ti_cyware_intel_exchange.indicator.custom_scores | | long | +| ti_cyware_intel_exchange.indicator.external_references | | flattened | +| ti_cyware_intel_exchange.indicator.id | | keyword | +| ti_cyware_intel_exchange.indicator.indicator_type.attribute_field | | keyword | +| ti_cyware_intel_exchange.indicator.indicator_type.type | | keyword | +| ti_cyware_intel_exchange.indicator.ioc_type | | keyword | +| ti_cyware_intel_exchange.indicator.ip | | ip | +| ti_cyware_intel_exchange.indicator.is_actioned | | boolean | +| ti_cyware_intel_exchange.indicator.is_deprecated | Returns a value to indicate if the threat data object is deprecated. | boolean | +| ti_cyware_intel_exchange.indicator.is_false_positive | Returns a value to indicate if the object is false positive. | boolean | +| ti_cyware_intel_exchange.indicator.is_reviewed | | boolean | +| ti_cyware_intel_exchange.indicator.is_revoked | | boolean | +| ti_cyware_intel_exchange.indicator.is_whitelist | Returns a value to indicate if the threat data object is whitelisted. | boolean | +| ti_cyware_intel_exchange.indicator.modified | | date | +| ti_cyware_intel_exchange.indicator.name | | keyword | +| ti_cyware_intel_exchange.indicator.report_types | | keyword | +| ti_cyware_intel_exchange.indicator.sdo_ip | | ip | +| ti_cyware_intel_exchange.indicator.sdo_name | | keyword | +| ti_cyware_intel_exchange.indicator.sdo_type | | keyword | +| ti_cyware_intel_exchange.indicator.severity | | keyword | +| ti_cyware_intel_exchange.indicator.source_description | | keyword | +| ti_cyware_intel_exchange.indicator.source_tlp | | keyword | +| ti_cyware_intel_exchange.indicator.sources.first_seen | | date | +| ti_cyware_intel_exchange.indicator.sources.last_seen | | date | +| ti_cyware_intel_exchange.indicator.sources.name | | keyword | +| ti_cyware_intel_exchange.indicator.sources.score | | long | +| ti_cyware_intel_exchange.indicator.sources.tlp | | keyword | +| ti_cyware_intel_exchange.indicator.tags_list | | keyword | +| ti_cyware_intel_exchange.indicator.tags_object | | flattened | +| ti_cyware_intel_exchange.indicator.tlp_value | Returns the TLP value associated with the threat data object. | keyword | +| ti_cyware_intel_exchange.indicator.valid_from | | date | +| ti_cyware_intel_exchange.indicator.valid_until | | date | + diff --git a/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/fields/base-fields.yml b/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/fields/base-fields.yml new file mode 100644 index 00000000000..80a48b71c4b --- /dev/null +++ b/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module. + value: ti_cyware_intel_exchange +- name: event.dataset + type: constant_keyword + description: Event dataset. + value: ti_cyware_intel_exchange.indicator +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/fields/beats.yml b/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/fields/beats.yml new file mode 100644 index 00000000000..4084f1dc7f5 --- /dev/null +++ b/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/fields/beats.yml @@ -0,0 +1,6 @@ +- name: input.type + type: keyword + description: Type of filebeat input. +- name: log.offset + type: long + description: Log offset. diff --git a/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/fields/ecs.yml b/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/fields/ecs.yml new file mode 100644 index 00000000000..a288eeef17f --- /dev/null +++ b/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/fields/ecs.yml @@ -0,0 +1,75 @@ +- external: ecs + name: ecs.version + type: keyword +- external: ecs + name: event.category + type: keyword +- external: ecs + name: event.created + type: date +- external: ecs + name: event.id + type: keyword +- external: ecs + name: event.ingested + type: date +- external: ecs + name: event.kind + type: keyword +- external: ecs + name: event.type + type: keyword +- external: ecs + name: message + type: keyword +- external: ecs + name: observer.product + type: constant_keyword +- external: ecs + name: observer.vendor + type: constant_keyword +- external: ecs + name: related.hash + type: keyword +- external: ecs + name: related.ip + type: ip +- external: ecs + name: related.user + type: keyword +- external: ecs + name: tags + type: keyword +- external: ecs + name: threat.indicator.description + type: keyword +- external: ecs + name: threat.indicator.first_seen + type: date +- external: ecs + name: threat.indicator.ip + type: ip +- external: ecs + name: threat.indicator.last_seen + type: date +- external: ecs + name: threat.indicator.marking.tlp + type: keyword +- external: ecs + name: threat.indicator.modified_at + type: date +- external: ecs + name: threat.indicator.name + type: keyword +- external: ecs + name: threat.indicator.provider + type: keyword +- external: ecs + name: threat.indicator.geo.country_name + type: keyword +- external: ecs + name: threat.indicator.reference + type: keyword +- external: ecs + name: threat.indicator.type + type: keyword diff --git a/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/fields/fields.yml b/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/fields/fields.yml new file mode 100644 index 00000000000..b0c23587506 --- /dev/null +++ b/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/fields/fields.yml @@ -0,0 +1,100 @@ +- name: ti_cyware_intel_exchange + type: group + fields: + - name: indicator + type: group + fields: + - name: analyst_description + type: keyword + - name: analyst_score + type: long + - name: analyst_tlp + type: keyword + - name: country + type: keyword + - name: created + type: date + - name: ctix_created + type: date + - name: ctix_modified + type: date + - name: ctix_score + type: long + - name: ctix_tlp + type: keyword + - name: custom_attributes + type: flattened + - name: custom_scores + type: long + - name: external_references + type: keyword + - name: id + type: keyword + - name: indicator_type + type: group + fields: + - name: attribute_field + type: keyword + - name: type + type: keyword + - name: ioc_type + type: keyword + - name: ip + type: ip + - name: is_actioned + type: boolean + - name: is_deprecated + type: boolean + description: Returns a value to indicate if the threat data object is deprecated. + - name: is_false_positive + type: boolean + description: Returns a value to indicate if the object is false positive. + - name: is_reviewed + type: boolean + - name: is_revoked + type: boolean + - name: is_whitelist + type: boolean + description: Returns a value to indicate if the threat data object is whitelisted. + - name: modified + type: date + - name: name + type: keyword + - name: report_types + type: keyword + - name: sdo_ip + type: ip + - name: sdo_name + type: keyword + - name: sdo_type + type: keyword + - name: severity + type: keyword + - name: source_description + type: keyword + - name: source_tlp + type: keyword + - name: sources + type: group + fields: + - name: first_seen + type: date + - name: last_seen + type: date + - name: name + type: keyword + - name: score + type: long + - name: tlp + type: keyword + - name: tags_list + type: keyword + - name: tags_object + type: flattened + - name: tlp_value + type: keyword + description: Returns the TLP value associated with the threat data object. + - name: valid_from + type: date + - name: valid_until + type: date diff --git a/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/fields/is-transform-source-false.yml b/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/fields/is-transform-source-false.yml new file mode 100644 index 00000000000..490a079e7a7 --- /dev/null +++ b/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/fields/is-transform-source-false.yml @@ -0,0 +1,4 @@ +- name: labels.is_transform_source + type: constant_keyword + description: Distinguishes between documents that are a source for a transform and documents that are an output of a transform, to facilitate easier filtering. + value: "false" diff --git a/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/manifest.yml b/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/manifest.yml new file mode 100644 index 00000000000..f5296fd0c0a --- /dev/null +++ b/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/manifest.yml @@ -0,0 +1,18 @@ +start: true +destination_index_template: + settings: + index: + sort: + field: + - "@timestamp" + order: + - desc + mappings: + dynamic: true + dynamic_templates: + - strings_as_keyword: + match_mapping_type: string + mapping: + ignore_above: 1024 + type: keyword + date_detection: false diff --git a/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/transform.yml b/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/transform.yml new file mode 100644 index 00000000000..304e9d3f8cf --- /dev/null +++ b/packages/ti_cyware_intel_exchange/elasticsearch/transform/latest_ioc/transform.yml @@ -0,0 +1,42 @@ +# Use of "*" to use all namespaces defined. +source: + index: + - "logs-ti_cyware_intel_exchange.indicator-*" +# The version suffix on the dest.index should be incremented if a breaking change +# is made to the index mapping. You must also bump the fleet_transform_version +# for any change to this transform configuration to take effect. The old destination +# index is not automatically deleted. We are dependent on https://github.com/elastic/package-spec/issues/523 to give +# us that ability in order to prevent having duplicate IoC data and prevent query +# time field type conflicts. +dest: + index: "logs-ti_cyware_intel_exchange_latest.dest_indicator-1" + aliases: + - alias: "logs-ti_cyware_intel_exchange_latest.indicator" + move_on_creation: true +latest: + unique_key: + - event.dataset + - threat.indicator.type + - event.id + sort: '@timestamp' +description: Latest IOC Indicator data retrieved from Cyware Intel Exchange API. +frequency: 2m +settings: + deduce_mappings: false + unattended: true +sync: + time: + field: "event.ingested" + # Updated to 120s because of refresh delay in Serverless. With default 60s, + # sometimes transform wouldn't process all documents. + delay: 120s +retention_policy: + time: + field: "@timestamp" + max_age: 30d +_meta: + managed: true + # Bump this version to delete, reinstall, and restart the transform during + # package installation. + fleet_transform_version: 0.1.0 + run_as_kibana_system: false diff --git a/packages/ti_cyware_intel_exchange/img/cyware-indicator-dashboard.png b/packages/ti_cyware_intel_exchange/img/cyware-indicator-dashboard.png new file mode 100644 index 00000000000..904f495d935 Binary files /dev/null and b/packages/ti_cyware_intel_exchange/img/cyware-indicator-dashboard.png differ diff --git a/packages/ti_cyware_intel_exchange/img/cyware_logo.svg b/packages/ti_cyware_intel_exchange/img/cyware_logo.svg new file mode 100644 index 00000000000..551b513db5b --- /dev/null +++ b/packages/ti_cyware_intel_exchange/img/cyware_logo.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/packages/ti_cyware_intel_exchange/kibana/dashboard/ti_cyware_intel_exchange-56ee88b2-39b0-44f1-a122-46ff83bdbcb0.json b/packages/ti_cyware_intel_exchange/kibana/dashboard/ti_cyware_intel_exchange-56ee88b2-39b0-44f1-a122-46ff83bdbcb0.json new file mode 100644 index 00000000000..d496d64fe5f --- /dev/null +++ b/packages/ti_cyware_intel_exchange/kibana/dashboard/ti_cyware_intel_exchange-56ee88b2-39b0-44f1-a122-46ff83bdbcb0.json @@ -0,0 +1,1504 @@ +{ + "attributes": { + "controlGroupInput": { + "chainingSystem": "HIERARCHICAL", + "controlStyle": "oneLine", + "ignoreParentSettingsJSON": { + "ignoreFilters": false, + "ignoreQuery": false, + "ignoreTimerange": false, + "ignoreValidations": false + }, + "panelsJSON": { + "30cefbe2-35ca-4dde-ab76-d8904d362364": { + "explicitInput": { + "dataViewId": "logs-*", + "fieldName": "threat.indicator.type", + "searchTechnique": "prefix", + "selectedOptions": [], + "sort": { + "by": "_count", + "direction": "desc" + }, + "title": "Indicator Type" + }, + "grow": true, + "order": 1, + "type": "optionsListControl", + "width": "small" + }, + "d24688bb-06e4-40b4-8b54-25d2f277dc0c": { + "explicitInput": { + "dataViewId": "logs-*", + "fieldName": "ti_cyware_intel_exchange.indicator.sdo_type", + "searchTechnique": "prefix", + "selectedOptions": [], + "sort": { + "by": "_count", + "direction": "desc" + }, + "title": "SDO Type" + }, + "grow": true, + "order": 0, + "type": "optionsListControl", + "width": "small" + }, + "f1f36c3b-9b2d-4c5b-90e5-c844c23be867": { + "explicitInput": { + "dataViewId": "logs-*", + "fieldName": "threat.indicator.marking.tlp", + "searchTechnique": "prefix", + "selectedOptions": [], + "sort": { + "by": "_count", + "direction": "desc" + }, + "title": "Source TLP" + }, + "grow": true, + "order": 2, + "type": "optionsListControl", + "width": "small" + } + }, + "showApplySelections": false + }, + "description": "Overview of Indicator logs from the Cyware CTIX", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "data_stream.dataset", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "ti_cyware_intel_exchange.indicator" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "ti_cyware_intel_exchange.indicator" + } + } + }, + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "labels.is_transform_source", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", + "key": "labels.is_transform_source", + "negate": false, + "params": { + "query": "false" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "labels.is_transform_source": "false" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "description": "", + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "sort": [] + }, + "gridData": { + "h": 14, + "i": "78816450-3cb6-450b-8d07-126aa1fe488e", + "w": 48, + "x": 0, + "y": 45 + }, + "panelIndex": "78816450-3cb6-450b-8d07-126aa1fe488e", + "panelRefName": "panel_78816450-3cb6-450b-8d07-126aa1fe488e", + "title": "Indicators Essential Details [Logs Cyware Intel Exchange]", + "type": "search" + }, + { + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "id": "", + "params": { + "fontSize": 12, + "markdown": "#### Cyware Intel Exchange\n \n**Indicator** \n\n#### Description\n\nThis dashboard provides comprehensive visibility into indicator activity within the Cyware Intel Exchange Exchange (CTIX) platform using the Indicator data stream.\n\nIt displays the total number of indicators (IOCs), along with counts of revoked, deprecated, actioned, and reviewed indicators to reflect their current lifecycle status. The dashboard includes time-based trends of indicator creation, and breaks down indicators by Traffic Light Protocol (TLP), type, SDO type, severity, and country of origin — offering insight into both the nature and reliability of threats. It also highlights the top 10 contributing sources and includes a saved search displaying essential IOC details such as type, source, status, and associated metadata to support faster analysis.\n\n**[Integration Page](/app/integrations/detail/ti_cyware_intel_exchange/overview)**", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 30, + "i": "7c4e052e-d8ec-4b21-871c-d569d94118e1", + "w": 8, + "x": 0, + "y": 0 + }, + "panelIndex": "7c4e052e-d8ec-4b21-871c-d569d94118e1", + "type": "visualization" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-66c8de3c-9fb7-4a8c-ac35-64ccccf64504", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "logs-*", + "layers": { + "66c8de3c-9fb7-4a8c-ac35-64ccccf64504": { + "columnOrder": [ + "efc90b85-e3f2-4675-b5c9-8120455d5195", + "f0bc2c76-3b19-43c3-9dbd-43d165ec8b23" + ], + "columns": { + "efc90b85-e3f2-4675-b5c9-8120455d5195": { + "customLabel": true, + "dataType": "number", + "isBucketed": true, + "label": "CTIX Score", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "f0bc2c76-3b19-43c3-9dbd-43d165ec8b23", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "secondaryFields": [], + "size": 5 + }, + "scale": "ordinal", + "sourceField": "ti_cyware_intel_exchange.indicator.ctix_score" + }, + "f0bc2c76-3b19-43c3-9dbd-43d165ec8b23": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Count", + "operationType": "count", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "___records___" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "indexPatternId": "logs-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "currentIndexPatternId": "logs-*", + "layers": {} + }, + "textBased": { + "indexPatternRefs": [ + { + "id": "logs-*", + "timeField": "@timestamp", + "title": "logs-*" + } + ], + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "categoryDisplay": "default", + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "66c8de3c-9fb7-4a8c-ac35-64ccccf64504", + "layerType": "data", + "legendDisplay": "show", + "metrics": [ + "f0bc2c76-3b19-43c3-9dbd-43d165ec8b23" + ], + "nestedLegend": false, + "numberDisplay": "percent", + "primaryGroups": [ + "efc90b85-e3f2-4675-b5c9-8120455d5195" + ], + "truncateLegend": false + } + ], + "shape": "pie" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsPie" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 15, + "i": "55425d85-e34d-4a3a-9e2e-82745b1acc93", + "w": 15, + "x": 0, + "y": 30 + }, + "panelIndex": "55425d85-e34d-4a3a-9e2e-82745b1acc93", + "title": "Indicators by CTIX Score [Logs Cyware Intel Exchange]", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-ccfd6f46-9413-4ca3-bc3e-c5cf60115fcf", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "ccfd6f46-9413-4ca3-bc3e-c5cf60115fcf": { + "columnOrder": [ + "5e2b8291-10ba-40c5-be94-1bb147fc5494", + "43e6f5a7-0582-49d6-9f40-fdc26dfcbadd" + ], + "columns": { + "43e6f5a7-0582-49d6-9f40-fdc26dfcbadd": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Count", + "operationType": "count", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "___records___" + }, + "5e2b8291-10ba-40c5-be94-1bb147fc5494": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Indicator Type", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "43e6f5a7-0582-49d6-9f40-fdc26dfcbadd", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "threat.indicator.type" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "43e6f5a7-0582-49d6-9f40-fdc26dfcbadd" + ], + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "ccfd6f46-9413-4ca3-bc3e-c5cf60115fcf", + "layerType": "data", + "seriesType": "bar", + "xAccessor": "5e2b8291-10ba-40c5-be94-1bb147fc5494" + } + ], + "legend": { + "isVisible": true, + "position": "right", + "shouldTruncate": false + }, + "preferredSeriesType": "bar_percentage_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 15, + "i": "1c2cf6f5-20f1-42ab-afda-9ff1f9ad9dd7", + "w": 18, + "x": 30, + "y": 30 + }, + "panelIndex": "1c2cf6f5-20f1-42ab-afda-9ff1f9ad9dd7", + "title": "Indicators by Type [Logs Cyware Intel Exchange]", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-66c8de3c-9fb7-4a8c-ac35-64ccccf64504", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "logs-*", + "layers": { + "66c8de3c-9fb7-4a8c-ac35-64ccccf64504": { + "columnOrder": [ + "efc90b85-e3f2-4675-b5c9-8120455d5195", + "f0bc2c76-3b19-43c3-9dbd-43d165ec8b23" + ], + "columns": { + "efc90b85-e3f2-4675-b5c9-8120455d5195": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "TLP", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "f0bc2c76-3b19-43c3-9dbd-43d165ec8b23", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "secondaryFields": [], + "size": 6 + }, + "scale": "ordinal", + "sourceField": "threat.indicator.marking.tlp" + }, + "f0bc2c76-3b19-43c3-9dbd-43d165ec8b23": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Count", + "operationType": "count", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "___records___" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "indexPatternId": "logs-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "currentIndexPatternId": "logs-*", + "layers": {} + }, + "textBased": { + "indexPatternRefs": [ + { + "id": "logs-*", + "timeField": "@timestamp", + "title": "logs-*" + } + ], + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "categoryDisplay": "default", + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "66c8de3c-9fb7-4a8c-ac35-64ccccf64504", + "layerType": "data", + "legendDisplay": "show", + "metrics": [ + "f0bc2c76-3b19-43c3-9dbd-43d165ec8b23" + ], + "nestedLegend": false, + "numberDisplay": "percent", + "primaryGroups": [ + "efc90b85-e3f2-4675-b5c9-8120455d5195" + ], + "truncateLegend": false + } + ], + "shape": "pie" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsPie" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 15, + "i": "8cd93eb4-e06a-4869-bcb6-8129d4bd3536", + "w": 15, + "x": 15, + "y": 30 + }, + "panelIndex": "8cd93eb4-e06a-4869-bcb6-8129d4bd3536", + "title": "Indicators by TLP [Logs Cyware Intel Exchange]", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-8d0d3886-dab7-41ad-8881-6471b50fcd29", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "8d0d3886-dab7-41ad-8881-6471b50fcd29": { + "columnOrder": [ + "c5a719fa-86c6-4f73-95ca-02e2a4d3ea95" + ], + "columns": { + "c5a719fa-86c6-4f73-95ca-02e2a4d3ea95": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total Indicators", + "operationType": "count", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "___records___" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "color": "#6092C0", + "layerId": "8d0d3886-dab7-41ad-8881-6471b50fcd29", + "layerType": "data", + "metricAccessor": "c5a719fa-86c6-4f73-95ca-02e2a4d3ea95" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "hidePanelTitles": true, + "query": { + "language": "kuery", + "query": "" + }, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 13, + "i": "40da1b1f-5894-4f7a-b202-9ded2562d9b7", + "w": 8, + "x": 8, + "y": 0 + }, + "panelIndex": "40da1b1f-5894-4f7a-b202-9ded2562d9b7", + "title": "", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-8d0d3886-dab7-41ad-8881-6471b50fcd29", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "logs-*", + "layers": { + "8d0d3886-dab7-41ad-8881-6471b50fcd29": { + "columnOrder": [ + "c5a719fa-86c6-4f73-95ca-02e2a4d3ea95" + ], + "columns": { + "c5a719fa-86c6-4f73-95ca-02e2a4d3ea95": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "ti_cyware_intel_exchange.indicator.is_deprecated : true " + }, + "isBucketed": false, + "label": "Total Deprecated Indicators", + "operationType": "count", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "___records___" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "indexPatternId": "logs-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "currentIndexPatternId": "logs-*", + "layers": {} + }, + "textBased": { + "indexPatternRefs": [ + { + "id": "logs-*", + "timeField": "@timestamp", + "title": "logs-*" + } + ], + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "color": "#6092C0", + "layerId": "8d0d3886-dab7-41ad-8881-6471b50fcd29", + "layerType": "data", + "metricAccessor": "c5a719fa-86c6-4f73-95ca-02e2a4d3ea95" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "hidePanelTitles": true, + "query": { + "language": "kuery", + "query": "" + }, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 13, + "i": "5656f829-91b3-4ea4-adb0-040aafa36780", + "w": 8, + "x": 16, + "y": 0 + }, + "panelIndex": "5656f829-91b3-4ea4-adb0-040aafa36780", + "title": "", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-8d0d3886-dab7-41ad-8881-6471b50fcd29", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "logs-*", + "layers": { + "8d0d3886-dab7-41ad-8881-6471b50fcd29": { + "columnOrder": [ + "c5a719fa-86c6-4f73-95ca-02e2a4d3ea95" + ], + "columns": { + "c5a719fa-86c6-4f73-95ca-02e2a4d3ea95": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "ti_cyware_intel_exchange.indicator.is_revoked : true " + }, + "isBucketed": false, + "label": "Total Revoked Indicators", + "operationType": "count", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "___records___" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "indexPatternId": "logs-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "currentIndexPatternId": "logs-*", + "layers": {} + }, + "textBased": { + "indexPatternRefs": [ + { + "id": "logs-*", + "timeField": "@timestamp", + "title": "logs-*" + } + ], + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "color": "#6092C0", + "layerId": "8d0d3886-dab7-41ad-8881-6471b50fcd29", + "layerType": "data", + "metricAccessor": "c5a719fa-86c6-4f73-95ca-02e2a4d3ea95" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "hidePanelTitles": true, + "query": { + "language": "kuery", + "query": "" + }, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 13, + "i": "d0b23b56-a979-45ab-a4c9-63d36755f2c4", + "w": 8, + "x": 24, + "y": 0 + }, + "panelIndex": "d0b23b56-a979-45ab-a4c9-63d36755f2c4", + "title": "", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-7305312d-37a1-4d68-9ad7-f6fbd3d88dd6", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "logs-*", + "layers": { + "7305312d-37a1-4d68-9ad7-f6fbd3d88dd6": { + "columnOrder": [ + "935caff9-8a71-409d-9929-d4a1a6c3ac89", + "5c765772-c506-49ef-a729-09bb8709993d" + ], + "columns": { + "5c765772-c506-49ef-a729-09bb8709993d": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Count", + "operationType": "count", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "___records___" + }, + "935caff9-8a71-409d-9929-d4a1a6c3ac89": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Source Name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "5c765772-c506-49ef-a729-09bb8709993d", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "secondaryFields": [], + "size": 10 + }, + "scale": "ordinal", + "sourceField": "threat.indicator.provider" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "indexPatternId": "logs-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "currentIndexPatternId": "logs-*", + "layers": {} + }, + "textBased": { + "indexPatternRefs": [ + { + "id": "logs-*", + "timeField": "@timestamp", + "title": "logs-*" + } + ], + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "columnId": "5c765772-c506-49ef-a729-09bb8709993d", + "isMetric": true, + "isTransposed": false + }, + { + "columnId": "935caff9-8a71-409d-9929-d4a1a6c3ac89", + "isMetric": false, + "isTransposed": false + } + ], + "layerId": "7305312d-37a1-4d68-9ad7-f6fbd3d88dd6", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 13, + "i": "05334bfb-3bbf-490d-94cd-46589531e9d8", + "w": 16, + "x": 32, + "y": 0 + }, + "panelIndex": "05334bfb-3bbf-490d-94cd-46589531e9d8", + "title": "Top 10 Sources [Logs Cyware Intel Exchange]", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-0d7c180c-5a2a-4482-b96a-faed852aeabb", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "logs-*", + "layers": { + "0d7c180c-5a2a-4482-b96a-faed852aeabb": { + "columnOrder": [ + "9785dfa3-b18f-4040-af9f-a2b447fc8968", + "a05c2235-f1c5-44eb-8b89-3a6dcc5475a7" + ], + "columns": { + "9785dfa3-b18f-4040-af9f-a2b447fc8968": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Country", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "a05c2235-f1c5-44eb-8b89-3a6dcc5475a7", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "secondaryFields": [], + "size": 10 + }, + "scale": "ordinal", + "sourceField": "threat.indicator.geo.country_name" + }, + "a05c2235-f1c5-44eb-8b89-3a6dcc5475a7": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Count", + "operationType": "count", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "___records___" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "indexPatternId": "logs-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "currentIndexPatternId": "logs-*", + "layers": {} + }, + "textBased": { + "indexPatternRefs": [ + { + "id": "logs-*", + "timeField": "@timestamp", + "title": "logs-*" + } + ], + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "accessors": [ + "a05c2235-f1c5-44eb-8b89-3a6dcc5475a7" + ], + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "0d7c180c-5a2a-4482-b96a-faed852aeabb", + "layerType": "data", + "position": "top", + "seriesType": "bar", + "showGridlines": false, + "xAccessor": "9785dfa3-b18f-4040-af9f-a2b447fc8968" + } + ], + "legend": { + "isVisible": true, + "position": "right", + "shouldTruncate": false + }, + "preferredSeriesType": "bar_stacked", + "title": "Empty XY chart", + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 17, + "i": "34f5848f-bf06-4391-8f23-3238cef60d83", + "w": 18, + "x": 8, + "y": 13 + }, + "panelIndex": "34f5848f-bf06-4391-8f23-3238cef60d83", + "title": "Indicators by Country Name [Logs Cyware Intel Exchange]", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-14fcd521-3899-40fe-ab27-960fccc156a7", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "14fcd521-3899-40fe-ab27-960fccc156a7": { + "columnOrder": [ + "4e09b1e1-6be8-4921-99a6-a39a3fa0817f", + "1b43bbb0-3c1d-4b3c-90d1-65dd0587eff6" + ], + "columns": { + "1b43bbb0-3c1d-4b3c-90d1-65dd0587eff6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Count", + "operationType": "count", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "___records___" + }, + "4e09b1e1-6be8-4921-99a6-a39a3fa0817f": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "accessors": [ + "1b43bbb0-3c1d-4b3c-90d1-65dd0587eff6" + ], + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "14fcd521-3899-40fe-ab27-960fccc156a7", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "xAccessor": "4e09b1e1-6be8-4921-99a6-a39a3fa0817f" + } + ], + "legend": { + "isVisible": true, + "position": "right", + "shouldTruncate": false + }, + "preferredSeriesType": "line", + "title": "Empty XY chart", + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 17, + "i": "bfaee063-14e5-432f-a499-09a8a8cb4f16", + "w": 22, + "x": 26, + "y": 13 + }, + "panelIndex": "bfaee063-14e5-432f-a499-09a8a8cb4f16", + "title": "Indicators Over Time [Logs Cyware Intel Exchange]", + "type": "lens" + } + ], + "timeRestore": false, + "title": "[Logs Cyware Intel Exchange] Indicator", + "version": 3 + }, + "coreMigrationVersion": "8.8.0", + "created_at": "2025-07-18T07:53:45.773Z", + "id": "ti_cyware_intel_exchange-56ee88b2-39b0-44f1-a122-46ff83bdbcb0", + "managed": false, + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", + "type": "index-pattern" + }, + { + "id": "ti_cyware_intel_exchange-d3c12e4c-1d77-4c81-8223-5f909ffb433f", + "name": "78816450-3cb6-450b-8d07-126aa1fe488e:panel_78816450-3cb6-450b-8d07-126aa1fe488e", + "type": "search" + }, + { + "id": "logs-*", + "name": "55425d85-e34d-4a3a-9e2e-82745b1acc93:indexpattern-datasource-layer-66c8de3c-9fb7-4a8c-ac35-64ccccf64504", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "1c2cf6f5-20f1-42ab-afda-9ff1f9ad9dd7:indexpattern-datasource-layer-ccfd6f46-9413-4ca3-bc3e-c5cf60115fcf", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "8cd93eb4-e06a-4869-bcb6-8129d4bd3536:indexpattern-datasource-layer-66c8de3c-9fb7-4a8c-ac35-64ccccf64504", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "40da1b1f-5894-4f7a-b202-9ded2562d9b7:indexpattern-datasource-layer-8d0d3886-dab7-41ad-8881-6471b50fcd29", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "5656f829-91b3-4ea4-adb0-040aafa36780:indexpattern-datasource-layer-8d0d3886-dab7-41ad-8881-6471b50fcd29", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "d0b23b56-a979-45ab-a4c9-63d36755f2c4:indexpattern-datasource-layer-8d0d3886-dab7-41ad-8881-6471b50fcd29", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "05334bfb-3bbf-490d-94cd-46589531e9d8:indexpattern-datasource-layer-7305312d-37a1-4d68-9ad7-f6fbd3d88dd6", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "34f5848f-bf06-4391-8f23-3238cef60d83:indexpattern-datasource-layer-0d7c180c-5a2a-4482-b96a-faed852aeabb", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "bfaee063-14e5-432f-a499-09a8a8cb4f16:indexpattern-datasource-layer-14fcd521-3899-40fe-ab27-960fccc156a7", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "controlGroup_d24688bb-06e4-40b4-8b54-25d2f277dc0c:optionsListDataView", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "controlGroup_30cefbe2-35ca-4dde-ab76-d8904d362364:optionsListDataView", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "controlGroup_f1f36c3b-9b2d-4c5b-90e5-c844c23be867:optionsListDataView", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", + "type": "index-pattern" + } + ], + "type": "dashboard", + "typeMigrationVersion": "10.2.0", + "updated_by": "u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0" +} \ No newline at end of file diff --git a/packages/ti_cyware_intel_exchange/kibana/search/ti_cyware_intel_exchange-d3c12e4c-1d77-4c81-8223-5f909ffb433f.json b/packages/ti_cyware_intel_exchange/kibana/search/ti_cyware_intel_exchange-d3c12e4c-1d77-4c81-8223-5f909ffb433f.json new file mode 100644 index 00000000000..e1ff5c78743 --- /dev/null +++ b/packages/ti_cyware_intel_exchange/kibana/search/ti_cyware_intel_exchange-d3c12e4c-1d77-4c81-8223-5f909ffb433f.json @@ -0,0 +1,47 @@ +{ + "attributes": { + "columns": [ + "event.id", + "ti_cyware_intel_exchange.indicator.sdo_type", + "ti_cyware_intel_exchange.indicator.tlp_value", + "threat.indicator.description", + "ti_cyware_intel_exchange.indicator.is_whitelist", + "threat.indicator.geo.country_name" + ], + "description": "", + "grid": {}, + "hideChart": false, + "isTextBasedQuery": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "timeRestore": false, + "title": "Indicators Essential Details [Logs Cyware Intel Exchange]" + }, + "coreMigrationVersion": "8.8.0", + "created_at": "2025-07-18T07:52:50.651Z", + "id": "ti_cyware_intel_exchange-d3c12e4c-1d77-4c81-8223-5f909ffb433f", + "managed": true, + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search", + "typeMigrationVersion": "10.5.0" +} \ No newline at end of file diff --git a/packages/ti_cyware_intel_exchange/manifest.yml b/packages/ti_cyware_intel_exchange/manifest.yml new file mode 100644 index 00000000000..2e5e9301830 --- /dev/null +++ b/packages/ti_cyware_intel_exchange/manifest.yml @@ -0,0 +1,41 @@ +format_version: 3.3.2 +name: ti_cyware_intel_exchange +title: Cyware Intel Exchange +version: 0.1.0 +description: Collect logs from Cyware Intel Exchange with Elastic Agent. +type: integration +categories: ["security", "threat_intel"] +conditions: + kibana: + version: ^8.18.0 || ^9.0.0 + elastic: + subscription: basic +icons: + - src: /img/cyware_logo.svg + title: Sample logo + size: 32x32 + type: image/svg+xml +screenshots: + - src: /img/cyware-indicator-dashboard.png + title: Indicator Dashboard + size: 600x600 + type: image/png +policy_templates: + - name: ti_cyware_intel_exchange + title: Cyware Intel Exchange + description: Collect indicator logs from Cyware Intel Exchange. + deployment_modes: + default: + enabled: true + agentless: + enabled: true + organization: security + division: engineering + team: security-service-integrations + inputs: + - type: cel + title: Collect Cyware Intel Exchange logs via API + description: Collecting Cyware Intel Exchange logs via API. +owner: + github: elastic/security-service-integrations + type: elastic diff --git a/packages/ti_cyware_intel_exchange/validation.yml b/packages/ti_cyware_intel_exchange/validation.yml new file mode 100644 index 00000000000..1189aa63c89 --- /dev/null +++ b/packages/ti_cyware_intel_exchange/validation.yml @@ -0,0 +1,3 @@ +errors: + exclude_checks: + - SVR00004 # References in dashboards.