System auth log fixes

packages/system/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.4.1"
+  changes:
+    - description: Fixed missing some fields in system auth messages
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/14456
 - version: "2.4.0"
   changes:
     - description: Use Links panel in Dashboards

packages/system/data_stream/auth/_dev/test/pipeline/test-auth-ubuntu1204.log-expected.json

@@ -2569,17 +2569,31 @@
             "input": {
                 "type": "log"
             },
-            "message": "pam_unix(sudo:auth): authentication failure; logname=vagrant uid=1001 euid=0 tty=/dev/pts/1 ruser=tsg rhost=  user=tsg",
             "process": {
                 "name": "sudo"
             },
             "related": {
                 "hosts": [
                     "precise32"
+                ],
+                "user": [
+                    "tsg"
                 ]
             },
+            "source": {
+                "address": "",
+                "domain": ""
+            },
             "system": {
                 "auth": {}
+            },
+            "user": {
+                "effective": {
+                    "id": "0",
+                    "name": "tsg"
+                },
+                "id": "1001",
+                "name": "tsg"
             }
         },
         {

packages/system/data_stream/auth/_dev/test/pipeline/test-auth.log

@@ -9,3 +9,4 @@ Feb 24 00:13:02 precise32 sudo:      tsg : user NOT in sudoers ; TTY=pts/1 ; PWD
 Feb 22 11:47:05 localhost groupadd[6991]: new group: name=apache, GID=48
 Feb 22 11:47:05 localhost useradd[6995]: new user: name=apache, UID=48, GID=48, home=/usr/share/httpd, shell=/sbin/nologin
 Feb 22 12:53:50 localhost-machine sshd[10161]: error: PAM: User not known to the underlying authentication module for illegal user test from test.example.com
+Jun  2 19:08:43 host.name sudo[654321]: pam_unix(sudo:auth): authentication failure; logname=bob uid=1117 euid=0 tty=/dev/pts/1 ruser=alice rhost= user=root

packages/system/data_stream/auth/_dev/test/pipeline/test-auth.log-expected.json

@@ -569,6 +569,53 @@
             "user": {
                 "name": "test"
             }
+        },
+        {
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "category": [
+                    "authentication"
+                ],
+                "outcome": "failure",
+                "kind": "event",
+                "timezone": "+0000"
+            },
+            "host": {
+                "hostname": "host.name"
+            },
+            "input": {
+                "type": "log"
+            },
+            "process": {
+                "name": "sudo",
+                "pid": 654321
+            },
+            "related": {
+                "hosts": [
+                    "host.name"
+                ],
+                "user": [
+                    "alice",
+                    "root"
+                ]
+            },
+            "source": {
+                "address": "",
+                "domain": ""
+            },
+            "system": {
+                "auth": {}
+            },
+            "user": {
+                "effective": {
+                    "id": "0",
+                    "name": "root"
+                },
+                "name": "alice",
+                "id": "1117"
+            }
         }
     ]
 }

packages/system/data_stream/auth/_dev/test/pipeline/test-secure-rhel7.log-expected.json

@@ -218,16 +218,51 @@
             "input": {
                 "type": "log"
             },
-            "message": "pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=89.160.20.156  user=root",
             "process": {
                 "name": "sshd",
                 "pid": 2742
             },
             "related": {
                 "hosts": [
                     "slave22"
+                ],
+                "ip": [
+                    "89.160.20.156"
+                ],
+                "user": [
+                    "root"
                 ]
             },
+            "source": {
+                "address": "89.160.20.156",
+                "as": {
+                    "number": 29518,
+                    "organization": {
+                        "name": "Bredband2 AB"
+                    }
+                },
+                "geo": {
+                    "city_name": "Linköping",
+                    "continent_name": "Europe",
+                    "country_iso_code": "SE",
+                    "country_name": "Sweden",
+                    "location": {
+                        "lat": 58.4167,
+                        "lon": 15.6167
+                    },
+                    "region_iso_code": "SE-E",
+                    "region_name": "Östergötland County"
+                },
+                "ip": "89.160.20.156"
+            },
+            "user": {
+                "effective": {
+                    "id": "0",
+                    "name": "root"
+                },
+                "id": "0",
+                "name": ""
+            },
             "system": {
                 "auth": {}
             }

packages/system/data_stream/auth/elasticsearch/ingest_pipeline/log.yml

@@ -34,11 +34,63 @@ processors:
         - '^%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}'
         - '^new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id}'
         - '^new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id}, GID=%{NUMBER:group.id}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$'
+        - '^%{WORD:pam.module}\(%{WORD:pam.service}:%{WORD:pam.facility}\):%{SPACE}%{DATA:pam.message};%{SPACE}%{DATA:_temp.pam_kv}$'
       on_failure:
         - rename:
             description: Leave the unmatched content in message.
             field: _temp.message
             target_field: message
+  - gsub:
+      ignore_missing: true
+      ignore_failure: true
+      field: _temp.pam_kv
+      tag: pam_kv_preparser
+      pattern: "\\s+"
+      replacement: ' '
+  - kv:
+      tag: pam_kv_parser
+      field: _temp.pam_kv
+      ignore_missing: true
+      field_split: ' '
+      value_split: '='
+      trim_key: " "
+      trim_value: " "
+      prefix: pam.data.
+      strip_brackets: true
+      on_failure:
+        - append:
+            field: error.message
+            value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}"
+  - rename:
+      tag: rename_user
+      field: pam.data.ruser
+      target_field: user.name
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      tag: rename_ruser
+      field: pam.data.user
+      target_field: user.effective.name
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      tag: rename_euid
+      field: pam.data.euid
+      target_field: user.effective.id
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      tag: rename_uid
+      field: pam.data.uid
+      target_field: user.id
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      tag: rename_rhost
+      field: pam.data.rhost
+      target_field: source.address
+      ignore_missing: true
+      ignore_failure: true
   - append:
       tag: append_category_process
       field: event.category
@@ -60,6 +112,8 @@ processors:
         - 'for user %{QUOTE}?%{DATA:_temp.foruser}%{QUOTE}?$'
         - 'by user %{QUOTE}?%{DATA:_temp.byuser}%{QUOTE}?$'
         - '%{BOUNDARY} user %{QUOTE}%{DATA:_temp.user}%{QUOTE}'
+        - '%{BOUNDARY}user=%{QUOTE}%{DATA:user.effective.name}%{QUOTE}'
+        - '%{BOUNDARY}ruser=%{QUOTE}%{DATA:user.name}%{QUOTE}'
       pattern_definitions:
         QUOTE: "['\"]"
         BOUNDARY: "(?<! )"
@@ -130,7 +184,7 @@ processors:
       field: event.category
       value: authentication
       allow_duplicates: false
-      if: ctx._temp?.category == 'auth'
+      if: ctx._temp?.category == 'auth' || ctx.pam?.facility == 'auth'
   - set:
       tag: set_outcome_success
       field: event.outcome
@@ -140,7 +194,7 @@ processors:
       tag: set_outcome_failure
       field: event.outcome
       value: failure
-      if: ctx.message != null && ctx.message.contains('authentication failure')
+      if: (ctx.message != null && ctx.message.contains('authentication failure')) || (ctx.pam?.message != null && ctx.pam?.message.contains('authentication failure'))
   - append:
       tag: append_category_authentication_2
       field: event.category
@@ -194,6 +248,10 @@ processors:
       tag: remove_temp_2
       field: _temp
       ignore_missing: true
+  - remove:
+      tag: remove_pam
+      field: pam
+      ignore_missing: true
   - convert:
       tag: convert_source-address
       field: source.address
@@ -227,6 +285,7 @@ processors:
       field: system.auth.timestamp
       target_field: '@timestamp'
       formats:
+        - MMM d HH:mm:ss
         - MMM  d HH:mm:ss
         - MMM dd HH:mm:ss
         - ISO8601

packages/system/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.0.2
 name: system
 title: System
-version: "2.4.0"
+version: "2.4.1"
 description: Collect system logs and metrics from your servers with Elastic Agent.
 type: integration
 categories: