New sandbox decoders for Microsft IIS and ULS log formats

CHANGES.txt

@@ -6,6 +6,8 @@ Backwards Incompatibilities
 Features
 --------
 
+Added Sandbox decoders for Microsoft iis and sharepoint uls logs 
+
 Bug Handling
 ------------
 

docs/source/config/decoders/iis.rst

@@ -0,0 +1,13 @@
+.. _config_iis_log_decoder:
+
+Microsoft iis log Decoder
+========================
+
+.. versionadded:: 0.1
+
+| Plugin Name: **SandboxDecoder**
+| File Name: **lua_decoders/iis.lua**
+
+.. include:: /../../sandbox/lua/decoders/iis.lua
+   :start-after: --[[
+   :end-before: --]]

docs/source/config/decoders/index.rst

@@ -13,6 +13,7 @@ Available Decoder Plugins
    apache_access
    geoip
    graylog_extended
+   iis
    linux_cpu_stats
    linux_disk_stats
    linux_load_avg
@@ -27,4 +28,5 @@ Available Decoder Plugins
    rsyslog
    sandbox
    scribble
+   sharepoint_iis
    stats_to_fields

docs/source/config/decoders/index_noref.rst

@@ -56,3 +56,9 @@ Decoders
 
 .. include:: /config/decoders/stats_to_fields.rst
    :start-line: 1
+
+.. include:: /config/decoders/iis.rst
+   :start-line: 1
+
+.. include:: /config/decoders/sharepoint_uls.rst
+   :start-line: 1

docs/source/config/decoders/sharepoint_uls.rst

@@ -0,0 +1,13 @@
+.. _config_sharepoint_uls_log_decoder:
+
+Microsoft sharepoint uls log Decoder
+========================
+
+.. versionadded:: 0.1
+
+| Plugin Name: **SandboxDecoder**
+| File Name: **lua_decoders/sharepoint_uls.lua**
+
+.. include:: /../../sandbox/lua/decoders/sharepoint_uls.lua
+   :start-after: --[[
+   :end-before: --]]

sandbox/lua/decoders/iis.lua

@@ -0,0 +1,151 @@
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this
+-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+--[[
+Parses the iis logs based on the Microsoft iis log formats. This decoder is tested for iis verions 7 and 8. 
+
+Config:
+
+- payload_keep (bool, optional, default false)
+    Always preserve the original log line in the message payload.
+
+- iis_version_7 (bool, optional, default flase)
+     Default configuration asssumes iis log format for version 8. 
+     For version 7 and similar formats, set this to true
+
+*Example Heka Configuration*
+
+.. code-block:: ini
+
+[hekad]
+share_dir = 'C:\heka-agent\heka\share\heka'
+base_dir = 'C:\var\cache\hekad'
+
+[IISLogs]
+type = "LogstreamerInput"
+log_directory = 'F:\Web_Logs'
+file_match = '(?P<dir>\w+)(?P<s>\S+)u_ex(?P<Index>\d+)\.log'
+differentiator = ["dir"]
+priority = ["Index"]
+decoder = "IISDecoder"
+
+[IISDecoder]
+type = "SandboxDecoder"
+script_type = "lua"
+filename = 'lua_decoders\iis.lua'
+
+[IISDecoder.config]
+payload_keep = true
+iis_version_7 = true
+tz = "UTC"
+
+*Example Heka Message*
+
+2015/08/02 00:34:43
+:Timestamp: 2014-09-22 06:32:29 +0000 UTC
+:Type: iis
+:Hostname: iis-host
+:Pid: 0
+:Uuid: 2dd1d363-02e2-4d61-ade8-e4ed6657fcd6
+:Logger: W3SVC1368505715
+:Payload: 2014-09-22 06:32:29 101.181.48.45 GET / - 6005 - 10.181.72.190 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) 401 0 64 46
+
+:EnvVersion:
+:Severity: 7
+:Fields:
+    | name:"substatus" type:string value:"0"
+    | name:"client_ip" type:string value:"101.181.72.190"
+    | name:"cs_method" type:string value:"GET"
+    | name:"cs_username" type:string value:""
+    | name:"cs_user_agent" type:string value:"Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0)"
+    | name:"time_taken" type:double value:46
+    | name:"port" type:string value:"6005"
+    | name:"win32_status" type:string value:"64"
+    | name:"status" type:double value:401
+    | name:"cs_uri_stem" type:string value:"/"
+    | name:"host_ip" type:string value:"10.181.48.45"
+    | name:"cs_uri_query" type:string value:""
+
+--]]
+
+local dt = require "date_time"
+local l = require 'lpeg'
+l.locale(l)
+
+local sp = l.space
+local num = l.digit^1 / tonumber
+
+local function extract_quote(openp,endp)
+    openp = l.P(openp)
+    endp = endp and l.P(endp) or openp
+    local upto_endp = (1 - endp)^1 
+    return openp * l.C(upto_endp) * endp
+end
+
+local sp = l.space
+
+local timestamp = l.Cg(dt.build_strftime_grammar("%Y-%m-%d %H:%M:%S") / dt.time_to_ns, "timestamp")
+local host_ip = l.Cg(extract_quote(" ", " "), "host_ip")
+local cs_method = l.Cg(extract_quote("", " "), "cs_method")
+local cs_uri_stem = l.Cg(extract_quote("", " "), "cs_uri_stem")
+local cs_uri_query = l.Cg(extract_quote("", " "), "cs_uri_query")
+local port = l.Cg(extract_quote("", " "), "port")
+local cs_username = l.Cg(extract_quote("", " "), "cs_username") 
+local client_ip = l.Cg(extract_quote("", " "), "client_ip") 
+local cs_user_agent = l.Cg(extract_quote("", " "), "cs_user_agent") 
+local cs_referer = l.Cg(extract_quote("", " "), "cs_referer") 
+local status = l.Cg(num, "status") 
+local substatus = l.Cg(extract_quote(" ", " "), "substatus")
+local win32_status = l.Cg(extract_quote("", " "), "win32_status")
+local time_taken = l.Cg(num, "time_taken")
+local version_8 = timestamp * host_ip * cs_method * cs_uri_stem * cs_uri_query * port * cs_username * client_ip * cs_user_agent * cs_referer * status * substatus * win32_status * time_taken
+local version_7 = timestamp * host_ip * cs_method * cs_uri_stem * cs_uri_query * port * cs_username * client_ip * cs_user_agent * status * substatus * win32_status * time_taken 
+
+local grammar = l.Ct(version_8)
+
+local iis_version = read_config("iis_version_7")
+local payload_keep = read_config("payload_keep")
+
+if iis_version then
+  grammar = l.Ct(version_7)
+end
+
+local msg = {
+    Timestamp = nil,
+    Payload = nil,
+    Hostname = nil,
+    Fields = nil,
+    Type = "iis"
+}
+
+function process_message()
+
+    local data = read_message("Payload")
+    local host = read_message("Hostname")
+    local fields = grammar:match(data)
+
+    if not fields then 
+      return -1
+    end
+
+    msg.Timestamp = fields.timestamp
+    msg.Hostname = string.lower(host)
+    fields.timestamp = nil
+    msg.Fields = fields
+
+    if msg.Fields.cs_username == "-" then
+	msg.Fields.cs_username = ""
+    end
+
+    if msg.Fields.cs_uri_query == "-" then
+	msg.Fields.cs_uri_query = ""
+    end
+
+    if payload_keep then
+        msg.Payload = data
+    end
+
+    inject_message(msg)
+    return 0
+end

sandbox/lua/decoders/sharepoint_uls.lua

@@ -0,0 +1,124 @@
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this
+-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+--[[
+Parses the Microsoft sharepoint uls logs based on the uls log format.
+
+Config:
+
+- payload_keep (bool, optional, default false)
+    Always preserve the original log line in the message payload.
+
+*Example Heka Configuration*
+
+.. code-block:: ini
+
+[hekad]
+share_dir = 'C:\heka-agent\heka\\share\heka'
+base_dir = 'C:\var\cache\hekad'
+
+[SharePointULSLogs]
+type = "LogstreamerInput"
+log_directory = 'F:\Trace_log'
+file_match = '(?P<first>\w+)-(?P<second>\S+)-(?P<Year>\d{4})(?P<Month>\d{2})(?P<Day>\d{2})-(?P<time>\d+).log'
+priority = ["Year","Month","Day","time"]
+decoder = "SharePointDecoder"
+
+[SharePointDecoder]
+type = "SandboxDecoder"
+script_type = "lua"
+filename = 'lua_decoders\sharepoint_uls.lua'
+
+[SharePointDecoder.config]
+payload_keep = true
+tz = "Local"
+
+*Example Heka Message*
+
+2015/08/02 01:05:58
+:Timestamp: 2015-04-21 00:04:46 +0000 UTC
+:Type: uls
+:Hostname: sharepoint-host
+:Pid: 0
+:Uuid: fb4f1f82-7c8b-4cdc-aa10-6e7f76cb2a0e
+:Logger: SharePointULSLogs
+:Payload: 04/20/2015 20:04:46.02        OWSTIMER.EXE (0x5A0C)                           0x7D8C  SharePoint Foundation           Monitoring
+                b4ly    Medium          Leaving Monitored Scope (Timer Job Search Health Monitoring - Trace Events). Execution Time=328.687724277806
+76b7fe9c-03c0-40dc-2ea6-6162b7e29775
+
+:EnvVersion:
+:Severity: 7
+:Fields:
+    | name:"Correlation" type:string value:"76b7fe9c-03c0-40dc-2ea6-6162b7e29775
+"
+    | name:"TID" type:string value:"0x7D8C"
+    | name:"EventID" type:string value:"b4ly"
+    | name:"Message" type:string value:"Leaving Monitored Scope (Timer Job Search Health Monitoring - Trace Events). Execution Time=328.687724277806"
+    | name:"Category" type:string value:"Monitoring                    "
+    | name:"Level" type:string value:"Medium  "
+    | name:"Process" type:string value:"OWSTIMER.EXE (0x5A0C)                   "
+    | name:"Area" type:string value:"SharePoint Foundation         "
+
+--]]
+
+local dt = require "date_time"
+local l = require 'lpeg'
+l.locale(l)
+
+local sp = l.space
+local num = l.digit^1 / tonumber
+
+local function extract_quote(openp,endp)
+    openp = l.P(openp)
+    endp = endp and l.P(endp) or openp
+    local upto_endp = (1 - endp)^0 
+    return openp * l.C(upto_endp) * endp
+end
+
+local datetime = dt.build_strftime_grammar("%m/%d/%Y %H:%M:%S") / dt.time_to_ns * "." * l.R("09","*/")^0
+local process = l.Cg(extract_quote(sp^1, "\t"), "Process")
+local t_id = l.Cg(extract_quote("", "\t"), "TID")
+local area = l.Cg(extract_quote("", "\t"), "Area")
+local category = l.Cg(extract_quote("", "\t"), "Category")
+local event_id = l.Cg(extract_quote("", "\t"), "EventID")
+local level = l.Cg(extract_quote("", "\t"), "Level")
+local message = l.Cg(extract_quote("", "\t"), "Message")
+local correlation = l.Cg(l.P(1)^0, "Correlation")
+
+local request = l.Cg(datetime,"DateTime") * process * t_id * area * category * event_id * level * message * correlation
+
+grammar = l.Ct(request)
+
+local payload_keep = read_config("payload_keep")
+
+local msg = {
+    Timestamp = nil,
+    Payload = nil,
+    Hostname = nil,
+    Fields = nil,
+    Type = "uls"
+}
+
+function process_message()
+
+    local data = read_message("Payload")
+    local host = read_message("Hostname")
+    local fields = grammar:match(data)
+
+    if not fields then
+	return -1
+    end
+
+    msg.Timestamp = fields.DateTime
+    msg.Hostname = string.lower(host)
+    fields.DateTime = nil
+    msg.Fields = fields
+
+    if payload_keep then
+        msg.Payload = data
+    end
+
+    inject_message(msg)
+    return 0
+end