Feature/windows event id filtering

.github/workflows/test-artifacts.yml

@@ -1319,4 +1319,4 @@ jobs:
             else
               cd terraform/eks/addon/gpu
             fi
-            terraform destroy -auto-approve
+            terraform destroy -auto-approve

cmd/config-translator/translator_test.go

@@ -103,13 +103,26 @@ func TestLogWindowsEventConfig(t *testing.T) {
 	checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidLogWindowsEventsWithInvalidEventName.json", false, expectedErrorMap)
 	expectedErrorMap1 := map[string]int{}
 	expectedErrorMap1["required"] = 2
+	expectedErrorMap1["number_any_of"] = 1
 	checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidLogWindowsEventsWithMissingEventNameAndLevel.json", false, expectedErrorMap1)
 	expectedErrorMap2 := map[string]int{}
 	expectedErrorMap2["invalid_type"] = 1
 	checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidLogWindowsEventsWithInvalidEventLevelType.json", false, expectedErrorMap2)
 	expectedErrorMap3 := map[string]int{}
 	expectedErrorMap3["enum"] = 1
 	checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidLogWindowsEventsWithInvalidEventFormatType.json", false, expectedErrorMap3)
+
+	//New tests for event_ids feature
+	checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidLogWindowsEventsWithInvalidEventFormatType.json", false, expectedErrorMap3)
+	expectedErrorMap4 := map[string]int{}
+	expectedErrorMap4["invalid_type"] = 1
+	checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidLogWindowsEventsWithInvalidEventIdsType.json", false, expectedErrorMap4)
+
+	expectedErrorMap5 := map[string]int{}
+	expectedErrorMap5["required"] = 1
+	expectedErrorMap5["number_any_of"] = 1
+	checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidLogWindowsEventsWithMissingEventIdsAndEventLevels.json", false, expectedErrorMap5)
+
 }
 
 func TestMetricsConfig(t *testing.T) {
@@ -198,7 +211,9 @@ func TestSampleConfigSchema(t *testing.T) {
 		for _, file := range files {
 			if re.MatchString(file.Name()) {
 				t.Logf("Validating ../../translator/tocwconfig/sampleConfig/%s\n", file.Name())
+
 				checkIfSchemaValidateAsExpected(t, "../../translator/tocwconfig/sampleConfig/"+file.Name(), true, map[string]int{})
+
 				t.Logf("Validated ../../translator/tocwconfig/sampleConfig/%s\n", file.Name())
 			}
 		}

plugins/inputs/windows_event_log/windows_event_log.go

@@ -31,6 +31,7 @@ var startOnlyOnce sync.Once
 type EventConfig struct {
 	Name          string   `toml:"event_name"`
 	Levels        []string `toml:"event_levels"`
+	EventIDs      []int    `toml:"event_ids"`
 	RenderFormat  string   `toml:"event_format"`
 	BatchReadSize int      `toml:"batch_read_size"`
 	LogGroupName  string   `toml:"log_group_name"`
@@ -39,7 +40,6 @@ type EventConfig struct {
 	Destination   string   `toml:"destination"`
 	Retention     int      `toml:"retention_in_days"`
 }
-
 type Plugin struct {
 	FileStateFolder string          `toml:"file_state_folder"`
 	Events          []EventConfig   `toml:"event_config"`
@@ -61,6 +61,7 @@ func (s *Plugin) SampleConfig() string {
 	[[inputs.windows_event_log.event_config]]
 	event_name = "System"
 	event_levels = ["2", "3"]
+	event_ids = [1001, 1002]
 	batch_read_size = 1
 	log_group_name = "System"
 	log_stream_name = "STREAM_NAME"
@@ -106,6 +107,7 @@ func (s *Plugin) Start(acc telegraf.Accumulator) error {
 		eventLog := wineventlog.NewEventLog(
 			eventConfig.Name,
 			eventConfig.Levels,
+			eventConfig.EventIDs,
 			eventConfig.LogGroupName,
 			eventConfig.LogStreamName,
 			eventConfig.RenderFormat,

plugins/inputs/windows_event_log/wineventlog/utils.go

@@ -1,29 +1,18 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: MIT
 
-//go:build windows
-// +build windows
-
 package wineventlog
 
 import (
-	"bytes"
 	"fmt"
-	"io"
-	"log"
-	"strconv"
-	"strings"
-	"syscall"
 	"time"
-
-	"golang.org/x/text/encoding/unicode"
-	"golang.org/x/text/transform"
 )
 
 const (
 	bookmarkTemplate         = `<BookmarkList><Bookmark Channel="%s" RecordId="%d" IsCurrent="True"/></BookmarkList>`
 	eventLogQueryTemplate    = `<QueryList><Query Id="0"><Select Path="%s">%s</Select></Query></QueryList>`
 	eventLogLevelFilter      = "Level='%s'"
+	eventLogeventIDFilter    = "EventID='%d'"
 	eventIgnoreOldFilter     = "TimeCreated[timediff(@SystemTime) &lt;= %d]"
 	emptySpaceScanLength     = 100
 	UnknownBytesPerCharacter = 0
@@ -36,34 +25,7 @@ const (
 	UNKNOWN     = "UNKNOWN"
 )
 
-var NumberOfBytesPerCharacter = UnknownBytesPerCharacter
-
-func RenderEventXML(eventHandle EvtHandle, renderBuf []byte) ([]byte, error) {
-	var bufferUsed, propertyCount uint32
-
-	if err := EvtRender(0, eventHandle, EvtRenderEventXml, uint32(len(renderBuf)), &renderBuf[0], &bufferUsed, &propertyCount); err != nil {
-		return nil, fmt.Errorf("error when rendering events. Details: %v", err)
-	}
-
-	// Per MSDN as of Mar 14th 2022(https://docs.microsoft.com/en-us/windows/win32/api/winevt/nf-winevt-evtrender)
-	// EvtRender function is still returning buffer used as BYTES, not characters. So keep using utf16ToUTF8Bytes()
-	return utf16ToUTF8Bytes(renderBuf, bufferUsed)
-}
-
-func CreateBookmark(channel string, recordID uint64) (h EvtHandle, err error) {
-	xml := fmt.Sprintf(bookmarkTemplate, channel, recordID)
-	p, err := syscall.UTF16PtrFromString(xml)
-	if err != nil {
-		return 0, err
-	}
-	h, err = EvtCreateBookmark(p)
-	if err != nil {
-		return 0, fmt.Errorf("error when creating a bookmark. Details: %v", err)
-	}
-	return h, nil
-}
-
-func CreateQuery(path string, levels []string) (*uint16, error) {
+func createFilterQuery(levels []string, eventIDs []int) string {
 	var filterLevels string
 	for _, level := range levels {
 		if filterLevels == "" {
@@ -73,149 +35,34 @@ func CreateQuery(path string, levels []string) (*uint16, error) {
 		}
 	}
 
-	//Ignore events older than 2 weeks
-	cutOffPeriod := (time.Hour * 24 * 14).Nanoseconds()
-	ignoreOlderThanTwoWeeksFilter := fmt.Sprintf(eventIgnoreOldFilter, cutOffPeriod/int64(time.Millisecond))
-	if filterLevels != "" {
-		filterLevels = "*[System[(" + filterLevels + ") and " + ignoreOlderThanTwoWeeksFilter + "]]"
-	} else {
-		filterLevels = "*[System[" + ignoreOlderThanTwoWeeksFilter + "]]"
-	}
-
-	xml := fmt.Sprintf(eventLogQueryTemplate, path, filterLevels)
-	return syscall.UTF16PtrFromString(xml)
-}
-
-func utf16ToUTF8Bytes(in []byte, length uint32) ([]byte, error) {
-
-	i := length
-
-	if length%2 != 0 {
-		i = length - 1
-	}
-
-	for ; i-2 > 0; i -= 2 {
-		v1 := uint16(in[i-2]) | uint16(in[i-1])<<8
-		// Stop at non-null char.
-		if v1 != 0 {
-			break
-		}
-	}
-
-	win16be := unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM)
-	utf16bom := unicode.BOMOverride(win16be.NewDecoder())
-	unicodeReader := transform.NewReader(bytes.NewReader(in[:i]), utf16bom)
-	decoded, err := io.ReadAll(unicodeReader)
-	return decoded, err
-}
-
-func UTF16ToUTF8BytesForWindowsEventBuffer(in []byte, length uint32) ([]byte, error) {
-	// Since Windows server 2022, the returned value of used buffer represents for double bytes char count,
-	// which is half of the actual buffer used by byte(what older Windows OS returns), checking if the length
-	//land on the end of used buffer, if no, double it.
-	if NumberOfBytesPerCharacter == UnknownBytesPerCharacter {
-		if isTheEndOfContent(in, length) {
-			log.Printf("I! Buffer used: %d is returning as single byte character count", length)
-			NumberOfBytesPerCharacter = 1
+	//EventID filtering
+	var filterEventIDs string
+	for i, eventID := range eventIDs {
+		if i == 0 {
+			filterEventIDs = fmt.Sprintf(eventLogeventIDFilter, eventID)
 		} else {
-			log.Printf("I! Buffer used: %d is returning as double byte character count, doubling it to get the whole buffer content.", length)
-			NumberOfBytesPerCharacter = 2
-		}
-	}
-
-	i := int(length) * NumberOfBytesPerCharacter
-
-	if i > cap(in) {
-		i = cap(in)
-	}
-
-	return utf16ToUTF8Bytes(in, uint32(i))
-}
-
-func isTheEndOfContent(in []byte, length uint32) bool {
-	// scan next (emptySpaceScanLength) bytes, if any of them is none '0', return false
-	i := int(length)
-
-	if i%2 != 0 {
-		i -= 1
-	}
-	max := len(in)
-	if i+emptySpaceScanLength < max {
-		max = i + emptySpaceScanLength
-	}
-
-	for ; i < max-2; i += 2 {
-		v1 := uint16(in[i+2]) | uint16(in[i+1])<<8
-		// Stop at non-null char.
-		if v1 != 0 {
-			return false
+			filterEventIDs = filterEventIDs + " or " + fmt.Sprintf(eventLogeventIDFilter, eventID)
 		}
 	}
-	return true
-}
 
-func WindowsEventLogLevelName(levelId int32) string {
-	switch levelId {
-	case 1:
-		return CRITICAL
-	case 2:
-		return ERROR
-	case 3:
-		return WARNING
-	case 0, 4:
-		return INFORMATION
-	case 5:
-		return VERBOSE
-	default:
-		return UNKNOWN
+	//query results
+	var query string
+	if filterLevels != "" && filterEventIDs != "" {
+		query = "(" + filterLevels + ") and (" + filterEventIDs + ")"
+	} else if filterLevels != "" && filterEventIDs == "" {
+		query = "(" + filterLevels + ")"
+	} else if filterLevels == "" && filterEventIDs != "" {
+		query = "(" + filterEventIDs + ")"
 	}
-}
-
-// insertPlaceholderValues formats the message with the correct values if we see those data
-// in evtDataValues.
-//
-// In some cases wevtapi does not insert values when formatting the message. The message
-// will contain insertion string placeholders, of the form %n, where %1 indicates the first
-// insertion string, and so on. Noted that wevtapi start the index with 1.
-// https://learn.microsoft.com/en-us/windows/win32/eventlog/event-identifiers#insertion-strings
-func insertPlaceholderValues(rawMessage string, evtDataValues []Datum) string {
-	if len(evtDataValues) == 0 || len(rawMessage) == 0 {
-		return rawMessage
-	}
-	var sb strings.Builder
-	prevIndex := 0
-	searchingIndex := false
-	for i, c := range rawMessage {
-		// found `%` previously. Determine the index number from the following character(s)
-		if searchingIndex && (c > '9' || c < '0') {
-			// Convert the Slice since the last `%` and see if it's a valid number.
-			ind, err := strconv.Atoi(rawMessage[prevIndex+1 : i])
-			// If the index is in [1 - len(evtDataValues)], get it from evtDataValues.
-			if err == nil && ind <= len(evtDataValues) && ind > 0 {
-				sb.WriteString(evtDataValues[ind-1].Value)
-			} else {
-				sb.WriteString(rawMessage[prevIndex:i])
-			}
-			prevIndex = i
-			// In case of consecutive `%`, continue searching for the next index
-			if c != '%' {
-				searchingIndex = false
-			}
-		} else {
-			if c == '%' {
-				sb.WriteString(rawMessage[prevIndex:i])
-				searchingIndex = true
-				prevIndex = i
-			}
 
-		}
-	}
-	// handle the slice since the last `%` to the end of rawMessage
-	ind, err := strconv.Atoi(rawMessage[prevIndex+1:])
-	if searchingIndex && err == nil && ind <= len(evtDataValues) && ind > 0 {
-		sb.WriteString(evtDataValues[ind-1].Value)
+	//Ignore events older than 2 weeks
+	cutOffPeriod := (time.Hour * 24 * 14).Nanoseconds()
+	ignoreOlderThanTwoWeeksFilter := fmt.Sprintf(eventIgnoreOldFilter, cutOffPeriod/int64(time.Millisecond))
+	if query != "" {
+		query = "*[System[" + query + " and " + ignoreOlderThanTwoWeeksFilter + "]]"
 	} else {
-		sb.WriteString(rawMessage[prevIndex:])
+		query = "*[System[" + ignoreOlderThanTwoWeeksFilter + "]]"
 	}
-	return sb.String()
+
+	return query
 }