kernel utilization metrics on EC2 AL2023 w/ 6.1 kernel (#515)

* kernel utilization metrics on EC2 AL2023 w/ 6.1 kernel

Signed-off-by: Harper, Jason M <[email protected]>

* fix ICX metric definition

Signed-off-by: Harper, Jason M <[email protected]>

* icx metric event translation to perf

Signed-off-by: Harper, Jason M <[email protected]>

* make deterministic expression parsing to replace fixed counter event names with corresponding perf event names

Signed-off-by: Harper, Jason M <[email protected]>

* cleanup fix counter event name replacement

Signed-off-by: Harper, Jason M <[email protected]>

---------

Signed-off-by: Harper, Jason M <[email protected]>

Author: harp-intel

cmd/metrics/loader_perfmon.go

@@ -309,6 +309,9 @@ func customizeOCREventNames(metrics []MetricDefinition) []MetricDefinition {
 // example formula: "( 1000000000 * (a / b) / (c / (d * socket_count) ) ) * DURATIONTIMEINSECONDS"
 // desired output: "( 1000000000 * ([event1] / [event2]) / ([constant1] / ([constant2] * socket_count) ) ) * 1"
 func getExpression(perfmonMetric PerfmonMetric) (string, error) {
+	if perfmonMetric.Formula == "" {
+		return "", fmt.Errorf("metric '%s' has no formula defined", perfmonMetric.MetricName)
+	}
 	expression := perfmonMetric.Formula
 	replacers := make(map[string]string)
 	for _, event := range perfmonMetric.Events {
@@ -329,10 +332,18 @@ func getExpression(perfmonMetric PerfmonMetric) (string, error) {
 	for commonEvent, alias := range commonEventReplacements {
 		expression = strings.ReplaceAll(expression, commonEvent, alias)
 	}
-	// replace fixed counter perfmon event names with their corresponding perf event names
-	for perfmonEventName, perfEventName := range fixedCounterEventNameTranslation {
-		// Replace event name as whole words only (not substrings)
-		expression = util.ReplaceWholeWord(expression, perfmonEventName, perfEventName)
+	// replace fixed counter perfmon event names with their corresponding perf
+	// event names found in the fixedCounterEventNameTranslation map
+	// example: "100 * ([CPU_CLK_UNHALTED.REF_TSC:k] / [TSC])"
+	// becomes  "100 * ([ref-cycles:k] / [TSC])"
+	expressionVarPattern := regexp.MustCompile(`\[[^\]]+\]`)
+	for _, match := range expressionVarPattern.FindAllString(expression, -1) {
+		// strip the brackets
+		match = strings.Trim(match, "[]")
+		// check if the match is in the translation map
+		if perfEventName, ok := fixedCounterEventNameTranslation[match]; ok {
+			expression = strings.ReplaceAll(expression, match, perfEventName)
+		}
 	}
 	return expression, nil
 }

cmd/metrics/loader_perfmon_event_core.go

@@ -145,6 +145,7 @@ var fixedCounterEventNameTranslation = map[string]string{
 	"CPU_CLK_UNHALTED.THREAD_P:SUP":   "cpu-cycles:k",
 	"CPU_CLK_UNHALTED.CORE_P:SUP":     "cpu-cycles:k", // srf - thread and core are the same
 	"CPU_CLK_UNHALTED.REF_TSC":        "ref-cycles",
+	"CPU_CLK_UNHALTED.REF_TSC:SUP":    "ref-cycles:k",
 	"CPU_CLK_UNHALTED.REF_TSC_P:SUP":  "ref-cycles:k",
 	"TOPDOWN.SLOTS:perf_metrics":      "topdown.slots",
 	"PERF_METRICS.BAD_SPECULATION":    "topdown-bad-spec",

cmd/metrics/loader_perfmon_group_core.go

@@ -241,6 +241,17 @@ func (group *CoreGroup) AddEvent(event CoreEvent, reorder bool, metadata Metadat
 			validCounters += fmt.Sprintf("%d,", i)
 		}
 	}
+	// When the fixed ref cycles counter is not supported, we cannot put CPU_CLK_UNHALTED.REF_TSC (ref-cycles) and
+	// CPU_CLK_UNHALTED.REF_TSC_P:SUP (ref-cycles:k) in the same group.
+	// Note: this was discovered through testing on AWS m7i.8xlarge instances with Amazon Linux 2023 w/ kernel
+	// 6.1.  The same platform but with kernel 6.12 supports the fixed ref cycles counter and doesn't have this limitation.
+	if !metadata.SupportsFixedRefCycles && strings.HasPrefix(event.EventName, "CPU_CLK_UNHALTED.REF_TSC") {
+		for _, existingEvent := range group.GeneralPurposeCounters {
+			if strings.HasPrefix(existingEvent.EventName, "CPU_CLK_UNHALTED.REF_TSC") {
+				return fmt.Errorf("cannot add %s to group as it contains %s and fixed reference cycles are not supported", event.EventName, existingEvent.EventName)
+			}
+		}
+	}
 	// otherwise, it is a general purpose event, check if we can place it in one of the general purpose counters
 	for i := range group.GeneralPurposeCounters {
 		if counter := group.GeneralPurposeCounters[i]; counter.IsEmpty() {

cmd/metrics/resources/perfmon/icx/icx_perfspect_metrics.json

@@ -9,7 +9,7 @@
       "BriefDescription": "CPU utilization percentage in kernel mode",
       "Events": [
         {
-          "Name": "CPU_CLK_UNHALTED.REF_TSC_P:SUP",
+          "Name": "CPU_CLK_UNHALTED.REF_TSC:SUP",
           "Alias": "a"
         },
         {