Merge pull request #179 from intel/processwatch

add instruction mix reporting to telemetry command

Author: harp-intel

cmd/flame/flame.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"perfspect/internal/common"
 	"perfspect/internal/report"
+	"perfspect/internal/script"
 	"perfspect/internal/util"
 	"strings"
 
@@ -138,8 +139,7 @@ func runCmd(cmd *cobra.Command, args []string) error {
 	reportingCommand := common.ReportingCommand{
 		Cmd:            cmd,
 		ReportNamePost: "flame",
-		Frequency:      flagFrequency,
-		Duration:       flagDuration,
+		ScriptParams:   script.ScriptParams{Frequency: flagFrequency, Duration: flagDuration},
 		TableNames:     []string{report.CodePathFrequencyTableName},
 	}
 	return reportingCommand.Run()

cmd/lock/lock.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"perfspect/internal/common"
 	"perfspect/internal/report"
+	"perfspect/internal/script"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -115,8 +116,7 @@ func runCmd(cmd *cobra.Command, args []string) error {
 	reportingCommand := common.ReportingCommand{
 		Cmd:            cmd,
 		ReportNamePost: "lock",
-		Frequency:      flagFrequency,
-		Duration:       flagDuration,
+		ScriptParams:   script.ScriptParams{Frequency: flagFrequency, Duration: flagDuration},
 		TableNames:     []string{report.KernelLockAnalysisTableName},
 	}
 	return reportingCommand.Run()

cmd/report/report.go

@@ -333,7 +333,7 @@ func runCmd(cmd *cobra.Command, args []string) error {
 	}
 	reportingCommand := common.ReportingCommand{
 		Cmd:              cmd,
-		StorageDir:       flagStorageDir,
+		ScriptParams:     script.ScriptParams{StorageDir: flagStorageDir},
 		TableNames:       tableNames,
 		SummaryFunc:      summaryFunc,
 		SummaryTableName: benchmarkSummaryTableName,

cmd/telemetry/telemetry.go

@@ -50,13 +50,17 @@ var (
 
 	flagAll bool
 
-	flagCpu     bool
-	flagCpuAvg  bool
-	flagIrq     bool
-	flagNetwork bool
-	flagStorage bool
-	flagMemory  bool
-	flagPower   bool
+	flagCpu      bool
+	flagCpuAvg   bool
+	flagIrq      bool
+	flagNetwork  bool
+	flagStorage  bool
+	flagMemory   bool
+	flagPower    bool
+	flagInstrMix bool
+
+	flagInstrMixPid    int
+	flagInstrMixFilter []string
 )
 
 const (
@@ -65,20 +69,25 @@ const (
 
 	flagAllName = "all"
 
-	flagCpuName     = "cpu"
-	flagCpuAvgName  = "cpuavg"
-	flagIrqName     = "irq"
-	flagNetworkName = "network"
-	flagStorageName = "storage"
-	flagMemoryName  = "memory"
-	flagPowerName   = "power"
+	flagCpuName      = "cpu"
+	flagCpuAvgName   = "cpuavg"
+	flagIrqName      = "irq"
+	flagNetworkName  = "network"
+	flagStorageName  = "storage"
+	flagMemoryName   = "memory"
+	flagPowerName    = "power"
+	flagInstrMixName = "instrmix"
+
+	flagInstrMixPidName    = "instrmix-pid"
+	flagInstrMixFilterName = "instrmix-filter"
 )
 
 var telemetrySummaryTableName = "Telemetry Summary"
 
 var categories = []common.Category{
 	{FlagName: flagCpuName, FlagVar: &flagCpu, DefaultValue: false, Help: "monitor cpu", TableNames: []string{report.CPUUtilizationTableName}},
 	{FlagName: flagCpuAvgName, FlagVar: &flagCpuAvg, DefaultValue: false, Help: "monitor cpu average", TableNames: []string{report.AverageCPUUtilizationTableName}},
+	{FlagName: flagInstrMixName, FlagVar: &flagInstrMix, DefaultValue: false, Help: "monitor instruction mix", TableNames: []string{report.InstructionMixTableName}},
 	{FlagName: flagIrqName, FlagVar: &flagIrq, DefaultValue: false, Help: "monitor irq", TableNames: []string{report.IRQRateTableName}},
 	{FlagName: flagStorageName, FlagVar: &flagStorage, DefaultValue: false, Help: "monitor storage", TableNames: []string{report.DriveStatsTableName}},
 	{FlagName: flagNetworkName, FlagVar: &flagNetwork, DefaultValue: false, Help: "monitor network", TableNames: []string{report.NetworkStatsTableName}},
@@ -96,6 +105,8 @@ func init() {
 	Cmd.Flags().StringSliceVar(&common.FlagFormat, common.FlagFormatName, []string{report.FormatAll}, "")
 	Cmd.Flags().IntVar(&flagDuration, flagDurationName, 30, "")
 	Cmd.Flags().IntVar(&flagInterval, flagIntervalName, 2, "")
+	Cmd.Flags().IntVar(&flagInstrMixPid, flagInstrMixPidName, 0, "")
+	Cmd.Flags().StringSliceVar(&flagInstrMixFilter, flagInstrMixFilterName, []string{"SSE", "AVX", "AVX2", "AVX512", "AMX_TILE"}, "")
 
 	common.AddTargetFlags(Cmd)
 
@@ -158,6 +169,14 @@ func getFlagGroups() []common.FlagGroup {
 			Name: flagIntervalName,
 			Help: "number of seconds between each sample",
 		},
+		{
+			Name: flagInstrMixPidName,
+			Help: "pid to monitor for instruction mix, no pid means all processes",
+		},
+		{
+			Name: flagInstrMixFilterName,
+			Help: "filter to apply to instruction mix",
+		},
 	}
 	groups = append(groups, common.FlagGroup{
 		GroupName: "Others Options",
@@ -229,8 +248,7 @@ func runCmd(cmd *cobra.Command, args []string) error {
 	reportingCommand := common.ReportingCommand{
 		Cmd:              cmd,
 		ReportNamePost:   "telem",
-		Interval:         flagInterval,
-		Duration:         flagDuration,
+		ScriptParams:     script.ScriptParams{Interval: flagInterval, Duration: flagDuration, PID: flagInstrMixPid, Filter: flagInstrMixFilter},
 		TableNames:       tableNames,
 		SummaryFunc:      summaryFunc,
 		SummaryTableName: telemetrySummaryTableName,

internal/common/common.go

@@ -80,10 +80,7 @@ type ReportingCommand struct {
 	Cmd              *cobra.Command
 	ReportNamePost   string
 	TableNames       []string
-	Duration         int
-	Interval         int
-	Frequency        int
-	StorageDir       string
+	ScriptParams     script.ScriptParams
 	SummaryFunc      SummaryFunc
 	SummaryTableName string
 	InsightsFunc     InsightsFunc
@@ -138,7 +135,7 @@ func (rc *ReportingCommand) Run() error {
 		// make a list of unique script definitions
 		var scriptsToRun []script.ScriptDefinition
 		for _, scriptName := range scriptNames {
-			scriptsToRun = append(scriptsToRun, script.GetParameterizedScriptByName(scriptName, rc.Duration, rc.Interval, rc.Frequency, rc.StorageDir))
+			scriptsToRun = append(scriptsToRun, script.GetParameterizedScriptByName(scriptName, rc.ScriptParams))
 		}
 		// do any of the scripts require elevated privileges?
 		elevated := false
@@ -188,7 +185,7 @@ func (rc *ReportingCommand) Run() error {
 		channelTargetScriptOutputs := make(chan TargetScriptOutputs)
 		channelError := make(chan error)
 		for _, target := range myTargets {
-			go collectOnTarget(rc.Cmd, rc.Duration, target, scriptsToRun, localTempDir, channelTargetScriptOutputs, channelError, multiSpinner.Status)
+			go collectOnTarget(rc.Cmd, rc.ScriptParams.Duration, target, scriptsToRun, localTempDir, channelTargetScriptOutputs, channelError, multiSpinner.Status)
 		}
 		// wait for scripts to run on all targets
 		var allTargetScriptOutputs []TargetScriptOutputs

internal/report/html.go

@@ -743,7 +743,7 @@ func cpuUtilizationTableHTMLRenderer(tableValues TableValues, targetName string)
 	}
 	chartConfig := scatterChartTemplateStruct{
 		ID:            fmt.Sprintf("cpuUtilization%d", rand.Intn(10000)),
-		XaxisText:     "Time/Samples",
+		XaxisText:     "Time",
 		YaxisText:     "% Utilization",
 		TitleText:     "",
 		DisplayTitle:  "false",
@@ -778,7 +778,7 @@ func averageCPUUtilizationTableHTMLRenderer(tableValues TableValues, targetName
 	}
 	chartConfig := scatterChartTemplateStruct{
 		ID:            fmt.Sprintf("avgCPUUtil%d", rand.Intn(10000)),
-		XaxisText:     "Time/Samples",
+		XaxisText:     "Time",
 		YaxisText:     "% Utilization",
 		TitleText:     "",
 		DisplayTitle:  "false",
@@ -819,7 +819,7 @@ func irqRateTableHTMLRenderer(tableValues TableValues, targetName string) string
 	}
 	chartConfig := scatterChartTemplateStruct{
 		ID:            fmt.Sprintf("irqRate%d", rand.Intn(10000)),
-		XaxisText:     "Time/Samples",
+		XaxisText:     "Time",
 		YaxisText:     "IRQ/s",
 		TitleText:     "",
 		DisplayTitle:  "false",
@@ -875,7 +875,7 @@ func driveStatsTableHTMLRenderer(tableValues TableValues, targetName string) str
 		}
 		chartConfig := scatterChartTemplateStruct{
 			ID:            fmt.Sprintf("driveStats%d", rand.Intn(10000)),
-			XaxisText:     "Time/Samples",
+			XaxisText:     "Time",
 			YaxisText:     "",
 			TitleText:     drive,
 			DisplayTitle:  "true",
@@ -933,7 +933,7 @@ func networkStatsTableHTMLRenderer(tableValues TableValues, targetName string) s
 		}
 		chartConfig := scatterChartTemplateStruct{
 			ID:            fmt.Sprintf("nicStats%d", rand.Intn(10000)),
-			XaxisText:     "Time/Samples",
+			XaxisText:     "Time",
 			YaxisText:     "",
 			TitleText:     nic,
 			DisplayTitle:  "true",
@@ -970,7 +970,7 @@ func memoryStatsTableHTMLRenderer(tableValues TableValues, targetName string) st
 	}
 	chartConfig := scatterChartTemplateStruct{
 		ID:            fmt.Sprintf("memoryStats%d", rand.Intn(10000)),
-		XaxisText:     "Time/Samples",
+		XaxisText:     "Time",
 		YaxisText:     "kilobytes",
 		TitleText:     "",
 		DisplayTitle:  "false",
@@ -1005,7 +1005,7 @@ func powerStatsTableHTMLRenderer(tableValues TableValues, targetName string) str
 	}
 	chartConfig := scatterChartTemplateStruct{
 		ID:            fmt.Sprintf("powerStats%d", rand.Intn(10000)),
-		XaxisText:     "Time/Samples",
+		XaxisText:     "Time",
 		YaxisText:     "Watts",
 		TitleText:     "",
 		DisplayTitle:  "false",
@@ -1061,3 +1061,38 @@ func kernelLockAnalysisHTMLRenderer(tableValues TableValues, targetName string)
 	}
 	return renderHTMLTable([]string{}, values, "pure-table pure-table-striped", tableValueStyles)
 }
+
+func instructionMixTableHTMLRenderer(tableValues TableValues, targetname string) string {
+	data := [][]scatterPoint{}
+	datasetNames := []string{}
+	for _, field := range tableValues.Fields[1:] {
+		points := []scatterPoint{}
+		for i, val := range field.Values {
+			if val == "" {
+				break
+			}
+			stat, err := strconv.ParseFloat(val, 64)
+			if err != nil {
+				slog.Error("error parsing stat", slog.String("error", err.Error()))
+				return ""
+			}
+			points = append(points, scatterPoint{float64(i), stat})
+		}
+		if len(points) > 0 {
+			data = append(data, points)
+			datasetNames = append(datasetNames, field.Name)
+		}
+	}
+	chartConfig := scatterChartTemplateStruct{
+		ID:            fmt.Sprintf("instrMix%d", rand.Intn(10000)),
+		XaxisText:     "Time",
+		YaxisText:     "% Samples",
+		TitleText:     "",
+		DisplayTitle:  "false",
+		DisplayLegend: "true",
+		AspectRatio:   "2",
+		SuggestedMin:  "0",
+		SuggestedMax:  "0",
+	}
+	return renderScatterChart(data, datasetNames, chartConfig)
+}

internal/report/table_defs.go

@@ -6,12 +6,14 @@ package report
 // table_defs.go defines the tables used for generating reports
 
 import (
+	"encoding/csv"
 	"fmt"
 	"log/slog"
 	"math"
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"perfspect/internal/cpudb"
 	"perfspect/internal/script"
@@ -118,6 +120,8 @@ const (
 	CodePathFrequencyTableName = "Code Path Frequency"
 	// lock table names
 	KernelLockAnalysisTableName = "Kernel Lock Analysis"
+	// process watch instruction mix table names
+	InstructionMixTableName = "Instruction Mix"
 )
 
 const (
@@ -612,6 +616,16 @@ var tableDefinitions = map[string]TableDefinition{
 		},
 		FieldsFunc:            powerStatsTableValues,
 		HTMLTableRendererFunc: powerStatsTableHTMLRenderer},
+	InstructionMixTableName: {
+		Name:      InstructionMixTableName,
+		MenuLabel: InstructionMixTableName,
+		HasRows:   true,
+		ScriptNames: []string{
+			script.InstructionMixScriptName,
+		},
+		FieldsFunc:            instructionMixTableValues,
+		HTMLTableRendererFunc: instructionMixTableHTMLRenderer,
+	},
 	//
 	// flamegraph tables
 	//
@@ -1937,3 +1951,84 @@ func kernelLockAnalysisTableValues(outputs map[string]script.ScriptOutput) []Fie
 	}
 	return fields
 }
+
+func instructionMixTableValues(outputs map[string]script.ScriptOutput) []Field {
+	// first two lines are not part of the CSV output, they are the start time and interval
+	var startTime time.Time
+	var interval int
+	for i, line := range strings.Split(outputs[script.InstructionMixScriptName].Stdout, "\n") {
+		if i == 0 {
+			if !strings.HasPrefix(line, "TIME") {
+				slog.Error("instruction mix output is not in expected format, missing TIME")
+				return []Field{}
+			} else {
+				val := strings.Split(line, " ")[1]
+				var err error
+				startTime, err = time.Parse("15:04:05", val)
+				if err != nil {
+					slog.Error(fmt.Sprintf("unable to parse instruction mix start time: %s", val))
+					return []Field{}
+				}
+			}
+		} else if i == 1 {
+			if !strings.HasPrefix(line, "INTERVAL") {
+				slog.Error("instruction mix output is not in expected format, missing INTERVAL")
+				return []Field{}
+			} else {
+				val := strings.Split(line, " ")[1]
+				var err error
+				interval, err = strconv.Atoi(val)
+				if err != nil {
+					slog.Error(fmt.Sprintf("unable to convert instruction mix interval to int: %s", val))
+					return []Field{}
+				}
+			}
+		} else {
+			break
+		}
+	}
+	// parse the CSV output
+	csvOutput := strings.Join(strings.Split(outputs[script.InstructionMixScriptName].Stdout, "\n")[2:], "\n")
+	r := csv.NewReader(strings.NewReader(csvOutput))
+	rows, err := r.ReadAll()
+	if err != nil {
+		slog.Error(err.Error())
+		return []Field{}
+	}
+	if len(rows) < 2 {
+		slog.Error("instruction mix output is not in expected format")
+		return []Field{}
+	}
+	fields := []Field{{Name: "Time"}}
+	// first row is the header, extract field names, skip the first three fields (interval, pid, name)
+	if len(rows[0]) < 3 {
+		slog.Error("instruction mix output is not in expected format")
+		return []Field{}
+	}
+	for _, field := range rows[0][3:] {
+		fields = append(fields, Field{Name: field})
+	}
+	sample := -1
+	// values start in 2nd row, we're only interested in the first row of the sample
+	for _, row := range rows[1:] {
+		if len(row) < 2+len(fields) {
+			continue
+		}
+		rowSample, err := strconv.Atoi(row[0])
+		if err != nil {
+			slog.Error(fmt.Sprintf("unable to convert instruction mix sample to int: %s", row[0]))
+			continue
+		}
+		if rowSample != sample { // new sample
+			sample = rowSample
+			for i := range fields {
+				if i == 0 {
+					fields[i].Values = append(fields[i].Values, startTime.Add(time.Duration(sample*interval)*time.Second).Format("15:04:05"))
+				} else {
+					fields[i].Values = append(fields[i].Values, row[i+2])
+				}
+			}
+		}
+	}
+	return fields
+}