diff --git a/cmd/ocifit/main.go b/cmd/ocifit/main.go index cbd434f..f390ccd 100644 --- a/cmd/ocifit/main.go +++ b/cmd/ocifit/main.go @@ -51,6 +51,7 @@ const ( var ( universalDeserializer = serializer.NewCodecFactory(runtime.NewScheme()).UniversalDeserializer() + skipInstanceTypes = make(map[string]bool) ) // WebhookServer with Node Cache and a direct k8s client @@ -408,12 +409,29 @@ func (ws *WebhookServer) getAllNodeFeatures(ctx context.Context) ([]map[string]i if err := ws.k8sClient.List(ctx, &nodeList, client.MatchingFields{"spec.unschedulable": "false"}); err != nil { return nil, err } + + // We will do a filtering here for types we don't want to include + identityLabelKey := os.Getenv("NODE_IDENTITY_LABEL") + if identityLabelKey == "" { + identityLabelKey = instanceTypeLabel + } + var featureMatrix []map[string]interface{} for _, node := range nodeList.Items { features := make(map[string]interface{}) for key, val := range node.Labels { features[key] = val } + + // Be conservative - don't include nodes we cannot identify + identityValue, ok := features[identityLabelKey].(string) + if !ok { + continue + } + if skipInstanceTypes[identityValue] { + log.Printf("Skipping dynamically discovered instance '%s' as it is in the skip list.", identityValue) + continue + } featureMatrix = append(featureMatrix, features) } return featureMatrix, nil @@ -422,6 +440,7 @@ func (ws *WebhookServer) getAllNodeFeatures(ctx context.Context) ([]map[string]i // getCombinedNodeFeatures merges dynamically discovered nodes with a static catalog from a directory. // It correctly handles static files that contain a JSON array (list) of node feature objects. func (ws *WebhookServer) getCombinedNodeFeatures(ctx context.Context, staticFeaturesDir string) ([]map[string]interface{}, error) { + // Get all currently running schedulable nodes. dynamicallyDiscoveredFeatures, err := ws.getAllNodeFeatures(ctx) if err != nil { @@ -484,6 +503,14 @@ func (ws *WebhookServer) getCombinedNodeFeatures(ctx context.Context, staticFeat } for _, staticFeatures := range featureList { + identityValue, ok := staticFeatures[identityLabelKey].(string) + if !ok { + continue + } + if skipInstanceTypes[identityValue] { + log.Printf("Skipping static instance '%s' from file '%s' as it is in the skip list.", identityValue, file.Name()) + continue + } // De-duplication check... if identityValue, ok := staticFeatures[identityLabelKey].(string); ok { @@ -606,7 +633,25 @@ func (ws *WebhookServer) handleMutate(w http.ResponseWriter, r *http.Request) { w.Write(respBody) } +func discoverSkipInstances() { + skipList := os.Getenv("SKIP_INSTANCE_TYPES") + if skipList != "" { + instances := strings.Split(skipList, ",") + for _, instance := range instances { + trimmed := strings.TrimSpace(instance) + if trimmed != "" { + log.Printf("Configuration: Will skip instance type '%s'", trimmed) + skipInstanceTypes[trimmed] = true + } + } + } +} + func main() { + + // Ensure we parse skip instances + discoverSkipInstances() + config, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) if err != nil { log.Fatalf("Error building kubeconfig: %s", err.Error()) diff --git a/mlserver/models/lasso_model_fom.joblib b/mlserver/models/lasso_model_fom.joblib index bbfb8b8..9424130 100644 Binary files a/mlserver/models/lasso_model_fom.joblib and b/mlserver/models/lasso_model_fom.joblib differ diff --git a/mlserver/models/lasso_model_fom_per_dollar.joblib b/mlserver/models/lasso_model_fom_per_dollar.joblib index f18ab45..88d2f4a 100644 Binary files a/mlserver/models/lasso_model_fom_per_dollar.joblib and b/mlserver/models/lasso_model_fom_per_dollar.joblib differ diff --git a/mlserver/models/lasso_model_gflops_per_second_ddot.joblib b/mlserver/models/lasso_model_gflops_per_second_ddot.joblib new file mode 100644 index 0000000..ff3a1c8 Binary files /dev/null and b/mlserver/models/lasso_model_gflops_per_second_ddot.joblib differ diff --git a/mlserver/models/lasso_model_gflops_per_second_mg.joblib b/mlserver/models/lasso_model_gflops_per_second_mg.joblib new file mode 100644 index 0000000..de77ce1 Binary files /dev/null and b/mlserver/models/lasso_model_gflops_per_second_mg.joblib differ diff --git a/mlserver/models/lasso_model_gflops_per_second_spmv.joblib b/mlserver/models/lasso_model_gflops_per_second_spmv.joblib new file mode 100644 index 0000000..d5dfd9c Binary files /dev/null and b/mlserver/models/lasso_model_gflops_per_second_spmv.joblib differ diff --git a/mlserver/models/lasso_model_gflops_per_second_waxpby.joblib b/mlserver/models/lasso_model_gflops_per_second_waxpby.joblib new file mode 100644 index 0000000..ad78ab6 Binary files /dev/null and b/mlserver/models/lasso_model_gflops_per_second_waxpby.joblib differ diff --git a/mlserver/models/lasso_model_memory_bandwidth_across_kernels_read.joblib b/mlserver/models/lasso_model_memory_bandwidth_across_kernels_read.joblib new file mode 100644 index 0000000..7d8aabc Binary files /dev/null and b/mlserver/models/lasso_model_memory_bandwidth_across_kernels_read.joblib differ diff --git a/mlserver/models/lasso_model_memory_bandwidth_across_kernels_total.joblib b/mlserver/models/lasso_model_memory_bandwidth_across_kernels_total.joblib new file mode 100644 index 0000000..db4f75a Binary files /dev/null and b/mlserver/models/lasso_model_memory_bandwidth_across_kernels_total.joblib differ diff --git a/mlserver/models/lasso_model_memory_bandwidth_across_kernels_write.joblib b/mlserver/models/lasso_model_memory_bandwidth_across_kernels_write.joblib new file mode 100644 index 0000000..08626e0 Binary files /dev/null and b/mlserver/models/lasso_model_memory_bandwidth_across_kernels_write.joblib differ diff --git a/mlserver/models/lasso_model_memory_used_data_total_gbytes.joblib b/mlserver/models/lasso_model_memory_used_data_total_gbytes.joblib new file mode 100644 index 0000000..6648a6a Binary files /dev/null and b/mlserver/models/lasso_model_memory_used_data_total_gbytes.joblib differ diff --git a/mlserver/models/lasso_model_mpi_allreduce_avg.joblib b/mlserver/models/lasso_model_mpi_allreduce_avg.joblib new file mode 100644 index 0000000..7a29fa4 Binary files /dev/null and b/mlserver/models/lasso_model_mpi_allreduce_avg.joblib differ diff --git a/mlserver/models/lasso_model_mpi_allreduce_max.joblib b/mlserver/models/lasso_model_mpi_allreduce_max.joblib new file mode 100644 index 0000000..f1f335a Binary files /dev/null and b/mlserver/models/lasso_model_mpi_allreduce_max.joblib differ diff --git a/mlserver/models/lasso_model_total_cg_iterations.joblib b/mlserver/models/lasso_model_total_cg_iterations.joblib new file mode 100644 index 0000000..728e314 Binary files /dev/null and b/mlserver/models/lasso_model_total_cg_iterations.joblib differ