Merge pull request #15 from NVIDIA/add-tuned-scripts

feat(tuned): tuned scripts can now be used by the tuned package

Author: riceriley59

tuned/README.md

@@ -145,11 +145,17 @@ The package expects configmaps to be available in `${SKYHOOK_DIR}/configmaps/`:
 
 #### Optional Configmaps
 
-- **Custom profile files**: Any other files in the configmaps directory will be treated as custom tuned profile configurations
+- **Custom profile files**: Any files in the configmaps directory (except `tuned_profile` and `*_script` files) will be treated as custom tuned profile configurations
   - File name becomes the profile name
   - File contents become the `tuned.conf` for that profile
   - Files are deployed to `/etc/tuned/<profile_name>/tuned.conf`
 
+- **Script files**: Any files ending with `_script` will be deployed as executable scripts to `/etc/tuned/scripts/`
+  - File name pattern: `<name>_script` (e.g., `setup_script`, `my_optimization_script`)
+  - Scripts are deployed to `/etc/tuned/scripts/<name>` (the `_script` suffix is removed)
+  - Scripts are automatically made executable (`chmod +x`)
+  - Can be referenced in tuned profiles using the `[script]` plugin
+
 ### Example Custom Profile
 
 ```ini
@@ -168,6 +174,49 @@ readahead=4096
 transparent_hugepages=never
 ```
 
+### Example Custom Profile with Scripts
+
+```ini
+# configmaps/ai-optimized-profile
+[main]
+summary=AI/ML optimized profile with custom scripts
+
+[cpu]
+governor=performance
+energy_perf_bias=performance
+
+[script]
+type=script
+script=/etc/tuned/scripts/ai_setup
+
+[vm]
+transparent_hugepages=always
+```
+
+```bash
+# configmaps/ai_setup_script
+#!/bin/bash
+# This script will be deployed to /etc/tuned/scripts/ai_setup
+
+echo "Setting up AI/ML optimizations..."
+
+# Configure GPU memory settings
+if [ -d /sys/class/drm ]; then
+    echo "Configuring GPU settings for AI workloads"
+    # Add GPU-specific optimizations here
+fi
+
+# Set up NUMA topology optimizations
+echo "Configuring NUMA settings for AI workloads"
+for node in /sys/devices/system/node/node*; do
+    if [ -d "$node" ]; then
+        echo 0 > "$node/compact"
+    fi
+done
+
+echo "AI optimization setup complete"
+```
+
 ## Usage Examples
 
 ### Basic Installation
@@ -223,6 +272,9 @@ spec:
 
             [bootloader]
             cmdline_myprofile=-kernel.panic +kernel.panic=20
+            
+            [script]
+            script=/etc/tuned/scripts/ai_init
         custom_profile_1: |-
             [main]
             summary=AI/ML performance profile
@@ -238,13 +290,31 @@ spec:
             [vm]
             transparent_hugepages=always     # large pages help with tensor allocations
             swappiness=10                    # avoid swapping under load
+        ai_init_script: |-
+            #!/bin/bash
+            # Custom AI/ML initialization script
+            echo "Initializing AI/ML optimizations..."
+            
+            # Configure GPU memory pools
+            if command -v nvidia-smi >/dev/null 2>&1; then
+                echo "Configuring NVIDIA GPU settings"
+                nvidia-smi -pm 1  # Enable persistence mode
+            fi
+            
+            # Set up memory allocation patterns for AI workloads
+            echo "Configuring memory allocation for AI workloads"
+            echo 1 > /proc/sys/vm/overcommit_memory
+            
+            echo "AI initialization complete"
 ```
 
 This example demonstrates:
 - **Node targeting**: Using `nodeSelectors` to target specific node groups
 - **Interrupt handling**: Configuring reboot interrupts for kernel-level changes
 - **Environment variables**: Setting `INTERRUPT=true` to handle verification during config changes
 - **Custom profiles**: Creating hierarchical profiles with `include` directive
+- **Custom scripts**: Using `_script` configmaps to deploy executable scripts
+- **Script integration**: Referencing deployed scripts in profiles using the `[script]` plugin
 - **AI/ML optimizations**: Performance settings optimized for machine learning workloads
 - **Kernel parameters**: Using `[sysctl]` and `[bootloader]` sections for low-level tuning
 

tuned/skyhook_dir/apply_tuned_profile.sh

@@ -21,16 +21,39 @@ set -u
 
 CONFIGMAP_DIR="${SKYHOOK_DIR}/configmaps"
 TUNED_DIR="/etc/tuned"
+SCRIPTS_DIR="/etc/tuned/scripts"
 
 # ensure tuned directory exists
 sudo mkdir -p "$TUNED_DIR"
+sudo mkdir -p "$SCRIPTS_DIR"
 
-# process all other files as custom profiles
+# First, deploy all scripts ending with "_script" to the shared scripts directory
+echo "Deploying scripts to $SCRIPTS_DIR..."
+for file in "$CONFIGMAP_DIR"/*_script; do
+    [ -f "$file" ] || continue # make sure file exists
+    
+    script_name=$(basename "$file")
+    script_path="$SCRIPTS_DIR/${script_name#*_script}"
+    
+    # If the script name is just "_script", use the full filename
+    if [ "${script_name#*_script}" = "" ]; then
+        script_path="$SCRIPTS_DIR/$(basename "$file")"
+    fi
+    
+    # Copy the script and make it executable
+    sudo cp "$file" "$script_path"
+    sudo chmod +x "$script_path"
+    echo "deployed script: $script_name -> $script_path"
+done
+
+# process all other files as custom profiles (skip tuned_profile and *_script files)
 for file in "$CONFIGMAP_DIR"/*; do
     [ -f "$file" ] || continue # make sure file exists
-    [ "$(basename "$file")" = "tuned_profile" ] && continue  # skip tuned_profile
 
     profile_name=$(basename "$file")
+    [ "$profile_name" = "tuned_profile" ] && continue  # skip tuned_profile
+    [[ "$profile_name" == *"_script" ]] && continue    # skip script files
+    
     custom_profile_dir="$TUNED_DIR/$profile_name"
 
     # Create a directory for the custom profile if it doesn't exist

tuned/skyhook_dir/apply_tuned_profile_check.sh

@@ -20,6 +20,7 @@ set -x
 
 CONFIGMAP_DIR="${SKYHOOK_DIR}/configmaps"
 TUNED_DIR="/etc/tuned"
+SCRIPTS_DIR="/etc/tuned/scripts"
 
 # check tuned service is installed and running
 if ! command -v tuned-adm >/dev/null 2>&1; then
@@ -38,12 +39,43 @@ if [ ! -d "$CONFIGMAP_DIR" ]; then
     exit 1
 fi
 
-# verify custom profiles
+if [ ! -d "$SCRIPTS_DIR" ]; then
+    echo "ERROR: scripts directory does not exist: $SCRIPTS_DIR"
+    exit 1
+fi
+
+# verify deployed scripts
+for file in "$CONFIGMAP_DIR"/*_script; do
+    [ -f "$file" ] || continue
+    
+    script_name=$(basename "$file")
+    script_path="$SCRIPTS_DIR/${script_name#*_script}"
+    
+    # If the script name is just "_script", use the full filename
+    if [ "${script_name#*_script}" = "" ]; then
+        script_path="$SCRIPTS_DIR/$(basename "$file")"
+    fi
+    
+    if [ ! -f "$script_path" ]; then
+        echo "ERROR: deployed script missing: $script_path"
+        exit 1
+    fi
+    
+    if [ ! -x "$script_path" ]; then
+        echo "ERROR: deployed script not executable: $script_path"
+        exit 1
+    fi
+    
+    echo "verified deployed script: $script_name -> $script_path"
+done
+
+# verify custom profiles (skip tuned_profile and *_script files)
 for file in "$CONFIGMAP_DIR"/*; do
     [ -f "$file" ] || continue
 
     base_file=$(basename "$file")
     [ "$base_file" = "tuned_profile" ] && continue
+    [[ "$base_file" == *"_script" ]] && continue    # skip script files
 
     custom_profile_dir="$TUNED_DIR/$base_file"
     custom_profile_file="$custom_profile_dir/tuned.conf"

tuned/skyhook_dir/post_interrupt_tuned_check.sh

@@ -20,6 +20,7 @@ set -x
 
 CONFIGMAP_DIR="${SKYHOOK_DIR}/configmaps"
 TUNED_DIR="/etc/tuned"
+SCRIPTS_DIR="/etc/tuned/scripts"
 
 # check tuned service is installed and running
 if ! command -v tuned-adm >/dev/null 2>&1; then
@@ -38,12 +39,43 @@ if [ ! -d "$CONFIGMAP_DIR" ]; then
     exit 1
 fi
 
-# verify custom profiles
+if [ ! -d "$SCRIPTS_DIR" ]; then
+    echo "ERROR: scripts directory does not exist: $SCRIPTS_DIR"
+    exit 1
+fi
+
+# verify deployed scripts
+for file in "$CONFIGMAP_DIR"/*_script; do
+    [ -f "$file" ] || continue
+    
+    script_name=$(basename "$file")
+    script_path="$SCRIPTS_DIR/${script_name#*_script}"
+    
+    # If the script name is just "_script", use the full filename
+    if [ "${script_name#*_script}" = "" ]; then
+        script_path="$SCRIPTS_DIR/$(basename "$file")"
+    fi
+    
+    if [ ! -f "$script_path" ]; then
+        echo "ERROR: deployed script missing: $script_path"
+        exit 1
+    fi
+    
+    if [ ! -x "$script_path" ]; then
+        echo "ERROR: deployed script not executable: $script_path"
+        exit 1
+    fi
+    
+    echo "verified deployed script: $script_name -> $script_path"
+done
+
+# verify custom profiles (skip tuned_profile and *_script files)
 for file in "$CONFIGMAP_DIR"/*; do
     [ -f "$file" ] || continue
 
     base_file=$(basename "$file")
     [ "$base_file" = "tuned_profile" ] && continue
+    [[ "$base_file" == *"_script" ]] && continue    # skip script files
 
     custom_profile_dir="$TUNED_DIR/$base_file"
     custom_profile_file="$custom_profile_dir/tuned.conf"