switch.sh

#!/usr/bin/env bash
# shellcheck disable=SC2002
set -euo pipefail

cd "$(dirname "$0")"
source_env_file() {
    local env_file="$1"
    if [[ -f $env_file ]]; then
        set -a
        # shellcheck disable=SC1090,SC1091
        source "$env_file"
        set +a
    fi
}
# Source .env from the resolved (real) directory first (in case $0 is a
# symlink), then from the directory of $0 so the latter can override.
source_env_file "$(dirname "$(readlink -f "$0")")/.env"
source_env_file "$(dirname "$0")/.env"
verbose=""
LEAD_HOST="f13"
ulimit -c unlimited

have() {
    local cmd="$1"
    if command -v "$cmd" &>/dev/null; then
        return 0
    fi
    return 1
}

log_step() {
    {
        echo "$(tput setaf 2)################################################################################$(tput sgr0)"
        echo "$1 (at $(date))"
        echo "$(tput setaf 2)################################################################################$(tput sgr0)"
    } >&2
}
log_info() {
    echo -e "info: $1" >&2
}
log_warning() {
    echo -e "$(tput setaf 3)warning: $1$(tput sgr0)" >&2
}
log_error() {
    echo -e "$(tput setaf 1)error: $1$(tput sgr0)" >&2
}

guard_pid() {
    local target="$1"
    local this_pid=$$
    local pidfile
    pidfile="/tmp/$(echo "$0" | sed 's/\//-/g').${target}.pid"
    if [[ -e $pidfile ]]; then
        old_pid="$(cat "$pidfile")"
        if [[ -e "/proc/$old_pid" ]]; then
            log_error "process $old_pid is running for $target"
            exit 1
        fi
    fi
    echo "$this_pid" >"$pidfile"
}

add_ssh_keys() {
    local key="$HOME/.ssh/id_rsa"
    if [[ -f $key ]] && ! ssh-add -l | grep -q "$key"; then
        echo "ssh-add ..." >&2
        ssh-add "$key"
    fi
}

gnupg_to_mutt() {
    log_step "running gnupg-to-mutt.pl"
    if type gnupg-to-mutt.pl &>/dev/null; then
        gnupg-to-mutt.pl
    else
        log_warning "gnupg-to-mutt.pl not found"
    fi
}

flake_update() {
    local update_mode="$1"
    log_step "updating flake in $update_mode mode"

    if [[ $update_mode == "full" ]]; then
        flake_update_recursively "."
        gnupg_to_mutt
    elif [[ $update_mode == "fast" ]]; then
        flake_update_one "."
        # grep '\.url = "path:' flake.nix | sed s/\.url.*// | sed 's/ //g' |
        #     while read flake; do
        #         (set -x; nix flake update ${verbose:+"--verbose"} "$flake")
        #     done
    else
        log_error "unknown update mode: $update_mode"
        exit 1
    fi
}

flake_update_recursively() (
    local path="$1"
    local flake="$path/flake.nix"
    echo ">> recursively walk flake $path"
    if [[ ! -e $flake ]]; then
        log_error "flake.nix not found in $path"
        return
    fi
    if grep -q '\.url = "path:' "$flake"; then
        local resolved_path
        resolved_path="$(realpath "$path")"
        if [[ $(basename "$resolved_path") == "myconfig" && $(hostname) != "$LEAD_HOST" ]]; then
            log_info "skip recursing beyond myconfig flake on host $(hostname) (not $LEAD_HOST)"
        else
            grep '\.url = "path:' "$flake" | sed s/.*path:// | sed 's/".*//g' |
                while read -r flake_path; do
                    if [[ $flake_path != /* ]]; then
                        flake_path="$(realpath "$path/$flake_path")"
                    fi
                    flake_update_recursively "$flake_path"
                done
        fi
    fi
    flake_update_one "$path"
)

flake_update_one() (
    local path="$1"
    local num_of_tries="${2:-1}"
    echo ">>> update flake $path and commit lock file"
    local resolved_path
    resolved_path="$(realpath "$path")"
    if [[ $(basename "$resolved_path") == "myconfig" && $(hostname) != "$LEAD_HOST" ]]; then
        log_info "skip updating myconfig flake on host $(hostname) (not $LEAD_HOST)"
        return
    fi
    if [[ $num_of_tries -gt 1 ]]; then
        log_warning "retry $num_of_tries times"
    fi
    cd "$path"
    set -x
    nix flake update ${verbose:+"--verbose"} --commit-lock-file || {
        set +x
        if [[ $num_of_tries -lt 3 ]]; then
            flake_update_one "$path" $((num_of_tries + 1))
        else
            log_error "failed to update flake $path"
            exit 1
        fi
    }
)
get_out_link_of_target() {
    local target="$1"
    echo '../result.'"$target"
}
git_repo_is_clean() {
    local repo="$1"
    [[ -d "$repo/.git" || -f "$repo/.git" ]] || return 0
    git -C "$repo" diff --quiet HEAD 2>/dev/null
}
# Returns 0 if the running system was built from exactly the current state of
# the myconfig repo: `myconfig-ahead` (see flake.lib.nix) compares the repo
# HEAD against the commit recorded in /run/myconfig/myconfig-commit and exits
# 1 while printing "0" iff HEAD matches and the tree is clean. Additionally
# the flake directory we build from (the cwd, e.g. priv) must be clean.
is_up_to_date() {
    local script_dir="$1"
    local out rc
    have myconfig-ahead || return 1
    rc=0
    out="$(MYCONFIG_REPO="$script_dir" myconfig-ahead 2>/dev/null)" || rc=$?
    [[ $rc -eq 1 && $out == "0" ]] || return 1
    git_repo_is_clean "." || return 1
    return 0
}
build() (
    local target="$1"
    shift
    local out_link="$1"
    shift
    local build_attr="${BUILD_ATTR:-toplevel}"
    log_step "building $build_attr for $target to $out_link"
    local system='.#nixosConfigurations.'"$target"'.config.system.build.'"$build_attr"

    set -x
    nix build \
        ${verbose:+"--verbose"} \
        -L \
        --fallback \
        --log-format bar-with-logs \
        --out-link "$out_link" \
        --keep-going \
        "$@" \
        "$system"
)
du_of_out_link() {
    local target="$1"
    local out_link
    out_link="$(get_out_link_of_target "$target")"

    local du_file="$out_link"'.du'
    local du_csv_file="$out_link"'.du.csv'

    log_step "compute $du_file"

    nix path-info -rhsS "$out_link" |
        tee "$du_file" |
        gawk '
        BEGIN {FS="\t"; OFS=","} {
            if ($3 ~ / *[0-9\.]+ KiB/)
                { a=gensub(/ *([0-9\.]+)( KiB)/,"\\1","g",$3); \
                printf "%s KiB,%s,%s,%s\n", a,$3,$2,$1} \
            else if ($3 ~ / *[0-9\.]+ MiB/)
                { a=gensub(/ *([0-9\.]+)( MiB)/,"\\1","g",$3); \
                printf "%s KiB,%s,%s,%s\n", a*1024,$3,$2,$1} \
            else if ($3 ~ / *[0-9\.]+ GiB/)
                { a=gensub(/ *([0-9\.]+)( GiB)/,"\\1","g",$3); \
                printf "%s KiB,%s,%s,%s\n", a*1024*1024,$3,$2,$1} \
            else {print "?",$3,$2,$1}
            }
        ' |
        sort -n |
        awk 'BEGIN {FS=","; OFS=","} {print $2,$3,$4}' >"$du_csv_file"
    echo "$du_file"
}
home_manager_files_from_du() {
    local target="$1"
    local du_file="$2"
    local hm_files_dir
    hm_files_dir="$(get_out_link_of_target "$target").home-manager-files"

    if [[ -d $hm_files_dir ]]; then
        chmod -R u+rwx "$hm_files_dir"
        rm -rf "$hm_files_dir"
    fi
    mkdir "$hm_files_dir"

    log_step "copy home-manager files to $hm_files_dir"

    cat "$du_file" |
        awk '{print $1}' |
        grep -E '.*-home-manager-files$' |
        while read -r hmfiles; do
            local target
            target="$hm_files_dir/$(basename "$hmfiles")"
            >&2 echo "copy to $target"
            cp -Lr "$hmfiles" "$target" &>/dev/null || true
            chmod -R u+rwx "$target"
        done
}
get_ip_of_target_from_metadata() {
    local metadata_file="$1"
    local target="$2"
    local use_wg="${3:-false}"
    cat "$metadata_file" |
        if [[ $use_wg == "true" ]]; then
            jq -r ".hosts.${target}.wireguard.wg0.ip4" || true
        else
            jq -r ".hosts.${target}.ip4" || true
        fi
}
get_ip_of_target() {
    local target="$1"
    local use_wg="${2:-false}"

    if [[ $use_wg == "true" ]]; then
        local hosts_alias="$target.wg0"
        if grep -q " $hosts_alias"'$' /etc/hosts; then
            # When using a jump host, return the raw IP so the jump host can reach it
            local ip_from_hosts
            ip_from_hosts="$(getent hosts "$hosts_alias" | awk '{print $1}')"
            if [[ -n $ip_from_hosts ]]; then
                echo "$ip_from_hosts"
                return
            fi
            echo "$hosts_alias"
            return
        fi
    fi

    ip="$(get_ip_of_target_from_metadata ./hosts/metadata.json "$target" "$use_wg")"
    if [[ $ip == "null" ]]; then
        ip="$(get_ip_of_target_from_metadata ../myconfig/hosts/metadata.json "$target" "$use_wg")"
    fi
    if [[ $ip == "null" ]]; then
        if grep -q " $target\$" /etc/hosts; then
            echo "$target"
            return
        fi
    fi
    if [[ $ip == "null" ]]; then
        log_error "ip for $target not found in metadata"
        exit 1
    fi
    if [[ $ip == "" ]]; then
        log_error "ip can not be empty"
        exit 1
    fi
    echo "$ip"
}
copy_closure_to_target() (
    local targetIP="$1"
    local out_link="$2"
    log_step "copying closure to $targetIP"
    set -x
    until nix-copy-closure --to "$targetIP" "$out_link"; do
        log_warning "retry nix-copy-closure in 10s"
        sleep 10
    done
)
diff_build_results() (
    local old_result="$1"
    local out_link="$2"
    if command -v nvd &>/dev/null; then
        log_step "diffing $old_result and $out_link"
        set -x
        nvd list -r "$out_link" >"$out_link"'.list'
        nvd diff "$old_result" "$out_link" | tee "$out_link"'.diff'
    fi
)
direct_deploy_locally() {
    local out_link="$1"
    local command="$2"
    log_step "direct deploying $out_link to $(hostname)"
    local store_path
    store_path="$(nix-store -q "$out_link")"
    set -x
    sudo nix-env --profile /nix/var/nix/profiles/system --set "$store_path"
    sudo "$store_path/bin/switch-to-configuration" "$command"
}
deploy() (
    local target="$1"
    local out_link="$2"
    local command="$3"

    # if [[ "$target" == "$(hostname)" ]]; then
    #   direct_deploy_locally "$out_link" "$command"
    #   return
    # fi

    log_step "deploying $out_link to $target"
    local use_wg="${4:-false}"

    local ssh_opts='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
    if [[ $use_wg == "true" ]]; then
        ssh_opts="$ssh_opts -J vserver"
    fi
    export NIX_SSHOPTS="$ssh_opts"

    cmd="nixos-rebuild"
    if [[ $target != "$(hostname)" ]]; then
        targetIP="root@$(get_ip_of_target "$target" "$use_wg")"
        targetIP="${TARGET_IP:-"$targetIP"}"

        copy_closure_to_target "$targetIP" "$out_link"

        cmd="$cmd --target-host $targetIP"
    else
        cmd="sudo $cmd"
    fi
    set -x
    until $cmd \
        `# --build-host localhost` \
        "$command" `#-p test` \
        ${verbose:+"--verbose"} \
        --flake '.#'"$target"; do
        echo "... retry nixos-rebuild"
    done
)
sbom() (
    local target="$1"
    local out_link="$2"
    local sbom_dir="$out_link.sbom"
    mkdir -p "$sbom_dir"
    log_step "sbom $sbom_dir"
    cd "$sbom_dir"
    sbomnix "$out_link"
    local cdx="sbom.cdx.json"
    if [[ -f $cdx ]]; then
        local dtrack_token="${DTRACK_TOKEN:-}"
        if [[ -z $dtrack_token ]]; then
            if have pass; then
                dtrack_token="$(pass "priv/home-lab/automation@dtrack.nuc.wg0.maxhbr.local" -p || true)"
            else
                log_warning "pass not found, skipping dtrack upload"
            fi
        fi
        if [[ -n $dtrack_token ]]; then
            local dtrack_api="https://dtrack-api.nuc.wg0.maxhbr.local"
            local project_name="host.$target"
            local version
            version="$(jq -r '.metadata.component.name' "$cdx")"

            # Find the latest existing project with a matching name so we can
            # replace its SBOM in place instead of creating a new project for
            # every version. Pick the one with the most recent BOM import.
            local project_uuid
            project_uuid="$(curl -sf \
                -H "X-Api-Key: $dtrack_token" \
                "$dtrack_api/api/v1/project?name=$project_name&excludeInactive=false" |
                jq -r --arg name "$project_name" '
                    [.[] | select(.name == $name)]
                    | sort_by(.lastBomImport // 0)
                    | last
                    | .uuid // empty' 2>/dev/null || true)"

            if [[ -n $project_uuid ]]; then
                log_info "uploading sbom to dtrack (updating existing project $project_uuid)"
                # Update the project version to reflect the current build.
                curl -sf -X PATCH "$dtrack_api/api/v1/project/$project_uuid" \
                    -H "X-Api-Key: $dtrack_token" \
                    -H "Content-Type: application/json" \
                    -d "$(jq -n --arg v "$version" '{version: $v}')" >/dev/null || true
                curl -X POST "$dtrack_api/api/v1/bom" \
                    -H "X-Api-Key: $dtrack_token" \
                    -F "project=$project_uuid" \
                    -F "bom=@$cdx"
            else
                log_info "uploading sbom to dtrack (creating project $project_name)"
                curl -X POST "$dtrack_api/api/v1/bom" \
                    -H "X-Api-Key: $dtrack_token" \
                    -F "autoCreate=true" \
                    -F "projectName=$project_name" \
                    -F "projectVersion=$version" \
                    -F "bom=@$cdx"
            fi
        else
            log_warning "no dtrack token"
        fi
    fi
)
main() {
    local MODE=""
    local COMMAND="switch"
    local no_args="false"
    if [[ $# -eq 0 ]]; then
        no_args="true"
    fi
    if [[ $# -gt 0 && ($1 == "--verbose" || $1 == "-v" || $1 == "-vv") ]]; then
        verbose="--verbose"
        if [[ $1 == "-vv" ]]; then
            set -x
        fi
        shift
    elif [[ $# -gt 0 && ($1 == "--quiet" || $1 == "-q") ]]; then
        verbose=""
        shift
    fi
    local local_build=()
    if [[ $# -gt 0 && $1 == "--fast" ]]; then
        MODE="$1"
        shift
    elif [[ $# -gt 0 && $1 == "--test" ]]; then
        MODE="$1"
        shift
    elif [[ $# -gt 0 && $1 == "--use-wg" ]]; then
        MODE="$1"
        shift
    elif [[ $# -gt 0 && $1 == "--local" ]]; then
        local_build=("--builders" "")
        shift
    elif [[ $# -gt 0 && $1 == "--build-vm" ]]; then
        MODE="$1"
        shift
    fi
    if [[ $# -gt 0 && $1 == "--boot" ]]; then
        COMMAND="boot"
        shift
    elif [[ $# -gt 0 && $1 == "--switch" ]]; then
        COMMAND="switch"
        shift
    fi

    local target="${1:-$(hostname)}"

    guard_pid "$target"

    if [[ $target != "$(hostname)" ]]; then
        add_ssh_keys
        if command -v nmcli &>/dev/null && nmcli connection show --active | grep -q " tun"; then
            log_error "VPN is active, refusing to update remote host $target"
            exit 1
        fi
    fi

    ################################################################################
    # setup logging
    start_time="$(date +%s)"
    exec &> >(while read -r line; do
        current_time="$(date +%s)"
        time_diff="$((current_time - start_time))"
        echo "$(date -u -d@"$time_diff" +"%H:%M:%S")| $line"
    done)
    local logsDir="../_logs"
    mkdir -p "$logsDir"
    local logfile
    logfile="$logsDir/$(date +%Y-%m-%d)-myconfig-${target}.log"
    echo -e "\n\n\n\n\n\n\n" >>"$logfile"
    exec &> >(tee -a "$logfile")
    log_info "starting with...\nMODE=$MODE\nCOMMAND=$COMMAND\ntarget=$target\nverbose=$verbose\n"
    ################################################################################

    if have pass; then
        local token
        token="$(pass github-bot-token2 -p || true)"
        if [[ -n $token ]]; then
            log_info "setting github token"
            NIX_CONFIG="access-tokens = github.com=$token"
            export NIX_CONFIG
        else
            log_warning "no github token"
        fi
    else
        log_warning "pass not found, skipping github token setup"
    fi

    local script_dir
    script_dir="$(dirname "$(readlink -f "$0")")"
    if [[ -d "$script_dir/.git" ]]; then
        log_info "checking for upstream commits"
        (
            cd "$script_dir"
            git fetch -q
            if git rev-list HEAD.."@{u}" --count 2>/dev/null | grep -q '^[1-9]'; then
                git pull --rebase || {
                    log_error "git pull --rebase failed due to conflicts. resolve conflicts and run again"
                    git rebase --abort
                    exit 1
                }
            fi
        )
    fi

    flake_update "$([[ $MODE == "" ]] && echo "full" || echo "fast")"

    local out_link
    out_link="$(get_out_link_of_target "$target")"
    if [[ $MODE == "--build-vm" ]]; then
        out_link="${out_link}.vm"
    fi
    local latest_logfile="${out_link}.log"
    ln -sf "$(realpath -m --relative-to="$(dirname "$latest_logfile")" "$logfile")" "$latest_logfile"

    if [[ $no_args == "true" && $target == "$(hostname)" ]] && is_up_to_date "$script_dir"; then
        log_info "nothing to do: $target already runs $(cat /run/myconfig/myconfig-commit) and there are no uncommitted changes"
        return 0
    fi

    if [[ $MODE == "--build-vm" ]]; then
        BUILD_ATTR="vm" build "$target" "$out_link" "${local_build[@]}" ||
            BUILD_ATTR="vm" build "$target" "$out_link" --keep-failed --no-eval-cache "${local_build[@]}"
        log_step "VM built for $target"
        local vm_run_script
        vm_run_script="$(realpath -m "$out_link")/bin/run-${target}-vm"
        # shellcheck disable=SC2016 # $PWD is a literal hint, not an expansion
        log_info 'run it via (from a writable dir; disk image is created in $PWD):'
        log_info "> NIX_DISK_IMAGE=/tmp/${target}-vm.qcow2 $vm_run_script"
        return
    fi

    local old_result
    old_result="$(readlink -f "$out_link" || true)"
    build "$target" "$out_link" "${local_build[@]}" || build "$target" "$out_link" --keep-failed --no-eval-cache "${local_build[@]}"
    if [[ -e $old_result ]]; then
        diff_build_results "$old_result" "$out_link"
    fi
    local du_file
    du_file="$(du_of_out_link "$target")"
    home_manager_files_from_du "$target" "$du_file"
    if [[ $MODE == "--use-wg" ]]; then
        deploy "$target" "$out_link" "$COMMAND" true
    elif [[ $MODE == "--test" ]]; then
        if [[ $target == "$(hostname)" ]]; then
            deploy "$target" "$out_link" dry-activate
        fi
    elif [[ $MODE != "--test" ]]; then
        deploy "$target" "$out_link" "$COMMAND"
        if [[ $COMMAND == "boot" ]]; then
            echo "manually enable via:"
            local store_path
            store_path="$(nix-store -q "$out_link")"
            echo "> sudo nix-env --profile /nix/var/nix/profiles/system --set $store_path"
            echo "> sudo $store_path/bin/switch-to-configuration switch"
        fi
        if [[ $target == "$(hostname)" ]]; then
            local generations="$out_link"'.generations'
            nixos-rebuild list-generations >"$generations"
            log_info "generations:"
            cat "$generations" | head -5
        fi
    fi
    if [[ $MODE == "--use-wg" || $MODE != "--test" ]]; then
        sbom "$target" "$out_link" || true
    fi
}

################################################################################

if ! have nix || ! have nvd || ! have sbomnix; then
    log_error "nvd not found"
    exec nix-shell -p nvd -p sbomnix --command "$0" "$@"
fi

main "$@"
times
date