|
4 | 4 |
|
5 | 5 | use std::{ |
6 | 6 | collections::{BTreeMap, BTreeSet}, |
| 7 | + fmt::Display, |
7 | 8 | ops::Deref, |
8 | 9 | path::{Path, PathBuf}, |
| 10 | + process::Command, |
| 11 | + str::FromStr, |
9 | 12 | }; |
10 | 13 |
|
11 | 14 | use anyhow::{anyhow, bail, Context, Result}; |
@@ -85,6 +88,67 @@ struct InstanceInfo { |
85 | 88 | app_id: Vec<u8>, |
86 | 89 | } |
87 | 90 |
|
| 91 | +#[derive(Debug, Clone, Copy, PartialEq, Default)] |
| 92 | +enum FsType { |
| 93 | + #[default] |
| 94 | + Zfs, |
| 95 | + Ext4, |
| 96 | +} |
| 97 | + |
| 98 | +impl Display for FsType { |
| 99 | + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { |
| 100 | + match self { |
| 101 | + FsType::Zfs => write!(f, "zfs"), |
| 102 | + FsType::Ext4 => write!(f, "ext4"), |
| 103 | + } |
| 104 | + } |
| 105 | +} |
| 106 | + |
| 107 | +impl FromStr for FsType { |
| 108 | + type Err = anyhow::Error; |
| 109 | + fn from_str(s: &str) -> Result<Self, Self::Err> { |
| 110 | + match s.to_lowercase().as_str() { |
| 111 | + "zfs" => Ok(FsType::Zfs), |
| 112 | + "ext4" => Ok(FsType::Ext4), |
| 113 | + _ => bail!("Invalid filesystem type: {s}, supported types: zfs, ext4"), |
| 114 | + } |
| 115 | + } |
| 116 | +} |
| 117 | + |
| 118 | +#[derive(Debug, Clone, Default)] |
| 119 | +struct DstackOptions { |
| 120 | + storage_encrypted: bool, |
| 121 | + storage_fs: FsType, |
| 122 | +} |
| 123 | + |
| 124 | +fn parse_dstack_options(shared: &HostShared) -> Result<DstackOptions> { |
| 125 | + let cmdline = fs::read_to_string("/proc/cmdline").context("Failed to read /proc/cmdline")?; |
| 126 | + |
| 127 | + let mut options = DstackOptions { |
| 128 | + storage_encrypted: true, // Default to encryption enabled |
| 129 | + storage_fs: FsType::Zfs, // Default to ZFS |
| 130 | + }; |
| 131 | + |
| 132 | + for param in cmdline.split_whitespace() { |
| 133 | + if let Some(value) = param.strip_prefix("dstack.storage_encrypted=") { |
| 134 | + match value { |
| 135 | + "0" | "false" | "no" | "off" => options.storage_encrypted = false, |
| 136 | + "1" | "true" | "yes" | "on" => options.storage_encrypted = true, |
| 137 | + _ => { |
| 138 | + bail!("Invalid value for dstack.storage_encrypted: {value}"); |
| 139 | + } |
| 140 | + } |
| 141 | + } else if let Some(value) = param.strip_prefix("dstack.storage_fs=") { |
| 142 | + options.storage_fs = value.parse().context("Failed to parse dstack.storage_fs")?; |
| 143 | + } |
| 144 | + } |
| 145 | + |
| 146 | + if let Some(fs) = &shared.app_compose.storage_fs { |
| 147 | + options.storage_fs = fs.parse().context("Failed to parse storage_fs")?; |
| 148 | + } |
| 149 | + Ok(options) |
| 150 | +} |
| 151 | + |
88 | 152 | impl InstanceInfo { |
89 | 153 | fn is_initialized(&self) -> bool { |
90 | 154 | !self.instance_id_seed.is_empty() |
@@ -621,38 +685,122 @@ impl<'a> Stage0<'a> { |
621 | 685 | } |
622 | 686 | } |
623 | 687 |
|
624 | | - async fn mount_data_disk(&self, initialized: bool, disk_crypt_key: &str) -> Result<()> { |
| 688 | + async fn mount_data_disk( |
| 689 | + &self, |
| 690 | + initialized: bool, |
| 691 | + disk_crypt_key: &str, |
| 692 | + opts: &DstackOptions, |
| 693 | + ) -> Result<()> { |
625 | 694 | let name = "dstack_data_disk"; |
626 | | - let fs_dev = "/dev/mapper/".to_string() + name; |
627 | 695 | let mount_point = &self.args.mount_point; |
| 696 | + |
| 697 | + // Determine the device to use based on encryption settings |
| 698 | + let fs_dev = if opts.storage_encrypted { |
| 699 | + format!("/dev/mapper/{name}") |
| 700 | + } else { |
| 701 | + self.args.device.to_string_lossy().to_string() |
| 702 | + }; |
| 703 | + |
| 704 | + cmd!(mkdir -p $mount_point).context("Failed to create mount point")?; |
| 705 | + |
628 | 706 | if !initialized { |
629 | 707 | self.vmm |
630 | 708 | .notify_q("boot.progress", "initializing data disk") |
631 | 709 | .await; |
632 | | - info!("Setting up disk encryption"); |
633 | | - self.luks_setup(disk_crypt_key, name)?; |
634 | | - cmd! { |
635 | | - mkdir -p $mount_point; |
636 | | - zpool create -o autoexpand=on dstack $fs_dev; |
637 | | - zfs create -o mountpoint=$mount_point -o atime=off -o checksum=blake3 dstack/data; |
| 710 | + |
| 711 | + if opts.storage_encrypted { |
| 712 | + info!("Setting up disk encryption"); |
| 713 | + self.luks_setup(disk_crypt_key, name)?; |
| 714 | + } else { |
| 715 | + info!("Skipping disk encryption as requested by kernel cmdline"); |
| 716 | + } |
| 717 | + |
| 718 | + match opts.storage_fs { |
| 719 | + FsType::Zfs => { |
| 720 | + info!("Creating ZFS filesystem"); |
| 721 | + cmd! { |
| 722 | + zpool create -o autoexpand=on dstack $fs_dev; |
| 723 | + zfs create -o mountpoint=$mount_point -o atime=off -o checksum=blake3 dstack/data; |
| 724 | + } |
| 725 | + .context("Failed to create zpool")?; |
| 726 | + } |
| 727 | + FsType::Ext4 => { |
| 728 | + info!("Creating ext4 filesystem"); |
| 729 | + cmd! { |
| 730 | + mkfs.ext4 -F $fs_dev; |
| 731 | + mount $fs_dev $mount_point; |
| 732 | + } |
| 733 | + .context("Failed to create ext4 filesystem")?; |
| 734 | + } |
638 | 735 | } |
639 | | - .context("Failed to create zpool")?; |
640 | 736 | } else { |
641 | 737 | self.vmm |
642 | 738 | .notify_q("boot.progress", "mounting data disk") |
643 | 739 | .await; |
644 | | - info!("Mounting encrypted data disk"); |
645 | | - self.open_encrypted_volume(disk_crypt_key, name)?; |
646 | | - cmd! { |
647 | | - zpool import dstack; |
648 | | - zpool status dstack; |
649 | | - zpool online -e dstack $fs_dev; // triggers autoexpand |
| 740 | + |
| 741 | + if opts.storage_encrypted { |
| 742 | + info!("Mounting encrypted data disk"); |
| 743 | + self.open_encrypted_volume(disk_crypt_key, name)?; |
| 744 | + } else { |
| 745 | + info!("Mounting unencrypted data disk"); |
| 746 | + } |
| 747 | + |
| 748 | + match opts.storage_fs { |
| 749 | + FsType::Zfs => { |
| 750 | + cmd! { |
| 751 | + zpool import dstack; |
| 752 | + zpool status dstack; |
| 753 | + zpool online -e dstack $fs_dev; // triggers autoexpand |
| 754 | + } |
| 755 | + .context("Failed to import zpool")?; |
| 756 | + if cmd!(mountpoint -q $mount_point).is_err() { |
| 757 | + cmd!(zfs mount dstack/data).context("Failed to mount zpool")?; |
| 758 | + } |
| 759 | + } |
| 760 | + FsType::Ext4 => { |
| 761 | + Self::mount_e2fs(&fs_dev, mount_point) |
| 762 | + .context("Failed to mount ext4 filesystem")?; |
| 763 | + } |
| 764 | + } |
| 765 | + } |
| 766 | + Ok(()) |
| 767 | + } |
| 768 | + |
| 769 | + fn mount_e2fs(dev: &impl AsRef<Path>, mount_point: &impl AsRef<Path>) -> Result<()> { |
| 770 | + let dev = dev.as_ref(); |
| 771 | + let mount_point = mount_point.as_ref(); |
| 772 | + info!("Checking filesystem"); |
| 773 | + |
| 774 | + let e2fsck_status = Command::new("e2fsck") |
| 775 | + .arg("-f") |
| 776 | + .arg("-p") |
| 777 | + .arg(dev) |
| 778 | + .status() |
| 779 | + .with_context(|| format!("Failed to run e2fsck on {}", dev.display()))?; |
| 780 | + |
| 781 | + match e2fsck_status.code() { |
| 782 | + Some(0 | 1) => {} |
| 783 | + Some(code) => { |
| 784 | + bail!( |
| 785 | + "e2fsck exited with status {code} while checking {}", |
| 786 | + dev.display() |
| 787 | + ); |
650 | 788 | } |
651 | | - .context("Failed to import zpool")?; |
652 | | - if cmd!(mountpoint -q $mount_point).is_err() { |
653 | | - cmd!(zfs mount dstack/data).context("Failed to mount zpool")?; |
| 789 | + None => { |
| 790 | + bail!( |
| 791 | + "e2fsck terminated by signal while checking {}", |
| 792 | + dev.display() |
| 793 | + ); |
654 | 794 | } |
655 | 795 | } |
| 796 | + |
| 797 | + cmd! { |
| 798 | + info "Trying to resize filesystem if needed"; |
| 799 | + resize2fs $dev; |
| 800 | + info "Mounting filesystem"; |
| 801 | + mount $dev $mount_point; |
| 802 | + } |
| 803 | + .context("Failed to prepare ext4 filesystem")?; |
656 | 804 | Ok(()) |
657 | 805 | } |
658 | 806 |
|
@@ -802,9 +950,21 @@ impl<'a> Stage0<'a> { |
802 | 950 | let keys_json = serde_json::to_string(&app_keys).context("Failed to serialize app keys")?; |
803 | 951 | fs::write(self.app_keys_file(), keys_json).context("Failed to write app keys")?; |
804 | 952 |
|
| 953 | + // Parse kernel command line options |
| 954 | + let opts = parse_dstack_options(&self.shared).context("Failed to parse kernel cmdline")?; |
| 955 | + extend_rtmr3("storage-fs", opts.storage_fs.to_string().as_bytes())?; |
| 956 | + info!( |
| 957 | + "Filesystem options: encryption={}, filesystem={:?}", |
| 958 | + opts.storage_encrypted, opts.storage_fs |
| 959 | + ); |
| 960 | + |
805 | 961 | self.vmm.notify_q("boot.progress", "unsealing env").await; |
806 | | - self.mount_data_disk(is_initialized, &hex::encode(&app_keys.disk_crypt_key)) |
807 | | - .await?; |
| 962 | + self.mount_data_disk( |
| 963 | + is_initialized, |
| 964 | + &hex::encode(&app_keys.disk_crypt_key), |
| 965 | + &opts, |
| 966 | + ) |
| 967 | + .await?; |
808 | 968 | self.vmm |
809 | 969 | .notify_q( |
810 | 970 | "instance.info", |
|
0 commit comments