//! Booting a sandbox: run script, vfkit invocation, guest results. use crate::disk::create_disk; use crate::exec; use crate::kernel::kernel; use crate::oci; use crate::sandbox::lock_stopped; use crate::stage::{STAGE_LIMIT, copy_tree}; use crate::tunnel::tunnel; use crate::util::{base, need, now, sanitize, sq, strs}; use crate::Opts; use anyhow::{Context, Result, anyhow, bail}; use serde_json::{Value, json}; use std::fs; use std::io::{IsTerminal, Read}; use std::os::unix::fs::{DirBuilderExt, PermissionsExt}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; const INIT: &str = include_str!("init.sh"); const DHCP: &str = include_str!("dhcp.sh"); const GUEST: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/wsbx-guest")); /// Agent-specific guest preparation. #[derive(Default)] pub struct Guest { /// (host path, name under /.wsbx/host/stage) copied before each start, symlinks resolved pub stage: Vec<(PathBuf, String)>, /// Symlinks inside staged trees may only resolve into these (canonical) roots pub roots: Vec, /// Shell run as root after HOME/env are set, before dropping to the image user pub setup: String, } pub fn run(h: &Path, o: Opts, image: &str, cmd: Vec, extra: &[String], g: Guest) -> Result<()> { let vfkit = need("vfkit", "vfkit")?; let kernel = match &o.kernel { Some(k) => k.clone(), None => kernel(h, false, None)?, }; let ws = o.workspace.as_ref().map(fs::canonicalize).transpose().context("workspace")?; vf(h)?; // everything vfkit gets lives under h, plus kernel + workspace: refuse bad paths before creating anything vf(&kernel)?; if let Some(ws) = &ws { vf(ws)?; let hc = fs::canonicalize(h)?; if ws.starts_with(&hc) || hc.starts_with(ws) { bail!("workspace {} overlaps the wsbx state dir {} (disks, kernel, control files)", ws.display(), hc.display()); } } let img = oci::pull(h, image, o.pull)?; let short = image.rsplit('/').next().unwrap().split([':', '@']).next().unwrap(); let name = sanitize(&o.name.clone().unwrap_or_else(|| format!("{short}-{}", ws.as_deref().map(base).unwrap_or("vm".into())))); let dir = h.join("sandboxes").join(&name); fs::create_dir_all(&dir)?; let _lock = lock_stopped(&dir).map_err(|_| anyhow!("sandbox {name} is already running"))?; // The control share is writable by the guest (virtio-fs has no host-enforced read-only). // Recreate it empty on every start -- while this sandbox's VM is stopped and we hold the lock -- // so nothing a previous guest left there (e.g. `run.sh -> ~/.zshrc`) is ever written through. // Persistent state lives in meta.json, which the guest cannot reach. let share = dir.join("share"); match fs::remove_dir_all(&share) { Err(e) if e.kind() != std::io::ErrorKind::NotFound => return Err(e.into()), _ => {} } std::fs::DirBuilder::new().mode(0o700).create(&share)?; let disk = dir.join("disk.img"); let meta_path = dir.join("meta.json"); if o.fresh { let _ = fs::remove_file(&disk); } let mut meta: Value = fs::read(&meta_path).ok().and_then(|m| serde_json::from_slice(&m).ok()).unwrap_or(Value::Null); if disk.exists() && meta["digest"] != img.digest.as_str() { eprintln!("wsbx: note: {name} was created from an older image; use --fresh to rebuild"); } if !disk.exists() { eprintln!("wsbx: creating sandbox {name}"); // meta first: it marks the image as in use for `wsbx gc` and makes a crashed create visible to ps/rm meta = json!({"image": image, "digest": img.digest, "workspace": ws, "created": now(), "provisioned": false}); fs::write(&meta_path, serde_json::to_vec_pretty(&meta)?)?; create_disk(h, &disk, o.disk)?; } // Build the guest-side run script (executed as root by init, then drops to the image user). let c = &img.config; let mut argv = strs(&c["Entrypoint"]); argv.extend(if cmd.is_empty() { strs(&c["Cmd"]) } else { cmd }); argv.extend(extra.iter().cloned()); if argv.is_empty() { argv.push("/bin/sh".into()); } let user = c["User"].as_str().unwrap_or("").to_string(); let wd = ws .as_ref() .map(|p| p.to_string_lossy().into_owned()) .or_else(|| c["WorkingDir"].as_str().filter(|s| !s.is_empty()).map(String::from)) .unwrap_or("/".into()); let mut s = String::from("B=/.wsbx/busybox\n"); s += &format!("$B date -s @{} >/dev/null\n", now()); // Some images ship no /etc/hosts: without a localhost entry, Go and others resolve it via DNS and fail. s += "$B grep -qw localhost /etc/hosts 2>/dev/null || printf '127.0.0.1 localhost\\n::1 localhost\\n' >> /etc/hosts\n"; s += &format!("$B hostname {name}; echo {name} > /etc/hostname; $B grep -q ' {name}$' /etc/hosts 2>/dev/null || echo '127.0.1.1 {name}' >> /etc/hosts\n"); if let Some((rows, cols)) = term_size() { s += &format!("$B stty rows {rows} cols {cols}\n"); } if let Some(ws) = &ws { let w = sq(&ws.to_string_lossy()); s += &format!("$B mkdir -p {w} && $B mount -t virtiofs ws {w}\n"); } let u = user.split(':').next().unwrap_or(""); s += &format!("export HOME=$($B awk -F: -v u={} '$1==u||$3==u{{print $6;exit}}' /etc/passwd)\n", sq(if u.is_empty() { "root" } else { u })); s += "[ -n \"$HOME\" ] || export HOME=/root\n"; s += &format!("export TERM={} TINI_SUBREAPER=1\n", sq(&std::env::var("TERM").unwrap_or("xterm-256color".into()))); let mut env = strs(&c["Env"]); for e in &o.env { match e.split_once('=') { Some(_) => env.push(e.clone()), None => env.push(format!("{e}={}", std::env::var(e).with_context(|| format!("env {e} not set on host"))?)), } } for e in env { match export(&e) { Some(line) => s += &line, None => eprintln!("wsbx: skipping env entry with invalid name: {e:?}"), } } s += &format!("export WSBX_USER={}\n", sq(&user)); let stage = share.join("stage"); fs::create_dir(&stage)?; let (mut budget, mut seen) = (STAGE_LIMIT, std::collections::HashSet::new()); for (src, to) in &g.stage { let dst = stage.join(to); fs::create_dir_all(dst.parent().unwrap())?; copy_tree(src, &dst, &g.roots, &mut budget, &mut seen)?; } s += &g.setup; s += &format!("cd {} 2>/dev/null || cd /\n", sq(&wd)); // Guest helper (guest/*.go), started after the cd so exec sessions inherit run.sh's env and working dir. // `wsbx exec`: host -> vfkit listening on exec.sock -> vsock -> helper (runs the command, pty or pipes) // Host tunnel: helper listens on guest localhost:PORT -> vsock -> vfkit -> unix socket -> host localhost:PORT fs::write(share.join("guest"), GUEST)?; fs::set_permissions(share.join("guest"), fs::Permissions::from_mode(0o755))?; let (mut vsock, mut args) = (Vec::new(), Vec::new()); let exec_sock = dir.join("exec.sock"); let _ = fs::remove_file(&exec_sock); // left over if vfkit was killed; vfkit can't listen on it then if exec_sock.as_os_str().len() <= 100 { // `connect`: vfkit listens on the unix socket and dials the guest (vfkit defaults to `listen`) vsock.push(format!("virtio-vsock,port={},socketURL={},connect", exec::VPORT, vf(&exec_sock)?)); args.push(format!("-exec {}", exec::VPORT)); } else { eprintln!("wsbx: note: `wsbx exec` unavailable, socket path too long (set WSBX_HOME to something shorter)"); } let mut ports = o.host_ports.clone(); ports.sort(); ports.dedup(); for (i, &p) in ports.iter().enumerate() { let vport = 10000 + i as u32; let sock = dir.join(format!("p{p}.sock")); tunnel(&sock, p)?; vsock.push(format!("virtio-vsock,port={vport},socketURL={},listen", vf(&sock)?)); args.push(format!("{p}={vport}")); } if !ports.is_empty() { eprintln!("wsbx: host localhost:{{{}}} tunneled into the VM", ports.iter().map(u16::to_string).collect::>().join(",")); } s += "$B mkfifo /run/wsbx-guest\n"; s += &format!("/.wsbx/host/guest {} >/run/wsbx-guest 2>/.wsbx/host/guest.log &\n", args.join(" ")); s += "read _ = argv.iter().map(|a| sq(a)).collect(); if u.is_empty() || u == "root" || u == "0" { s += &format!("exec {}\n", argv.join(" ")); } else { s += &format!("exec $B chpst -u {} {}\n", sq(&user), argv.join(" ")); } fs::write(share.join("run.sh"), s)?; for (f, body) in [("init", INIT), ("dhcp", DHCP)] { fs::write(share.join(f), body)?; fs::set_permissions(share.join(f), fs::Permissions::from_mode(0o755))?; } let _ = fs::remove_file(share.join("exit")); let initrd = h.join("empty.initrd"); // vfkit insists on an initrd; an empty one is fine if !initrd.exists() { fs::write(&initrd, "")?; } let mac = { let x = name.bytes().fold(0xcbf29ce484222325u64, |a, b| (a ^ b as u64).wrapping_mul(0x100000001b3)); format!("02:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", x as u8, (x >> 8) as u8, (x >> 16) as u8, (x >> 24) as u8, (x >> 32) as u8) }; // panic=1 reboots, which VZ turns into a restart; init powers off on any second boot (see init.sh). let cmdline = "console=hvc0 root=/dev/vda rw rootfstype=ext4 init=/.wsbx/init quiet panic=1"; let tty = std::io::stdin().is_terminal(); let mut v = Command::new(vfkit); v.args(["--cpus", &o.cpus.to_string(), "--memory", &o.memory.to_string(), "--log-level", "error"]) .arg("--bootloader") .arg(format!("linux,kernel={},initrd={},cmdline=\"{cmdline}\"", vf(&kernel)?, vf(&initrd)?)) .arg("--device") .arg(format!("virtio-blk,path={}", vf(&disk)?)) .arg("--device") .arg(format!("virtio-net,nat,mac={mac}")) .arg("--device") .arg(format!("virtio-fs,sharedDir={},mountTag=wsbx", vf(&share)?)) .args(["--device", "virtio-rng", "--device"]) // vfkit's stdio console needs a TTY; otherwise stream output only (no stdin) .arg(if tty { "virtio-serial,stdio".into() } else { format!("virtio-serial,logFilePath={}", vf(&out_path())?) }) .stderr(fs::File::create(dir.join("vfkit.log"))?); if let Some(ws) = &ws { v.arg("--device").arg(format!("virtio-fs,sharedDir={},mountTag=ws", vf(ws)?)); } // Image tar is only needed until the guest has unpacked it; read-only so sandboxes can share it. let provisioned = meta["provisioned"].as_bool().unwrap_or(false); if !provisioned { v.arg("--device").arg(format!("virtio-blk,path={},readonly", vf(&img.rootfs)?)); } for d in vsock { v.arg("--device").arg(d); } let saved = stty(&["-g"]); if saved.is_some() { stty(&["raw", "-echo"]); } let status = v.stdin(Stdio::inherit()).status(); if let Some(s) = saved { stty(&[s.trim()]); } let status = status?; // The VM is gone now, so the guest can't race these reads; still never follow its symlinks. let code = guest_output(&share, "exit").and_then(|s| s.trim().parse().ok()); if !provisioned && guest_output(&share, "provisioned").is_some() && meta.is_object() { meta["provisioned"] = json!(true); fs::write(&meta_path, serde_json::to_vec_pretty(&meta)?)?; } let _ = fs::remove_dir_all(&share); // run.sh contains forwarded env match code { Some(c) => std::process::exit(c), None if !status.success() => bail!("vm failed, see {}", dir.join("vfkit.log").display()), None => bail!("VM stopped without an exit status (kernel panic, reboot or poweroff inside the VM)"), } } /// `export K='v'` line; None unless K is a shell identifier (image Env is hostile and run.sh runs as root). fn export(e: &str) -> Option { let (k, v) = e.split_once('=')?; let ok = k.bytes().next().is_some_and(|b| b.is_ascii_alphabetic() || b == b'_') && k.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_'); ok.then(|| format!("export {k}={}\n", sq(v))) } /// Small result file written by the guest; regular files only (no symlinks), size-capped. fn guest_output(share: &Path, name: &str) -> Option { let p = share.join(name); fs::symlink_metadata(&p).ok().filter(|m| m.is_file())?; let mut s = String::new(); fs::File::open(p).ok()?.take(64).read_to_string(&mut s).ok()?; Some(s) } /// vfkit splits device options on ',' and the bootloader cmdline is quoted: a path containing /// those (e.g. a workspace named `repo,sharedDir=/Users/me`) would inject options. Refuse them. fn vf(p: &Path) -> Result { let s = p.to_str().ok_or_else(|| anyhow!("non-UTF-8 path {}", p.display()))?; if s.chars().any(|c| c == ',' || c == '"' || c.is_control()) { bail!("unsupported character in path {s:?} (',', '\"' and control characters would inject vfkit options)"); } Ok(s.to_string()) } /// Where our stdout points (vfkit only accepts a path for non-TTY console output). fn out_path() -> PathBuf { fs::read_link("/dev/fd/1").unwrap_or("/dev/stdout".into()) } pub fn stty(args: &[&str]) -> Option { let out = Command::new("stty").args(args).stdin(Stdio::inherit()).stderr(Stdio::null()).output().ok()?; out.status.success().then(|| String::from_utf8_lossy(&out.stdout).into_owned()) } /// Rows and columns of the terminal on stdin. pub fn term_size() -> Option<(u16, u16)> { // SAFETY: TIOCGWINSZ only writes a winsize struct. let mut ws: libc::winsize = unsafe { std::mem::zeroed() }; let ok = unsafe { libc::ioctl(0, libc::TIOCGWINSZ, &mut ws) } == 0; (ok && ws.ws_row > 0 && ws.ws_col > 0).then_some((ws.ws_row, ws.ws_col)) } #[cfg(test)] mod tests { use std::path::Path; #[test] fn env_keys() { assert_eq!(super::export("_A1=x y").as_deref(), Some("export _A1='x y'\n")); for bad in ["PWN$(id > /ws/pwn)=x", "=x", "1A=x", "A B=x", "A;id=x", "noequals"] { assert!(super::export(bad).is_none(), "{bad}"); } } #[test] fn vfkit_injection_rejected() { assert!(super::vf(Path::new("/Users/me/repo")).is_ok()); for bad in ["/shared/repo/,sharedDir=/Users/me", "/a\"b", "/a\nb"] { assert!(super::vf(Path::new(bad)).is_err(), "{bad}"); } } }