Files
woiken-sandboxes/src/vm.rs
T
niklas 2231e4efe1 fix(#5): write sandbox meta.json before creating the disk
Until meta existed the sandbox's image looked unreferenced, so a concurrent
`wsbx gc` deleted it mid-create; a crashed create was also invisible to ps and
refused by rm.
2026-10-22 19:53:03 +02:00

310 lines
14 KiB
Rust

//! Booting a sandbox: run script, vfkit invocation, guest results.
use crate::disk::create_disk;
use crate::exec;
use crate::kernel::kernel;
use crate::oci;
use crate::sandbox::lock_stopped;
use crate::stage::{STAGE_LIMIT, copy_tree};
use crate::tunnel::tunnel;
use crate::util::{base, need, now, sanitize, sq, strs};
use crate::Opts;
use anyhow::{Context, Result, anyhow, bail};
use serde_json::{Value, json};
use std::fs;
use std::io::{IsTerminal, Read};
use std::os::unix::fs::{DirBuilderExt, PermissionsExt};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
const INIT: &str = include_str!("init.sh");
const DHCP: &str = include_str!("dhcp.sh");
const GUEST: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/wsbx-guest"));
/// Agent-specific guest preparation.
#[derive(Default)]
pub struct Guest {
/// (host path, name under /.wsbx/host/stage) copied before each start, symlinks resolved
pub stage: Vec<(PathBuf, String)>,
/// Symlinks inside staged trees may only resolve into these (canonical) roots
pub roots: Vec<PathBuf>,
/// Shell run as root after HOME/env are set, before dropping to the image user
pub setup: String,
}
pub fn run(h: &Path, o: Opts, image: &str, cmd: Vec<String>, extra: &[String], g: Guest) -> Result<()> {
let vfkit = need("vfkit", "vfkit")?;
let kernel = match &o.kernel {
Some(k) => k.clone(),
None => kernel(h, false, None)?,
};
let ws = o.workspace.as_ref().map(fs::canonicalize).transpose().context("workspace")?;
vf(h)?; // everything vfkit gets lives under h, plus kernel + workspace: refuse bad paths before creating anything
vf(&kernel)?;
if let Some(ws) = &ws {
vf(ws)?;
let hc = fs::canonicalize(h)?;
if ws.starts_with(&hc) || hc.starts_with(ws) {
bail!("workspace {} overlaps the wsbx state dir {} (disks, kernel, control files)", ws.display(), hc.display());
}
}
let img = oci::pull(h, image, o.pull)?;
let short = image.rsplit('/').next().unwrap().split([':', '@']).next().unwrap();
let name = sanitize(&o.name.clone().unwrap_or_else(|| format!("{short}-{}", ws.as_deref().map(base).unwrap_or("vm".into()))));
let dir = h.join("sandboxes").join(&name);
fs::create_dir_all(&dir)?;
let _lock = lock_stopped(&dir).map_err(|_| anyhow!("sandbox {name} is already running"))?;
// The control share is writable by the guest (virtio-fs has no host-enforced read-only).
// Recreate it empty on every start -- while this sandbox's VM is stopped and we hold the lock --
// so nothing a previous guest left there (e.g. `run.sh -> ~/.zshrc`) is ever written through.
// Persistent state lives in meta.json, which the guest cannot reach.
let share = dir.join("share");
match fs::remove_dir_all(&share) {
Err(e) if e.kind() != std::io::ErrorKind::NotFound => return Err(e.into()),
_ => {}
}
std::fs::DirBuilder::new().mode(0o700).create(&share)?;
let disk = dir.join("disk.img");
let meta_path = dir.join("meta.json");
if o.fresh {
let _ = fs::remove_file(&disk);
}
let mut meta: Value = fs::read(&meta_path).ok().and_then(|m| serde_json::from_slice(&m).ok()).unwrap_or(Value::Null);
if disk.exists() && meta["digest"] != img.digest.as_str() {
eprintln!("wsbx: note: {name} was created from an older image; use --fresh to rebuild");
}
if !disk.exists() {
eprintln!("wsbx: creating sandbox {name}");
// meta first: it marks the image as in use for `wsbx gc` and makes a crashed create visible to ps/rm
meta = json!({"image": image, "digest": img.digest, "workspace": ws, "created": now(), "provisioned": false});
fs::write(&meta_path, serde_json::to_vec_pretty(&meta)?)?;
create_disk(h, &disk, o.disk)?;
}
// Build the guest-side run script (executed as root by init, then drops to the image user).
let c = &img.config;
let mut argv = strs(&c["Entrypoint"]);
argv.extend(if cmd.is_empty() { strs(&c["Cmd"]) } else { cmd });
argv.extend(extra.iter().cloned());
if argv.is_empty() {
argv.push("/bin/sh".into());
}
let user = c["User"].as_str().unwrap_or("").to_string();
let wd = ws
.as_ref()
.map(|p| p.to_string_lossy().into_owned())
.or_else(|| c["WorkingDir"].as_str().filter(|s| !s.is_empty()).map(String::from))
.unwrap_or("/".into());
let mut s = String::from("B=/.wsbx/busybox\n");
s += &format!("$B date -s @{} >/dev/null\n", now());
// Some images ship no /etc/hosts: without a localhost entry, Go and others resolve it via DNS and fail.
s += "$B grep -qw localhost /etc/hosts 2>/dev/null || printf '127.0.0.1 localhost\\n::1 localhost\\n' >> /etc/hosts\n";
s += &format!("$B hostname {name}; echo {name} > /etc/hostname; $B grep -q ' {name}$' /etc/hosts 2>/dev/null || echo '127.0.1.1 {name}' >> /etc/hosts\n");
if let Some((rows, cols)) = term_size() {
s += &format!("$B stty rows {rows} cols {cols}\n");
}
if let Some(ws) = &ws {
let w = sq(&ws.to_string_lossy());
s += &format!("$B mkdir -p {w} && $B mount -t virtiofs ws {w}\n");
}
let u = user.split(':').next().unwrap_or("");
s += &format!("export HOME=$($B awk -F: -v u={} '$1==u||$3==u{{print $6;exit}}' /etc/passwd)\n", sq(if u.is_empty() { "root" } else { u }));
s += "[ -n \"$HOME\" ] || export HOME=/root\n";
s += &format!("export TERM={} TINI_SUBREAPER=1\n", sq(&std::env::var("TERM").unwrap_or("xterm-256color".into())));
let mut env = strs(&c["Env"]);
for e in &o.env {
match e.split_once('=') {
Some(_) => env.push(e.clone()),
None => env.push(format!("{e}={}", std::env::var(e).with_context(|| format!("env {e} not set on host"))?)),
}
}
for e in env {
match export(&e) {
Some(line) => s += &line,
None => eprintln!("wsbx: skipping env entry with invalid name: {e:?}"),
}
}
s += &format!("export WSBX_USER={}\n", sq(&user));
let stage = share.join("stage");
fs::create_dir(&stage)?;
let (mut budget, mut seen) = (STAGE_LIMIT, std::collections::HashSet::new());
for (src, to) in &g.stage {
let dst = stage.join(to);
fs::create_dir_all(dst.parent().unwrap())?;
copy_tree(src, &dst, &g.roots, &mut budget, &mut seen)?;
}
s += &g.setup;
s += &format!("cd {} 2>/dev/null || cd /\n", sq(&wd));
// Guest helper (guest/*.go), started after the cd so exec sessions inherit run.sh's env and working dir.
// `wsbx exec`: host -> vfkit listening on exec.sock -> vsock -> helper (runs the command, pty or pipes)
// Host tunnel: helper listens on guest localhost:PORT -> vsock -> vfkit -> unix socket -> host localhost:PORT
fs::write(share.join("guest"), GUEST)?;
fs::set_permissions(share.join("guest"), fs::Permissions::from_mode(0o755))?;
let (mut vsock, mut args) = (Vec::new(), Vec::new());
let exec_sock = dir.join("exec.sock");
let _ = fs::remove_file(&exec_sock); // left over if vfkit was killed; vfkit can't listen on it then
if exec_sock.as_os_str().len() <= 100 {
// `connect`: vfkit listens on the unix socket and dials the guest (vfkit defaults to `listen`)
vsock.push(format!("virtio-vsock,port={},socketURL={},connect", exec::VPORT, vf(&exec_sock)?));
args.push(format!("-exec {}", exec::VPORT));
} else {
eprintln!("wsbx: note: `wsbx exec` unavailable, socket path too long (set WSBX_HOME to something shorter)");
}
let mut ports = o.host_ports.clone();
ports.sort();
ports.dedup();
for (i, &p) in ports.iter().enumerate() {
let vport = 10000 + i as u32;
let sock = dir.join(format!("p{p}.sock"));
tunnel(&sock, p)?;
vsock.push(format!("virtio-vsock,port={vport},socketURL={},listen", vf(&sock)?));
args.push(format!("{p}={vport}"));
}
if !ports.is_empty() {
eprintln!("wsbx: host localhost:{{{}}} tunneled into the VM", ports.iter().map(u16::to_string).collect::<Vec<_>>().join(","));
}
s += "$B mkfifo /run/wsbx-guest\n";
s += &format!("/.wsbx/host/guest {} >/run/wsbx-guest 2>/.wsbx/host/guest.log &\n", args.join(" "));
s += "read _ </run/wsbx-guest\n";
let argv: Vec<String> = argv.iter().map(|a| sq(a)).collect();
if u.is_empty() || u == "root" || u == "0" {
s += &format!("exec {}\n", argv.join(" "));
} else {
s += &format!("exec $B chpst -u {} {}\n", sq(&user), argv.join(" "));
}
fs::write(share.join("run.sh"), s)?;
for (f, body) in [("init", INIT), ("dhcp", DHCP)] {
fs::write(share.join(f), body)?;
fs::set_permissions(share.join(f), fs::Permissions::from_mode(0o755))?;
}
let _ = fs::remove_file(share.join("exit"));
let initrd = h.join("empty.initrd"); // vfkit insists on an initrd; an empty one is fine
if !initrd.exists() {
fs::write(&initrd, "")?;
}
let mac = {
let x = name.bytes().fold(0xcbf29ce484222325u64, |a, b| (a ^ b as u64).wrapping_mul(0x100000001b3));
format!("02:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", x as u8, (x >> 8) as u8, (x >> 16) as u8, (x >> 24) as u8, (x >> 32) as u8)
};
// panic=1 reboots, which VZ turns into a restart; init powers off on any second boot (see init.sh).
let cmdline = "console=hvc0 root=/dev/vda rw rootfstype=ext4 init=/.wsbx/init quiet panic=1";
let tty = std::io::stdin().is_terminal();
let mut v = Command::new(vfkit);
v.args(["--cpus", &o.cpus.to_string(), "--memory", &o.memory.to_string(), "--log-level", "error"])
.arg("--bootloader")
.arg(format!("linux,kernel={},initrd={},cmdline=\"{cmdline}\"", vf(&kernel)?, vf(&initrd)?))
.arg("--device")
.arg(format!("virtio-blk,path={}", vf(&disk)?))
.arg("--device")
.arg(format!("virtio-net,nat,mac={mac}"))
.arg("--device")
.arg(format!("virtio-fs,sharedDir={},mountTag=wsbx", vf(&share)?))
.args(["--device", "virtio-rng", "--device"])
// vfkit's stdio console needs a TTY; otherwise stream output only (no stdin)
.arg(if tty { "virtio-serial,stdio".into() } else { format!("virtio-serial,logFilePath={}", vf(&out_path())?) })
.stderr(fs::File::create(dir.join("vfkit.log"))?);
if let Some(ws) = &ws {
v.arg("--device").arg(format!("virtio-fs,sharedDir={},mountTag=ws", vf(ws)?));
}
// Image tar is only needed until the guest has unpacked it; read-only so sandboxes can share it.
let provisioned = meta["provisioned"].as_bool().unwrap_or(false);
if !provisioned {
v.arg("--device").arg(format!("virtio-blk,path={},readonly", vf(&img.rootfs)?));
}
for d in vsock {
v.arg("--device").arg(d);
}
let saved = stty(&["-g"]);
if saved.is_some() {
stty(&["raw", "-echo"]);
}
let status = v.stdin(Stdio::inherit()).status();
if let Some(s) = saved {
stty(&[s.trim()]);
}
let status = status?;
// The VM is gone now, so the guest can't race these reads; still never follow its symlinks.
let code = guest_output(&share, "exit").and_then(|s| s.trim().parse().ok());
if !provisioned && guest_output(&share, "provisioned").is_some() && meta.is_object() {
meta["provisioned"] = json!(true);
fs::write(&meta_path, serde_json::to_vec_pretty(&meta)?)?;
}
let _ = fs::remove_dir_all(&share); // run.sh contains forwarded env
match code {
Some(c) => std::process::exit(c),
None if !status.success() => bail!("vm failed, see {}", dir.join("vfkit.log").display()),
None => bail!("VM stopped without an exit status (kernel panic, reboot or poweroff inside the VM)"),
}
}
/// `export K='v'` line; None unless K is a shell identifier (image Env is hostile and run.sh runs as root).
fn export(e: &str) -> Option<String> {
let (k, v) = e.split_once('=')?;
let ok = k.bytes().next().is_some_and(|b| b.is_ascii_alphabetic() || b == b'_') && k.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_');
ok.then(|| format!("export {k}={}\n", sq(v)))
}
/// Small result file written by the guest; regular files only (no symlinks), size-capped.
fn guest_output(share: &Path, name: &str) -> Option<String> {
let p = share.join(name);
fs::symlink_metadata(&p).ok().filter(|m| m.is_file())?;
let mut s = String::new();
fs::File::open(p).ok()?.take(64).read_to_string(&mut s).ok()?;
Some(s)
}
/// vfkit splits device options on ',' and the bootloader cmdline is quoted: a path containing
/// those (e.g. a workspace named `repo,sharedDir=/Users/me`) would inject options. Refuse them.
fn vf(p: &Path) -> Result<String> {
let s = p.to_str().ok_or_else(|| anyhow!("non-UTF-8 path {}", p.display()))?;
if s.chars().any(|c| c == ',' || c == '"' || c.is_control()) {
bail!("unsupported character in path {s:?} (',', '\"' and control characters would inject vfkit options)");
}
Ok(s.to_string())
}
/// Where our stdout points (vfkit only accepts a path for non-TTY console output).
fn out_path() -> PathBuf {
fs::read_link("/dev/fd/1").unwrap_or("/dev/stdout".into())
}
pub fn stty(args: &[&str]) -> Option<String> {
let out = Command::new("stty").args(args).stdin(Stdio::inherit()).stderr(Stdio::null()).output().ok()?;
out.status.success().then(|| String::from_utf8_lossy(&out.stdout).into_owned())
}
/// Rows and columns of the terminal on stdin.
pub fn term_size() -> Option<(u16, u16)> {
// SAFETY: TIOCGWINSZ only writes a winsize struct.
let mut ws: libc::winsize = unsafe { std::mem::zeroed() };
let ok = unsafe { libc::ioctl(0, libc::TIOCGWINSZ, &mut ws) } == 0;
(ok && ws.ws_row > 0 && ws.ws_col > 0).then_some((ws.ws_row, ws.ws_col))
}
#[cfg(test)]
mod tests {
use std::path::Path;
#[test]
fn env_keys() {
assert_eq!(super::export("_A1=x y").as_deref(), Some("export _A1='x y'\n"));
for bad in ["PWN$(id > /ws/pwn)=x", "=x", "1A=x", "A B=x", "A;id=x", "noequals"] {
assert!(super::export(bad).is_none(), "{bad}");
}
}
#[test]
fn vfkit_injection_rejected() {
assert!(super::vf(Path::new("/Users/me/repo")).is_ok());
for bad in ["/shared/repo/,sharedDir=/Users/me", "/a\"b", "/a\nb"] {
assert!(super::vf(Path::new(bad)).is_err(), "{bad}");
}
}
}