279 lines
12 KiB
Rust
279 lines
12 KiB
Rust
//! Booting a sandbox: run script, vfkit invocation, guest results.
|
|
use crate::disk::create_disk;
|
|
use crate::kernel::kernel;
|
|
use crate::oci;
|
|
use crate::sandbox::lock_stopped;
|
|
use crate::stage::{STAGE_LIMIT, copy_tree};
|
|
use crate::tunnel::tunnel;
|
|
use crate::util::{base, need, now, sanitize, sq, strs};
|
|
use crate::Opts;
|
|
use anyhow::{Context, Result, anyhow, bail};
|
|
use serde_json::{Value, json};
|
|
use std::fs;
|
|
use std::io::{IsTerminal, Read};
|
|
use std::os::unix::fs::{DirBuilderExt, PermissionsExt};
|
|
use std::path::{Path, PathBuf};
|
|
use std::process::{Command, Stdio};
|
|
|
|
const INIT: &str = include_str!("init.sh");
|
|
const DHCP: &str = include_str!("dhcp.sh");
|
|
const FWD: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/wsbx-fwd"));
|
|
|
|
/// Agent-specific guest preparation.
|
|
#[derive(Default)]
|
|
pub struct Guest {
|
|
/// (host path, name under /.wsbx/host/stage) copied before each start, symlinks resolved
|
|
pub stage: Vec<(PathBuf, String)>,
|
|
/// Symlinks inside staged trees may only resolve into these (canonical) roots
|
|
pub roots: Vec<PathBuf>,
|
|
/// Shell run as root after HOME/env are set, before dropping to the image user
|
|
pub setup: String,
|
|
}
|
|
|
|
pub fn run(h: &Path, o: Opts, image: &str, cmd: Vec<String>, extra: &[String], g: Guest) -> Result<()> {
|
|
let vfkit = need("vfkit", "vfkit")?;
|
|
let kernel = match &o.kernel {
|
|
Some(k) => k.clone(),
|
|
None => kernel(h, false, None)?,
|
|
};
|
|
let ws = o.workspace.as_ref().map(fs::canonicalize).transpose().context("workspace")?;
|
|
vf(h)?; // everything vfkit gets lives under h, plus kernel + workspace: refuse bad paths before creating anything
|
|
vf(&kernel)?;
|
|
if let Some(ws) = &ws {
|
|
vf(ws)?;
|
|
let hc = fs::canonicalize(h)?;
|
|
if ws.starts_with(&hc) || hc.starts_with(ws) {
|
|
bail!("workspace {} overlaps the wsbx state dir {} (disks, kernel, control files)", ws.display(), hc.display());
|
|
}
|
|
}
|
|
let img = oci::pull(h, image, o.pull)?;
|
|
let short = image.rsplit('/').next().unwrap().split([':', '@']).next().unwrap();
|
|
let name = sanitize(&o.name.clone().unwrap_or_else(|| format!("{short}-{}", ws.as_deref().map(base).unwrap_or("vm".into()))));
|
|
let dir = h.join("sandboxes").join(&name);
|
|
fs::create_dir_all(&dir)?;
|
|
let _lock = lock_stopped(&dir).map_err(|_| anyhow!("sandbox {name} is already running"))?;
|
|
|
|
// The control share is writable by the guest (virtio-fs has no host-enforced read-only).
|
|
// Recreate it empty on every start -- while this sandbox's VM is stopped and we hold the lock --
|
|
// so nothing a previous guest left there (e.g. `run.sh -> ~/.zshrc`) is ever written through.
|
|
// Persistent state lives in meta.json, which the guest cannot reach.
|
|
let share = dir.join("share");
|
|
match fs::remove_dir_all(&share) {
|
|
Err(e) if e.kind() != std::io::ErrorKind::NotFound => return Err(e.into()),
|
|
_ => {}
|
|
}
|
|
std::fs::DirBuilder::new().mode(0o700).create(&share)?;
|
|
|
|
let disk = dir.join("disk.img");
|
|
let meta_path = dir.join("meta.json");
|
|
if o.fresh {
|
|
let _ = fs::remove_file(&disk);
|
|
}
|
|
let mut meta: Value = fs::read(&meta_path).ok().and_then(|m| serde_json::from_slice(&m).ok()).unwrap_or(Value::Null);
|
|
if disk.exists() && meta["digest"] != img.digest.as_str() {
|
|
eprintln!("wsbx: note: {name} was created from an older image; use --fresh to rebuild");
|
|
}
|
|
if !disk.exists() {
|
|
eprintln!("wsbx: creating sandbox {name}");
|
|
create_disk(h, &disk, o.disk)?;
|
|
meta = json!({"image": image, "digest": img.digest, "workspace": ws, "created": now(), "provisioned": false});
|
|
fs::write(&meta_path, serde_json::to_vec_pretty(&meta)?)?;
|
|
}
|
|
|
|
// Build the guest-side run script (executed as root by init, then drops to the image user).
|
|
let c = &img.config;
|
|
let mut argv = strs(&c["Entrypoint"]);
|
|
argv.extend(if cmd.is_empty() { strs(&c["Cmd"]) } else { cmd });
|
|
argv.extend(extra.iter().cloned());
|
|
if argv.is_empty() {
|
|
argv.push("/bin/sh".into());
|
|
}
|
|
let user = c["User"].as_str().unwrap_or("").to_string();
|
|
let wd = ws
|
|
.as_ref()
|
|
.map(|p| p.to_string_lossy().into_owned())
|
|
.or_else(|| c["WorkingDir"].as_str().filter(|s| !s.is_empty()).map(String::from))
|
|
.unwrap_or("/".into());
|
|
let mut s = String::from("B=/.wsbx/busybox\n");
|
|
s += &format!("$B date -s @{} >/dev/null\n", now());
|
|
s += &format!("$B hostname {name}; echo {name} > /etc/hostname; $B grep -q ' {name}$' /etc/hosts 2>/dev/null || echo '127.0.1.1 {name}' >> /etc/hosts\n");
|
|
if let Some((rows @ 1.., cols @ 1..)) = term_size() {
|
|
s += &format!("$B stty rows {rows} cols {cols}\n");
|
|
}
|
|
if let Some(ws) = &ws {
|
|
let w = sq(&ws.to_string_lossy());
|
|
s += &format!("$B mkdir -p {w} && $B mount -t virtiofs ws {w}\n");
|
|
}
|
|
let u = user.split(':').next().unwrap_or("");
|
|
s += &format!("export HOME=$($B awk -F: -v u={} '$1==u||$3==u{{print $6;exit}}' /etc/passwd)\n", sq(if u.is_empty() { "root" } else { u }));
|
|
s += "[ -n \"$HOME\" ] || export HOME=/root\n";
|
|
s += &format!("export TERM={} TINI_SUBREAPER=1\n", sq(&std::env::var("TERM").unwrap_or("xterm-256color".into())));
|
|
let mut env = strs(&c["Env"]);
|
|
for e in &o.env {
|
|
match e.split_once('=') {
|
|
Some(_) => env.push(e.clone()),
|
|
None => env.push(format!("{e}={}", std::env::var(e).with_context(|| format!("env {e} not set on host"))?)),
|
|
}
|
|
}
|
|
for e in env {
|
|
if let Some((k, v)) = e.split_once('=') {
|
|
s += &format!("export {k}={}\n", sq(v));
|
|
}
|
|
}
|
|
s += &format!("export WSBX_USER={}\n", sq(&user));
|
|
|
|
let stage = share.join("stage");
|
|
fs::create_dir(&stage)?;
|
|
let (mut budget, mut seen) = (STAGE_LIMIT, std::collections::HashSet::new());
|
|
for (src, to) in &g.stage {
|
|
let dst = stage.join(to);
|
|
fs::create_dir_all(dst.parent().unwrap())?;
|
|
copy_tree(src, &dst, &g.roots, &mut budget, &mut seen)?;
|
|
}
|
|
s += &g.setup;
|
|
|
|
// Host tunnel: guest wsbx-fwd listens on localhost:PORT -> vsock -> vfkit -> unix socket -> host localhost:PORT
|
|
let mut ports = o.host_ports.clone();
|
|
ports.sort();
|
|
ports.dedup();
|
|
let mut vsock = Vec::new();
|
|
if !ports.is_empty() {
|
|
let fwd = share.join("fwd");
|
|
fs::write(&fwd, FWD)?;
|
|
fs::set_permissions(&fwd, fs::Permissions::from_mode(0o755))?;
|
|
let mut specs = Vec::new();
|
|
for (i, &p) in ports.iter().enumerate() {
|
|
let vport = 10000 + i as u32;
|
|
let sock = dir.join(format!("p{p}.sock"));
|
|
tunnel(&sock, p)?;
|
|
vsock.push(format!("virtio-vsock,port={vport},socketURL={},listen", vf(&sock)?));
|
|
specs.push(format!("{p}={vport}"));
|
|
}
|
|
s += "$B mkfifo /run/wsbx-fwd\n";
|
|
s += &format!("/.wsbx/host/fwd {} >/run/wsbx-fwd 2>/.wsbx/host/fwd.log &\n", specs.join(" "));
|
|
s += "read _ </run/wsbx-fwd\n";
|
|
eprintln!("wsbx: host localhost:{{{}}} tunneled into the VM", ports.iter().map(u16::to_string).collect::<Vec<_>>().join(","));
|
|
}
|
|
|
|
s += &format!("cd {} 2>/dev/null || cd /\n", sq(&wd));
|
|
let argv: Vec<String> = argv.iter().map(|a| sq(a)).collect();
|
|
if u.is_empty() || u == "root" || u == "0" {
|
|
s += &format!("exec {}\n", argv.join(" "));
|
|
} else {
|
|
s += &format!("exec $B chpst -u {} {}\n", sq(&user), argv.join(" "));
|
|
}
|
|
fs::write(share.join("run.sh"), s)?;
|
|
for (f, body) in [("init", INIT), ("dhcp", DHCP)] {
|
|
fs::write(share.join(f), body)?;
|
|
fs::set_permissions(share.join(f), fs::Permissions::from_mode(0o755))?;
|
|
}
|
|
let _ = fs::remove_file(share.join("exit"));
|
|
|
|
let initrd = h.join("empty.initrd"); // vfkit insists on an initrd; an empty one is fine
|
|
if !initrd.exists() {
|
|
fs::write(&initrd, "")?;
|
|
}
|
|
let mac = {
|
|
let x = name.bytes().fold(0xcbf29ce484222325u64, |a, b| (a ^ b as u64).wrapping_mul(0x100000001b3));
|
|
format!("02:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", x as u8, (x >> 8) as u8, (x >> 16) as u8, (x >> 24) as u8, (x >> 32) as u8)
|
|
};
|
|
// panic=1 reboots, which VZ turns into a restart; init powers off on any second boot (see init.sh).
|
|
let cmdline = "console=hvc0 root=/dev/vda rw rootfstype=ext4 init=/.wsbx/init quiet panic=1";
|
|
let tty = std::io::stdin().is_terminal();
|
|
let mut v = Command::new(vfkit);
|
|
v.args(["--cpus", &o.cpus.to_string(), "--memory", &o.memory.to_string(), "--log-level", "error"])
|
|
.arg("--bootloader")
|
|
.arg(format!("linux,kernel={},initrd={},cmdline=\"{cmdline}\"", vf(&kernel)?, vf(&initrd)?))
|
|
.arg("--device")
|
|
.arg(format!("virtio-blk,path={}", vf(&disk)?))
|
|
.arg("--device")
|
|
.arg(format!("virtio-net,nat,mac={mac}"))
|
|
.arg("--device")
|
|
.arg(format!("virtio-fs,sharedDir={},mountTag=wsbx", vf(&share)?))
|
|
.args(["--device", "virtio-rng", "--device"])
|
|
// vfkit's stdio console needs a TTY; otherwise stream output only (no stdin)
|
|
.arg(if tty { "virtio-serial,stdio".into() } else { format!("virtio-serial,logFilePath={}", vf(&out_path())?) })
|
|
.stderr(fs::File::create(dir.join("vfkit.log"))?);
|
|
if let Some(ws) = &ws {
|
|
v.arg("--device").arg(format!("virtio-fs,sharedDir={},mountTag=ws", vf(ws)?));
|
|
}
|
|
// Image tar is only needed until the guest has unpacked it; read-only so sandboxes can share it.
|
|
let provisioned = meta["provisioned"].as_bool().unwrap_or(false);
|
|
if !provisioned {
|
|
v.arg("--device").arg(format!("virtio-blk,path={},readonly", vf(&img.rootfs)?));
|
|
}
|
|
for d in vsock {
|
|
v.arg("--device").arg(d);
|
|
}
|
|
|
|
let saved = stty(&["-g"]);
|
|
if saved.is_some() {
|
|
stty(&["raw", "-echo"]);
|
|
}
|
|
let status = v.stdin(Stdio::inherit()).status();
|
|
if let Some(s) = saved {
|
|
stty(&[s.trim()]);
|
|
}
|
|
let status = status?;
|
|
// The VM is gone now, so the guest can't race these reads; still never follow its symlinks.
|
|
let code = guest_output(&share, "exit").and_then(|s| s.trim().parse().ok());
|
|
if !provisioned && guest_output(&share, "provisioned").is_some() && meta.is_object() {
|
|
meta["provisioned"] = json!(true);
|
|
fs::write(&meta_path, serde_json::to_vec_pretty(&meta)?)?;
|
|
}
|
|
let _ = fs::remove_dir_all(&share); // run.sh contains forwarded env
|
|
match code {
|
|
Some(c) => std::process::exit(c),
|
|
None if !status.success() => bail!("vm failed, see {}", dir.join("vfkit.log").display()),
|
|
None => bail!("VM stopped without an exit status (kernel panic, reboot or poweroff inside the VM)"),
|
|
}
|
|
}
|
|
|
|
/// Small result file written by the guest; regular files only (no symlinks), size-capped.
|
|
fn guest_output(share: &Path, name: &str) -> Option<String> {
|
|
let p = share.join(name);
|
|
fs::symlink_metadata(&p).ok().filter(|m| m.is_file())?;
|
|
let mut s = String::new();
|
|
fs::File::open(p).ok()?.take(64).read_to_string(&mut s).ok()?;
|
|
Some(s)
|
|
}
|
|
|
|
/// vfkit splits device options on ',' and the bootloader cmdline is quoted: a path containing
|
|
/// those (e.g. a workspace named `repo,sharedDir=/Users/me`) would inject options. Refuse them.
|
|
fn vf(p: &Path) -> Result<String> {
|
|
let s = p.to_str().ok_or_else(|| anyhow!("non-UTF-8 path {}", p.display()))?;
|
|
if s.chars().any(|c| c == ',' || c == '"' || c.is_control()) {
|
|
bail!("unsupported character in path {s:?} (',', '\"' and control characters would inject vfkit options)");
|
|
}
|
|
Ok(s.to_string())
|
|
}
|
|
|
|
/// Where our stdout points (vfkit only accepts a path for non-TTY console output).
|
|
fn out_path() -> PathBuf {
|
|
fs::read_link("/dev/fd/1").unwrap_or("/dev/stdout".into())
|
|
}
|
|
|
|
fn stty(args: &[&str]) -> Option<String> {
|
|
let out = Command::new("stty").args(args).stdin(Stdio::inherit()).stderr(Stdio::null()).output().ok()?;
|
|
out.status.success().then(|| String::from_utf8_lossy(&out.stdout).into_owned())
|
|
}
|
|
|
|
fn term_size() -> Option<(u32, u32)> {
|
|
let s = stty(&["size"])?;
|
|
let mut it = s.split_whitespace().map(|x| x.parse().ok());
|
|
Some((it.next()??, it.next()??))
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use std::path::Path;
|
|
|
|
#[test]
|
|
fn vfkit_injection_rejected() {
|
|
assert!(super::vf(Path::new("/Users/me/repo")).is_ok());
|
|
for bad in ["/shared/repo/,sharedDir=/Users/me", "/a\"b", "/a\nb"] {
|
|
assert!(super::vf(Path::new(bad)).is_err(), "{bad}");
|
|
}
|
|
}
|
|
}
|