From cff6dae88fae1b24dc3e0f855b7a31d959c0d1a7 Mon Sep 17 00:00:00 2001 From: Niklas Haiden Date: Fri, 2 Oct 2026 15:32:14 +0200 Subject: [PATCH] Add image gc, stop VMs on kernel panic, harden first boot - Add `wsbx gc` to delete cached images that no sandbox uses, and drop the superseded image automatically when a tag moves to a new digest (unless another ref still points at it). - Boot the guest with panic=1. Virtualization.framework restarts a rebooting guest, so init now leaves a marker in the per-launch control share and powers off when it is already there. A kernel panic (or a reboot inside the VM) now ends the run with exit 1 instead of hanging or looping forever. - Report a VM that stops without an exit status as an error. - Sync the first-boot image unpack before marking the sandbox provisioned; a crash right after first boot previously left an empty root filesystem. - Validate kernel, workspace and state-dir paths before creating any sandbox files, so rejected runs no longer leave a sandbox behind. - Ignore .llm-wiki/, commit .gitignore, fix the remaining Clippy warning, and document gc and the panic behavior. --- .gitignore | 2 ++ PLAN.md | 11 ++++++++ README.md | 5 +++- src/init.sh | 7 ++++- src/main.rs | 29 ++++++++++++++++----- src/oci.rs | 73 ++++++++++++++++++++++++++++++++++++++++++----------- 6 files changed, 104 insertions(+), 23 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..98306ca --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/target +/.llm-wiki/ diff --git a/PLAN.md b/PLAN.md index 01f628d..dc8c7b9 100644 --- a/PLAN.md +++ b/PLAN.md @@ -91,5 +91,16 @@ define the device order), vsock ports, 20 GiB sparse disk, the agent image names | Medium: pi copy follows symlinks | contained to staged roots (incl. `~/.agents/skills`, which pi reads natively), cycle-safe, no special files, 256 MB cap | unit test + `show-me` still arrives | | (own) | reference/tag/registry validated before any request (`alpine@sha256:../../x` hit the registry as `/manifests/x`), busybox must be a regular file, Kata tag validated | unit tests | +## Iteration 5: cleanup ✅ +- `wsbx gc`: keeps only images whose tag a sandbox uses (freed 1.74 GB here). A tag that moves now deletes the superseded + image automatically, unless another ref still points at it. +- Kernel panics: `panic=1` makes the kernel reboot, and VZ turns that into a restart loop. init writes a `booted` marker into + the per-launch share and powers off when it's already there. A real panic (`panic_on_oom`) now stops in about 2 s with + exit 1 and a message. Limit: a panic before the root disk mounts still loops. +- Found while testing panics: the first-boot unpack wasn't synced before `provisioned` was recorded, so a crash right after + first boot left an empty system. Fixed with a sync before marking. +- Found while reviewing: path validation ran after disk creation, so rejected runs left a sandbox behind. It now runs first. +- `.gitignore` covers `.llm-wiki/`; Clippy is clean. + ## Out of scope (add when needed) Port forwarding, network policy / credential-injecting egress proxy, snapshots, x86_64 hosts, zstd layers. diff --git a/README.md b/README.md index 61518e6..de6ba82 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,12 @@ wsbx run alpine # bare VM, interactive shell wsbx run -w . ubuntu -- make test # one-off command, exit code propagated wsbx run -p 8000 alpine # host's localhost:8000 reachable as localhost:8000 in the VM wsbx ls / wsbx rm NAME / wsbx pull IMAGE +wsbx gc # delete cached images no sandbox uses wsbx kernel [--update | --version TAG] # latest Kata release by default, sha256-verified ``` Defaults for `--cpus`/`--memory` come from the host (half the cores, a quarter of RAM up to 8 GiB). Image tags -are re-resolved after 24 h (cached copy used when offline). `WSBX_KATA_VERSION` pins the kernel release. +are re-resolved after 24 h (cached copy used when offline); the superseded image is deleted automatically. `WSBX_KATA_VERSION` pins the kernel release. Options: `--name --workspace --cpus --memory --disk --env K[=V] --host-port/-p PORT --fresh --pull --kernel`. @@ -56,4 +57,6 @@ Remaining risks: IDE tasks, package scripts). Review changes before running them on the host. - Egress is unrestricted NAT. Anything the VM holds (keys passed with `--keys`, API keys in pi's `models.json`, `--pi-auth`) can be exfiltrated. A host-side credential-injecting egress proxy would fix this; it isn't built yet. +- A kernel panic before the root disk mounts (e.g. a corrupt `disk.img`) still restart-loops; panics after boot stop + the VM (exit 1). - Image layers are unpacked on the host into one tar, so a decompression bomb can fill the disk. diff --git a/src/init.sh b/src/init.sh index b2c1516..b8d3ef2 100644 --- a/src/init.sh +++ b/src/init.sh @@ -4,11 +4,16 @@ B=/.wsbx/busybox export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +# The host recreates the share per launch, so a marker here means this is a reboot (kernel panic=1 or +# `reboot` in the VM). VZ would restart the guest forever; stop instead. +if [ -e /.wsbx/host/booted ]; then echo "wsbx: guest rebooted (kernel panic?), stopping"; $B poweroff -f; fi +: > /.wsbx/host/booted + if [ ! -e /.wsbx/provisioned ]; then echo "wsbx: first boot, unpacking image..." $B mount -t devtmpfs dev /.wsbx/dev if $B tar -xf /.wsbx/dev/vdb -C /; then - $B chown 0:0 / /.wsbx && $B touch /.wsbx/provisioned + $B chown 0:0 / /.wsbx && $B touch /.wsbx/provisioned && $B sync # durable before the host is told else echo "wsbx: WARNING: image unpack reported errors" fi diff --git a/src/main.rs b/src/main.rs index 1347e4b..7c3a5ce 100644 --- a/src/main.rs +++ b/src/main.rs @@ -73,6 +73,8 @@ enum Cmd { Ls, /// Remove sandboxes Rm { names: Vec }, + /// Delete cached images that no sandbox uses + Gc, } #[derive(Args)] @@ -156,6 +158,17 @@ fn main() -> Result<()> { Cmd::Gemini(a) => agent(&h, a, "gemini", "gemini", vec![], &["GEMINI_API_KEY", "GOOGLE_API_KEY"], Guest::default()), Cmd::Pi { a, pi_auth } => pi(&h, a, pi_auth), Cmd::Ls => ls(&h), + Cmd::Gc => { + let used: Vec = fs::read_dir(h.join("sandboxes")) + .into_iter() + .flatten() + .flatten() + .filter_map(|e| serde_json::from_slice::(&fs::read(e.path().join("meta.json")).ok()?).ok()) + .filter_map(|m| m["image"].as_str().map(String::from)) + .collect(); + println!("freed {} MB", oci::gc(&h, &used)? / 1_000_000); + Ok(()) + } Cmd::Rm { names } => { for n in names { if n.is_empty() || sanitize(&n) != n { @@ -310,10 +323,10 @@ fn kernel(h: &Path, update: bool, version: Option) -> Result { if k.exists() && !update && (pin.is_none() || pin == installed) { return Ok(k); } - if let Some(v) = &pin { - if v.is_empty() || !v.chars().all(|c| c.is_ascii_alphanumeric() || "._-".contains(c)) { - bail!("invalid Kata version {v:?}"); - } + if let Some(v) = &pin + && (v.is_empty() || !v.chars().all(|c| c.is_ascii_alphanumeric() || "._-".contains(c))) + { + bail!("invalid Kata version {v:?}"); } let api = match &pin { Some(v) => format!("https://api.github.com/repos/{KATA_REPO}/releases/tags/{v}"), @@ -426,7 +439,10 @@ fn run(h: &Path, o: Opts, image: &str, cmd: Vec, extra: &[String], g: Gu None => kernel(h, false, None)?, }; let ws = o.workspace.as_ref().map(fs::canonicalize).transpose().context("workspace")?; + vf(h)?; // everything vfkit gets lives under h, plus kernel + workspace: refuse bad paths before creating anything + vf(&kernel)?; if let Some(ws) = &ws { + vf(ws)?; let hc = fs::canonicalize(h)?; if ws.starts_with(&hc) || hc.starts_with(ws) { bail!("workspace {} overlaps the wsbx state dir {} (disks, kernel, control files)", ws.display(), hc.display()); @@ -564,7 +580,8 @@ fn run(h: &Path, o: Opts, image: &str, cmd: Vec, extra: &[String], g: Gu let x = name.bytes().fold(0xcbf29ce484222325u64, |a, b| (a ^ b as u64).wrapping_mul(0x100000001b3)); format!("02:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", x as u8, (x >> 8) as u8, (x >> 16) as u8, (x >> 24) as u8, (x >> 32) as u8) }; - let cmdline = "console=hvc0 root=/dev/vda rw rootfstype=ext4 init=/.wsbx/init quiet"; + // panic=1 reboots, which VZ turns into a restart; init powers off on any second boot (see init.sh). + let cmdline = "console=hvc0 root=/dev/vda rw rootfstype=ext4 init=/.wsbx/init quiet panic=1"; let tty = std::io::stdin().is_terminal(); let mut v = Command::new(vfkit); v.args(["--cpus", &o.cpus.to_string(), "--memory", &o.memory.to_string(), "--log-level", "error"]) @@ -611,7 +628,7 @@ fn run(h: &Path, o: Opts, image: &str, cmd: Vec, extra: &[String], g: Gu match code { Some(c) => std::process::exit(c), None if !status.success() => bail!("vm failed, see {}", dir.join("vfkit.log").display()), - None => Ok(()), + None => bail!("VM stopped without an exit status (kernel panic, reboot or poweroff inside the VM)"), } } diff --git a/src/oci.rs b/src/oci.rs index ef90963..5fc7670 100644 --- a/src/oci.rs +++ b/src/oci.rs @@ -7,6 +7,7 @@ use std::collections::HashSet; use std::fs::{self, File}; use std::io::{BufReader, Read, Write}; use std::net::{IpAddr, SocketAddr, ToSocketAddrs}; +use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; const ACCEPT: &str = "application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json"; @@ -177,24 +178,66 @@ const REF_TTL: std::time::Duration = std::time::Duration::from_secs(24 * 3600); /// Pull (or reuse cached) image, returning its flattened rootfs tar + config. pub fn pull(home: &Path, image: &str, refresh: bool) -> Result { validate_ref(&parse_ref(image)).with_context(|| format!("invalid image reference {image:?}"))?; - let refs = home.join("images/refs"); - fs::create_dir_all(&refs)?; - let ref_file = refs.join(image.replace(['/', ':', '@'], "_")); - let cached = fs::read_to_string(&ref_file).ok().and_then(|d| load(home, d.trim()).ok()); + fs::create_dir_all(home.join("images/refs"))?; + let ref_file = ref_path(home, image); + let old = fs::read_to_string(&ref_file).ok().map(|d| d.trim().to_string()); + let Some(img) = old.as_deref().and_then(|d| load(home, d).ok()) else { return fetch(home, image, &ref_file) }; let age = fs::metadata(&ref_file).and_then(|m| m.modified()).ok().and_then(|t| t.elapsed().ok()); - if let Some(img) = cached { - if !refresh && (image.contains('@') || age.is_some_and(|a| a < REF_TTL)) { - return Ok(img); + if !refresh && (image.contains('@') || age.is_some_and(|a| a < REF_TTL)) { + return Ok(img); + } + match fetch(home, image, &ref_file) { + Ok(new) => { + drop_unreferenced(home, &img.digest)?; // tag moved: the old multi-GB tar is dead weight + Ok(new) } - return fetch(home, image, &ref_file).or_else(|e| { - if refresh { - return Err(e); - } + Err(e) if !refresh => { eprintln!("wsbx: could not re-resolve {image} ({e:#}); using cached copy"); Ok(img) - }); + } + Err(e) => Err(e), } - fetch(home, image, &ref_file) +} + +fn ref_path(home: &Path, image: &str) -> PathBuf { + home.join("images/refs").join(image.replace(['/', ':', '@'], "_")) +} + +fn image_dir(home: &Path, digest: &str) -> PathBuf { + home.join("images").join(digest.replace(':', "-")) +} + +/// Remove an image unless some ref still points at it. A VM still provisioning from its tar keeps +/// its open file descriptor, so unlinking under it is safe. +fn drop_unreferenced(home: &Path, digest: &str) -> Result<()> { + let used = fs::read_dir(home.join("images/refs"))?.flatten().any(|e| fs::read_to_string(e.path()).is_ok_and(|d| d.trim() == digest)); + if !used && check_digest(digest).is_ok() { + let _ = fs::remove_dir_all(image_dir(home, digest)); + } + Ok(()) +} + +/// Delete every cached image not referenced (via its tag) by one of `keep`; returns bytes freed. +pub fn gc(home: &Path, keep: &[String]) -> Result { + let keep: HashSet = keep.iter().map(|i| ref_path(home, i)).collect(); + let mut live = HashSet::new(); + for e in fs::read_dir(home.join("images/refs"))?.flatten() { + match fs::read_to_string(e.path()) { + Ok(d) if keep.contains(&e.path()) => drop(live.insert(image_dir(home, d.trim()))), + _ => fs::remove_file(e.path())?, + } + } + let mut freed = 0; + for e in fs::read_dir(home.join("images"))?.flatten() { + let p = e.path(); + if e.file_name() == "refs" || live.contains(&p) { + continue; + } + // ponytail: also removes a .tmp dir of a pull running right now; that pull fails and can be retried + freed += fs::read_dir(&p)?.flatten().filter_map(|f| f.metadata().ok()).map(|m| m.blocks() * 512).sum::(); + fs::remove_dir_all(p)?; + } + Ok(freed) } fn fetch(home: &Path, image: &str, ref_file: &Path) -> Result { @@ -217,7 +260,7 @@ fn fetch(home: &Path, image: &str, ref_file: &Path) -> Result { } (manifest, digest) = (m2, d2); } - let dir = home.join("images").join(digest.replace(':', "-")); + let dir = image_dir(home, &digest); if let Ok(img) = load(home, &digest) { fs::write(ref_file, &digest)?; // also bumps mtime -> TTL restarts return Ok(img); @@ -256,7 +299,7 @@ fn fetch(home: &Path, image: &str, ref_file: &Path) -> Result { } fn load(home: &Path, digest: &str) -> Result { - let dir = home.join("images").join(digest.replace(':', "-")); + let dir = image_dir(home, digest); let cfg: Value = serde_json::from_slice(&fs::read(dir.join("config.json"))?)?; let rootfs = dir.join("rootfs.tar"); if !rootfs.exists() {