Compare commits
4
Commits
v0.1.2
...
ae91de745e
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ae91de745e | ||
|
|
687cafb392 | ||
|
|
1ac5193b0e | ||
|
|
230927d8db |
Generated
+1
-1
@@ -854,7 +854,7 @@ checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc"
|
||||
|
||||
[[package]]
|
||||
name = "wsbx"
|
||||
version = "0.1.2"
|
||||
version = "0.1.3"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "wsbx"
|
||||
version = "0.1.2"
|
||||
version = "0.2.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
|
||||
@@ -16,7 +16,10 @@ wsbx run -p 8000 alpine # host's localhost:8000 reachable as localh
|
||||
wsbx exec NAME # shell in a running sandbox (alias: sh), as the image's user
|
||||
wsbx exec -u root NAME -- apk add git # command, exit code propagated
|
||||
wsbx forward NAME 3000 8080:80 # sandbox's localhost:3000 on the Mac's localhost:3000 (until Ctrl-C; alias fwd)
|
||||
wsbx ps # sandboxes: running/stopped, image, disk use, workspace (alias: ls)
|
||||
wsbx code NAME # VS Code in the browser, running in the sandbox (until Ctrl-C)
|
||||
wsbx code NAME --desktop # desktop VS Code via Remote-SSH; also: ssh NAME.wsbx
|
||||
wsbx ps # sandboxes: status, CPU, memory, net/block I/O, PIDs, disk, image (alias: ls)
|
||||
wsbx ps -f # same, refreshed every second like `docker stats`
|
||||
wsbx rm NAME... # delete stopped sandboxes; --disk deletes only the disk (fresh on next start)
|
||||
wsbx pull IMAGE
|
||||
wsbx gc # delete cached images no sandbox uses
|
||||
@@ -34,6 +37,14 @@ You get a pty (with resize) when stdin and stdout are terminals, plain pipes oth
|
||||
**forward**: host `127.0.0.1:HOSTPORT` → `exec.sock` → vsock → guest helper → guest `localhost:PORT`, one exec-style session
|
||||
per connection. Works for services bound only to the guest's loopback, on running sandboxes, no restart. `0:PORT` picks a free host port.
|
||||
|
||||
**code**: browser mode fetches the official VS Code CLI into the sandbox (`~/.wsbx-vscode`, needs internet in the VM) and runs
|
||||
`code serve-web` on the guest's loopback, forwarded like `forward` to a random host port behind a random connection token.
|
||||
The CLI downloads the matching server on first open (glibc or Alpine images). `--desktop` writes `~/.wsbx/ssh_config` (a `NAME.wsbx`
|
||||
host per sandbox) and adds an `Include` for it at the top of `~/.ssh/config` once. ssh reaches an SSH server in the guest helper through `wsbx exec`
|
||||
(`ProxyCommand`): no port, no keys, you're the image user. Remote-SSH then installs its server in the VM as usual.
|
||||
If the VM was killed mid-download and the server won't start, delete `~/.vscode/cli/serve-web` (browser) or `~/.vscode-server` (desktop) in the sandbox.
|
||||
Both need a sandbox started by this wsbx version (the guest helper ships with each start).
|
||||
|
||||
**Host tunnel** (`-p PORT`): guest `localhost:PORT` → vsock → vfkit → unix socket → host `localhost:PORT`.
|
||||
Nothing listens on a host network interface, and only the listed ports are reachable.
|
||||
|
||||
@@ -65,6 +76,10 @@ The VM is the boundary. The guest (including a prompt-injected agent or a malici
|
||||
not from guest processes via vsock loopback.
|
||||
- **forward**: listens on the host's loopback only. Browsers don't isolate `localhost` cookies by port, so a forwarded guest
|
||||
page can read non-HttpOnly cookies your other localhost apps set. Forward only what you need.
|
||||
- **code**: the SSH server is only reachable through `wsbx exec` (your user's `~/.wsbx`), so it skips auth, with an ephemeral host key.
|
||||
It allows sessions and host → guest forwarding only; guest → host forwarding (`-R`) and agent forwarding are refused. Desktop VS Code
|
||||
auto-forwards ports the guest opens to your localhost (turn off `remote.autoForwardPorts` if unwanted), and its client
|
||||
trusts the remote server more than a browser tab does. The browser mode has the same localhost-cookie caveat as `forward`.
|
||||
- **Host tunnel**: only the ports you pass with `-p` (or the local `baseUrl`s in pi's `models.json`), over vsock.
|
||||
|
||||
Remaining risks:
|
||||
|
||||
@@ -2,15 +2,13 @@
|
||||
use std::process::Command;
|
||||
|
||||
fn main() {
|
||||
const SRC: [&str; 2] = ["guest/fwd.go", "guest/exec.go"];
|
||||
for f in SRC {
|
||||
println!("cargo:rerun-if-changed={f}");
|
||||
}
|
||||
println!("cargo:rerun-if-changed=guest"); // a directory: cargo watches everything in it
|
||||
let out = std::path::PathBuf::from(std::env::var("OUT_DIR").unwrap()).join("wsbx-guest");
|
||||
let ok = Command::new("go")
|
||||
.args(["build", "-trimpath", "-ldflags=-s -w", "-o"])
|
||||
.arg(&out)
|
||||
.args(SRC)
|
||||
.arg(".")
|
||||
.current_dir("guest")
|
||||
.env("GOOS", "linux")
|
||||
.env("GOARCH", "arm64")
|
||||
.env("CGO_ENABLED", "0")
|
||||
|
||||
+12
-6
@@ -1,8 +1,10 @@
|
||||
// `wsbx exec` server: host -> vfkit unix socket -> vsock -> here.
|
||||
//
|
||||
// Protocol: the host sends one JSON request line, then frames go both ways: kind byte, u32 BE length, payload.
|
||||
// host -> guest: 'd' stdin data, 'e' stdin EOF, 'w' window size (rows, cols: u16 BE)
|
||||
// guest -> host: 'o' stdout, 'r' stderr, 'k' keepalive (ignored), 'x' exit status (u32 BE), always last
|
||||
//
|
||||
// host -> guest: 'd' stdin data, 'e' stdin EOF, 'w' window size (rows, cols: u16 BE)
|
||||
// guest -> host: 'o' stdout, 'r' stderr, 'k' keepalive (ignored), 'x' exit status (u32 BE), always last
|
||||
//
|
||||
// With Tty the command gets a pty (stdout carries everything); otherwise plain pipes.
|
||||
// With Connect (`wsbx forward`) there's no command: 'd'/'e' go to the guest's localhost:Connect, its output
|
||||
// comes back as 'o', and 'x' marks its EOF (status 0) or a failed dial (1, after an 'r' message).
|
||||
@@ -174,11 +176,15 @@ func session(c *os.File) {
|
||||
case <-output:
|
||||
case <-time.After(500 * time.Millisecond): // a background job still holds the pty
|
||||
}
|
||||
code := uint32(cmd.ProcessState.ExitCode())
|
||||
if ws, ok := cmd.ProcessState.Sys().(syscall.WaitStatus); ok && ws.Signaled() {
|
||||
code = 128 + uint32(ws.Signal())
|
||||
send('x', binary.BigEndian.AppendUint32(nil, exitCode(cmd.ProcessState)))
|
||||
}
|
||||
|
||||
// exitCode like a shell's $?: 128+N for a signal.
|
||||
func exitCode(ps *os.ProcessState) uint32 {
|
||||
if ws, ok := ps.Sys().(syscall.WaitStatus); ok && ws.Signaled() {
|
||||
return 128 + uint32(ws.Signal())
|
||||
}
|
||||
send('x', binary.BigEndian.AppendUint32(nil, code))
|
||||
return uint32(ps.ExitCode())
|
||||
}
|
||||
|
||||
// forward pipes a session to the guest's localhost:port until both sides sent EOF or the host is gone.
|
||||
|
||||
+10
-1
@@ -1,5 +1,6 @@
|
||||
// wsbx-guest: guest helper, started by run.sh as root.
|
||||
// Usage: wsbx-guest [-exec VSOCKPORT] PORT=VSOCKPORT...
|
||||
// Usage: wsbx-guest [-exec VSOCKPORT] PORT=VSOCKPORT... | wsbx-guest -ssh-stdio
|
||||
// -ssh-stdio serves one SSH connection on stdin/stdout (ssh.go), run through `wsbx exec`.
|
||||
// -exec serves `wsbx exec` sessions (exec.go). Each PORT=VSOCKPORT is a host tunnel: listens on
|
||||
// 127.0.0.1:PORT and [::1]:PORT and pipes each connection to the host (vsock CID 2).
|
||||
package main
|
||||
@@ -65,7 +66,15 @@ func handle(c *net.TCPConn, port uint32) {
|
||||
|
||||
func main() {
|
||||
execPort := flag.Uint("exec", 0, "serve `wsbx exec` sessions on this vsock port")
|
||||
sshStdio := flag.Bool("ssh-stdio", false, "serve one SSH connection on stdin/stdout")
|
||||
flag.Parse()
|
||||
if *sshStdio {
|
||||
if err := serveSSH(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "wsbx-guest: ssh:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
// ash starts '&' jobs with SIGINT/SIGQUIT ignored, and ignored signals survive exec: Ctrl-C would
|
||||
// not work in exec sessions. Handled signals are reset to default in children, so take them over.
|
||||
signal.Notify(make(chan os.Signal, 1), syscall.SIGINT, syscall.SIGQUIT)
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
module wsbx-guest
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require golang.org/x/crypto v0.58.0
|
||||
|
||||
require golang.org/x/sys v0.49.0 // indirect
|
||||
@@ -0,0 +1,6 @@
|
||||
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
|
||||
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
|
||||
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM=
|
||||
golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY=
|
||||
+211
@@ -0,0 +1,211 @@
|
||||
// `wsbx code --desktop`: an SSH server on stdin/stdout for VS Code Remote-SSH (or plain ssh), one per
|
||||
// connection, started by the host's ProxyCommand: `wsbx exec NAME -- /.wsbx/host/guest -ssh-stdio`.
|
||||
// The exec channel is the authentication (only the host user can open it), so the client gets in with no
|
||||
// auth against an ephemeral host key, and runs as the user exec started us as (the image user).
|
||||
// Supported: sessions (exec, shell, pty, env, window-change) and host -> guest forwarding (direct-tcpip,
|
||||
// direct-streamlocal). Guest -> host forwarding (tcpip-forward) and agent forwarding are refused.
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ed25519"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
func serveSSH() error {
|
||||
_, key, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
signer, err := ssh.NewSignerFromKey(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg := &ssh.ServerConfig{NoClientAuth: true}
|
||||
cfg.AddHostKey(signer)
|
||||
_, chans, reqs, err := ssh.NewServerConn(stdio{}, cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go ssh.DiscardRequests(reqs) // global requests: tcpip-forward etc. are refused
|
||||
for nc := range chans {
|
||||
switch nc.ChannelType() {
|
||||
case "session":
|
||||
go sshSession(nc)
|
||||
case "direct-tcpip":
|
||||
var p struct {
|
||||
Host string
|
||||
Port uint32
|
||||
OrigHost string
|
||||
OrigPort uint32
|
||||
}
|
||||
if ssh.Unmarshal(nc.ExtraData(), &p) != nil {
|
||||
nc.Reject(ssh.ConnectionFailed, "bad request")
|
||||
continue
|
||||
}
|
||||
go sshForward(nc, "tcp", net.JoinHostPort(p.Host, strconv.Itoa(int(p.Port))))
|
||||
case "direct-streamlocal@openssh.com":
|
||||
var p struct {
|
||||
Path string
|
||||
Reserved0 string
|
||||
Reserved1 uint32
|
||||
}
|
||||
if ssh.Unmarshal(nc.ExtraData(), &p) != nil {
|
||||
nc.Reject(ssh.ConnectionFailed, "bad request")
|
||||
continue
|
||||
}
|
||||
go sshForward(nc, "unix", p.Path)
|
||||
default:
|
||||
nc.Reject(ssh.UnknownChannelType, "not supported by wsbx")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sshForward(nc ssh.NewChannel, network, addr string) {
|
||||
c, err := net.Dial(network, addr)
|
||||
if err != nil {
|
||||
nc.Reject(ssh.ConnectionFailed, err.Error())
|
||||
return
|
||||
}
|
||||
defer c.Close()
|
||||
ch, reqs, err := nc.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer ch.Close()
|
||||
go ssh.DiscardRequests(reqs)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
io.Copy(ch, c)
|
||||
ch.CloseWrite()
|
||||
close(done)
|
||||
}()
|
||||
io.Copy(c, ch)
|
||||
if cw, ok := c.(interface{ CloseWrite() error }); ok {
|
||||
cw.CloseWrite()
|
||||
}
|
||||
<-done
|
||||
}
|
||||
|
||||
func sshSession(nc ssh.NewChannel) {
|
||||
ch, reqs, err := nc.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer ch.Close()
|
||||
var env []string
|
||||
var pty, tty *os.File
|
||||
defer func() {
|
||||
if pty != nil {
|
||||
pty.Close()
|
||||
}
|
||||
}()
|
||||
started, command := false, ""
|
||||
for req := range reqs {
|
||||
ok, start := false, false
|
||||
switch req.Type {
|
||||
case "env":
|
||||
var kv struct{ K, V string }
|
||||
if ssh.Unmarshal(req.Payload, &kv) == nil {
|
||||
env, ok = append(env, kv.K+"="+kv.V), true
|
||||
}
|
||||
case "pty-req":
|
||||
var p struct {
|
||||
Term string
|
||||
Cols, Rows, W, H uint32
|
||||
Modes string
|
||||
}
|
||||
if pty == nil && !started && ssh.Unmarshal(req.Payload, &p) == nil {
|
||||
if m, s, err := openPty(); err == nil {
|
||||
pty, tty, ok = m, s, true
|
||||
setWinsize(m, uint16(p.Rows), uint16(p.Cols))
|
||||
env = append(env, "TERM="+p.Term)
|
||||
}
|
||||
}
|
||||
case "window-change":
|
||||
var p struct{ Cols, Rows, W, H uint32 }
|
||||
if pty != nil && ssh.Unmarshal(req.Payload, &p) == nil {
|
||||
setWinsize(pty, uint16(p.Rows), uint16(p.Cols))
|
||||
ok = true
|
||||
}
|
||||
case "shell", "exec":
|
||||
var p struct{ Cmd string }
|
||||
if !started && (req.Type == "shell" || ssh.Unmarshal(req.Payload, &p) == nil) {
|
||||
started, ok, start, command = true, true, true, p.Cmd
|
||||
}
|
||||
}
|
||||
if req.WantReply {
|
||||
req.Reply(ok, nil)
|
||||
}
|
||||
if start {
|
||||
go func(env []string) {
|
||||
code := sshRun(ch, command, env, pty, tty)
|
||||
ch.SendRequest("exit-status", false, ssh.Marshal(struct{ C uint32 }{code}))
|
||||
ch.Close()
|
||||
}(env)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sshRun runs command (empty: a login shell) like sshd: `$SHELL -c command`, in run.sh's env and dir.
|
||||
func sshRun(ch ssh.Channel, command string, env []string, pty, tty *os.File) uint32 {
|
||||
shell := "/bin/sh"
|
||||
if _, err := os.Stat("/bin/bash"); err == nil {
|
||||
shell = "/bin/bash"
|
||||
}
|
||||
cmd := exec.Command(shell, "-c", command)
|
||||
if command == "" {
|
||||
cmd = exec.Command(shell, "-l")
|
||||
}
|
||||
cmd.Env = append(os.Environ(), env...)
|
||||
if pty != nil {
|
||||
cmd.Stdin, cmd.Stdout, cmd.Stderr = tty, tty, tty
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true, Setctty: true}
|
||||
} else {
|
||||
cmd.Stdin, cmd.Stdout, cmd.Stderr = ch, ch, ch.Stderr()
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
|
||||
cmd.WaitDelay = time.Second // don't hang on daemons (the VS Code server) that keep stdout open
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
fmt.Fprintln(ch.Stderr(), "wsbx:", err)
|
||||
return 127
|
||||
}
|
||||
if pty == nil {
|
||||
cmd.Wait()
|
||||
return exitCode(cmd.ProcessState)
|
||||
}
|
||||
tty.Close() // our copy: the master reads EIO once the session's are gone
|
||||
go io.Copy(pty, ch)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
io.Copy(ch, pty)
|
||||
close(done)
|
||||
}()
|
||||
cmd.Wait()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(500 * time.Millisecond): // a background job still holds the pty
|
||||
}
|
||||
return exitCode(cmd.ProcessState)
|
||||
}
|
||||
|
||||
// stdio is our stdin/stdout as the net.Conn the SSH server runs on.
|
||||
type stdio struct{}
|
||||
|
||||
func (stdio) Read(b []byte) (int, error) { return os.Stdin.Read(b) }
|
||||
func (stdio) Write(b []byte) (int, error) { return os.Stdout.Write(b) }
|
||||
func (stdio) Close() error { os.Stdin.Close(); return os.Stdout.Close() }
|
||||
func (stdio) LocalAddr() net.Addr { return &net.UnixAddr{Name: "stdio", Net: "unix"} }
|
||||
func (stdio) RemoteAddr() net.Addr { return &net.UnixAddr{Name: "wsbx", Net: "unix"} }
|
||||
func (stdio) SetDeadline(time.Time) error { return nil }
|
||||
func (stdio) SetReadDeadline(time.Time) error { return nil }
|
||||
func (stdio) SetWriteDeadline(time.Time) error { return nil }
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
//! `wsbx code NAME`: VS Code for a running sandbox.
|
||||
//! Browser (default): `code serve-web` runs in the VM on its localhost, forwarded to the host's localhost
|
||||
//! (like `wsbx forward`) behind a random connection token, and opened in the browser.
|
||||
//! --desktop: desktop VS Code via Remote-SSH. ssh reaches the SSH server in the guest helper (guest/ssh.go)
|
||||
//! through `wsbx exec` as ProxyCommand: no network port, no keys.
|
||||
use crate::exec::{self, sock};
|
||||
use crate::forward;
|
||||
use crate::util::sq;
|
||||
use anyhow::{Context, Result, bail};
|
||||
use serde_json::Value;
|
||||
use std::fs;
|
||||
use std::io::Read;
|
||||
use std::net::TcpListener;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Guest side of the browser mode: fetch the VS Code CLI once (static musl binary, onto the sandbox disk),
|
||||
/// then serve. The CLI downloads the matching server on first use. $1 port, $2 token.
|
||||
const WEB_SH: &str = r#"set -e
|
||||
B=/.wsbx/busybox D="$HOME/.wsbx-vscode"
|
||||
U=https://update.code.visualstudio.com/latest/cli-alpine-arm64/stable
|
||||
if [ ! -x "$D/code" ]; then
|
||||
echo "wsbx: downloading the VS Code CLI..."
|
||||
$B rm -rf "$D/tmp" && $B mkdir -p "$D/tmp"
|
||||
if command -v curl >/dev/null; then curl -fsSL "$U"; else wget -qO- "$U"; fi | $B tar -xzf - -C "$D/tmp"
|
||||
$B sync && $B mv "$D/tmp/code" "$D/code" && $B rmdir "$D/tmp" # never a half-written code, even if the VM is killed
|
||||
fi
|
||||
exec "$D/code" serve-web --host 127.0.0.1 --port "$1" --connection-token "$2" --accept-server-license-terms
|
||||
"#;
|
||||
|
||||
pub fn code(h: &Path, name: &str, desktop: bool) -> Result<()> {
|
||||
let sock = sock(h, name)?; // also checks the name and that it's running
|
||||
let meta: Value = serde_json::from_slice(&fs::read(h.join("sandboxes").join(name).join("meta.json"))?)?;
|
||||
let folder = enc(meta["workspace"].as_str().unwrap_or("/"));
|
||||
if desktop {
|
||||
return self::desktop(h, name, &folder);
|
||||
}
|
||||
|
||||
let l = TcpListener::bind(("127.0.0.1", 0))?;
|
||||
let port = l.local_addr()?.port(); // the same port in the guest: random and high, so most likely free there too
|
||||
let mut t = [0u8; 16];
|
||||
fs::File::open("/dev/urandom")?.read_exact(&mut t)?;
|
||||
let token = hex::encode(t);
|
||||
eprintln!("wsbx: starting VS Code in {name} (the first start downloads it)");
|
||||
eprintln!("wsbx: by using it you accept the VS Code Server license: https://aka.ms/vscode-server-license");
|
||||
let (s, tk) = (sock.clone(), token.clone());
|
||||
let server = std::thread::spawn(move || exec::stream(&s, &["/.wsbx/busybox", "sh", "-c", WEB_SH, "sh", &port.to_string(), &tk]));
|
||||
let t0 = Instant::now();
|
||||
while !forward::listening(&sock, port) {
|
||||
if server.is_finished() {
|
||||
let r = server.join().unwrap();
|
||||
bail!("VS Code server in {name} stopped ({:?})", r.map_err(|e| e.to_string()));
|
||||
}
|
||||
if t0.elapsed() > Duration::from_secs(300) {
|
||||
bail!("VS Code server in {name} didn't start within 5 minutes");
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(500));
|
||||
}
|
||||
let (s, n) = (sock.clone(), name.to_string());
|
||||
std::thread::spawn(move || forward::serve(&s, &n, vec![(l, port)]));
|
||||
let url = format!("http://127.0.0.1:{port}/?tkn={token}&folder={folder}");
|
||||
eprintln!("wsbx: VS Code for {name}: {url}\nwsbx: Ctrl-C stops it");
|
||||
let _ = Command::new("open").arg(&url).status();
|
||||
match server.join().unwrap()? {
|
||||
Some(c) => bail!("VS Code server in {name} exited with {c}"),
|
||||
None => bail!("lost connection to {name} (VM stopped?)"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Write ssh config for every sandbox, include it from ~/.ssh/config once, open VS Code on NAME.wsbx.
|
||||
fn desktop(h: &Path, name: &str, folder: &str) -> Result<()> {
|
||||
let exe = std::env::current_exe()?;
|
||||
let mut cfg = String::from("# Written by `wsbx code --desktop`, regenerated each time: don't edit.\n");
|
||||
for e in fs::read_dir(h.join("sandboxes"))?.flatten().filter(|e| e.path().join("meta.json").exists()) {
|
||||
let n = e.file_name().to_string_lossy().into_owned(); // sandbox names are [a-z0-9-]
|
||||
cfg += &format!(
|
||||
"\nHost {n}.wsbx\n ProxyCommand {} exec {n} -- /.wsbx/host/guest -ssh-stdio\n \
|
||||
StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n LogLevel ERROR\n ForwardAgent no\n ForwardX11 no\n",
|
||||
sq(&exe.to_string_lossy())
|
||||
);
|
||||
}
|
||||
let ours = h.join("ssh_config");
|
||||
fs::write(&ours, cfg)?;
|
||||
|
||||
let home = std::path::PathBuf::from(std::env::var_os("HOME").context("HOME")?);
|
||||
let user_cfg = home.join(".ssh/config");
|
||||
let include = format!("Include \"{}\"", ours.display());
|
||||
let cur = match fs::read_to_string(&user_cfg) {
|
||||
Ok(s) => s,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => String::new(),
|
||||
Err(e) => return Err(e).context("~/.ssh/config"),
|
||||
};
|
||||
if !cur.lines().any(|l| l.trim() == include) {
|
||||
fs::create_dir_all(home.join(".ssh"))?;
|
||||
// At the top: an Include after a `Host` line would only apply to that host.
|
||||
fs::write(&user_cfg, format!("{include}\n\n{cur}"))?;
|
||||
eprintln!("wsbx: added `{include}` to the top of ~/.ssh/config");
|
||||
}
|
||||
let url = format!("vscode://vscode-remote/ssh-remote+{name}.wsbx{folder}");
|
||||
eprintln!("wsbx: opening {url} (needs the Remote - SSH extension); plain ssh works too: ssh {name}.wsbx");
|
||||
if !Command::new("open").arg(&url).status()?.success() {
|
||||
bail!("couldn't open {url} (is VS Code installed?)");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Percent-encode a path for a URL, keeping '/'.
|
||||
fn enc(p: &str) -> String {
|
||||
p.bytes()
|
||||
.map(|b| if b.is_ascii_alphanumeric() || b"/-._~".contains(&b) { (b as char).to_string() } else { format!("%{b:02X}") })
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn enc() {
|
||||
assert_eq!(super::enc("/Users/me/my repo/ä&x"), "/Users/me/my%20repo/%C3%A4%26x");
|
||||
}
|
||||
}
|
||||
+35
-6
@@ -76,7 +76,7 @@ pub fn exec(h: &Path, name: &str, user: Option<String>, argv: Vec<String>) -> Re
|
||||
}
|
||||
});
|
||||
}
|
||||
let code = output(s);
|
||||
let code = output(s, &mut io::stdout().lock(), &mut io::stderr().lock());
|
||||
if let Some(s) = saved {
|
||||
stty(&[s.trim()]);
|
||||
}
|
||||
@@ -90,10 +90,39 @@ fn send(w: &Mutex<UnixStream>, kind: u8, p: &[u8]) -> io::Result<()> {
|
||||
w.lock().unwrap().write_all(&frame(kind, p))
|
||||
}
|
||||
|
||||
/// Copy guest output frames to stdout/stderr until the exit status arrives (None: connection closed first).
|
||||
fn output(s: UnixStream) -> io::Result<Option<i32>> {
|
||||
/// Run a command as root in a running sandbox (no tty, empty stdin) and return its stdout, at most 64 KiB.
|
||||
pub fn capture(sock: &Path, argv: &[&str]) -> Result<String> {
|
||||
// uid 0, not "root": works even if the image has no /etc/passwd
|
||||
let s = start(sock, argv, Some("0"))?;
|
||||
s.set_read_timeout(Some(Duration::from_secs(2)))?;
|
||||
const CAP: usize = 64 << 10;
|
||||
let mut buf = vec![0u8; CAP];
|
||||
let mut w = &mut buf[..]; // a slice writer errors once full: the guest can't make us buffer more
|
||||
match output(s, &mut w, &mut io::sink())? {
|
||||
Some(0) => {
|
||||
let n = CAP - w.len();
|
||||
Ok(String::from_utf8_lossy(&buf[..n]).into_owned())
|
||||
}
|
||||
c => bail!("guest command failed ({c:?})"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Start a command without tty, with empty stdin; user None: the image user.
|
||||
fn start(sock: &Path, argv: &[&str], user: Option<&str>) -> Result<UnixStream> {
|
||||
let mut s = UnixStream::connect(sock)?;
|
||||
s.write_all(format!("{}\n", json!({"argv": argv, "user": user, "tty": false})).as_bytes())?;
|
||||
s.write_all(&frame(b'e', &[]))?;
|
||||
Ok(s)
|
||||
}
|
||||
|
||||
/// Run a command as the image user, all its output to our stderr. None: the connection ended first.
|
||||
pub fn stream(sock: &Path, argv: &[&str]) -> Result<Option<i32>> {
|
||||
Ok(output(start(sock, argv, None)?, &mut io::stderr(), &mut io::stderr())?)
|
||||
}
|
||||
|
||||
/// Copy guest output frames to out/err until the exit status arrives (None: connection closed first).
|
||||
fn output(s: UnixStream, out: &mut impl Write, err: &mut impl Write) -> io::Result<Option<i32>> {
|
||||
let mut r = io::BufReader::new(s);
|
||||
let (mut out, mut err) = (io::stdout().lock(), io::stderr().lock());
|
||||
loop {
|
||||
let mut h = [0u8; 5];
|
||||
if r.read_exact(&mut h).is_err() {
|
||||
@@ -102,11 +131,11 @@ fn output(s: UnixStream) -> io::Result<Option<i32>> {
|
||||
let mut p = (&mut r).take(u32::from_be_bytes([h[1], h[2], h[3], h[4]]).into());
|
||||
match h[0] {
|
||||
b'o' => {
|
||||
io::copy(&mut p, &mut out)?;
|
||||
io::copy(&mut p, out)?;
|
||||
out.flush()?;
|
||||
}
|
||||
b'r' => {
|
||||
io::copy(&mut p, &mut err)?;
|
||||
io::copy(&mut p, err)?;
|
||||
}
|
||||
b'x' => {
|
||||
let mut c = [0u8; 4];
|
||||
|
||||
+16
-2
@@ -28,11 +28,17 @@ pub fn forward(h: &Path, name: &str, specs: Vec<String>) -> Result<()> {
|
||||
ls.push((l, guest));
|
||||
eprintln!("wsbx: localhost:{host} -> {name}:{guest}");
|
||||
}
|
||||
serve(&sock, name, ls);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forward connections on each listener to its guest port, forever.
|
||||
pub fn serve(sock: &Path, name: &str, ls: Vec<(TcpListener, u16)>) {
|
||||
// ponytail: thread per connection, fine for dev servers; async if someone forwards thousands of sockets.
|
||||
let threads: Vec<_> = ls
|
||||
.into_iter()
|
||||
.map(|(l, port)| {
|
||||
let (sock, name) = (sock.clone(), name.to_string());
|
||||
let (sock, name) = (sock.to_path_buf(), name.to_string());
|
||||
thread::spawn(move || {
|
||||
for c in l.incoming().flatten() {
|
||||
let (sock, name) = (sock.clone(), name.clone());
|
||||
@@ -48,7 +54,15 @@ pub fn forward(h: &Path, name: &str, specs: Vec<String>) -> Result<()> {
|
||||
for t in threads {
|
||||
let _ = t.join();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether something accepts connections on the guest's localhost:port.
|
||||
pub fn listening(sock: &Path, port: u16) -> bool {
|
||||
let Ok(mut s) = std::os::unix::net::UnixStream::connect(sock) else { return false };
|
||||
let req = format!("{}\n", json!({"connect": port}));
|
||||
let mut h = [0u8; 1];
|
||||
// A failed dial answers with an 'r' (error) frame; a connected one with output, keepalive or EOF ('x').
|
||||
s.write_all(req.as_bytes()).is_ok() && s.write_all(&frame(b'e', &[])).is_ok() && s.read_exact(&mut h).is_ok() && h[0] != b'r'
|
||||
}
|
||||
|
||||
fn conn(sock: &Path, name: &str, c: TcpStream, port: u16) -> Result<()> {
|
||||
|
||||
+16
-3
@@ -1,5 +1,6 @@
|
||||
//! wsbx — Docker-Sandboxes-style microVMs on macOS (Virtualization.framework via vfkit).
|
||||
mod agents;
|
||||
mod code;
|
||||
mod disk;
|
||||
mod exec;
|
||||
mod forward;
|
||||
@@ -81,9 +82,20 @@ enum Cmd {
|
||||
#[arg(required = true)]
|
||||
ports: Vec<String>,
|
||||
},
|
||||
/// List sandboxes and whether they're running
|
||||
/// Open VS Code in a running sandbox: in the browser (default), or desktop VS Code via Remote-SSH
|
||||
Code {
|
||||
name: String,
|
||||
/// Desktop VS Code via Remote-SSH (adds an Include line to ~/.ssh/config once)
|
||||
#[arg(long)]
|
||||
desktop: bool,
|
||||
},
|
||||
/// List sandboxes with live stats (CPU, memory, network, block I/O, processes)
|
||||
#[command(visible_alias = "ps")]
|
||||
Ls,
|
||||
Ls {
|
||||
/// Keep refreshing every second, like `docker stats` (Ctrl-C to stop)
|
||||
#[arg(short, long)]
|
||||
follow: bool,
|
||||
},
|
||||
/// Remove stopped sandboxes
|
||||
Rm {
|
||||
#[arg(required = true)]
|
||||
@@ -168,7 +180,8 @@ fn main() -> Result<()> {
|
||||
Cmd::Pi { a, pi_auth } => agents::pi(&h, a, pi_auth),
|
||||
Cmd::Exec { name, user, cmd } => exec::exec(&h, &name, user, cmd),
|
||||
Cmd::Forward { name, ports } => forward::forward(&h, &name, ports),
|
||||
Cmd::Ls => sandbox::ls(&h),
|
||||
Cmd::Code { name, desktop } => code::code(&h, &name, desktop),
|
||||
Cmd::Ls { follow } => sandbox::ls(&h, follow),
|
||||
Cmd::Gc => sandbox::gc(&h),
|
||||
Cmd::Rm { names, disk } => sandbox::rm(&h, names, disk),
|
||||
Cmd::Version => Ok(println!("wsbx {}", env!("CARGO_PKG_VERSION"))),
|
||||
|
||||
+168
-19
@@ -1,11 +1,15 @@
|
||||
//! Sandbox bookkeeping: listing, removal, image gc.
|
||||
use crate::exec;
|
||||
use crate::oci;
|
||||
use crate::util::sanitize;
|
||||
use anyhow::{Context, Result, anyhow, bail};
|
||||
use serde_json::Value;
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::io::Write;
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use std::path::Path;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
/// The lock `run` holds for the VM's lifetime; Err if the sandbox is running.
|
||||
pub fn lock_stopped(dir: &Path) -> Result<fs::File> {
|
||||
@@ -14,26 +18,156 @@ pub fn lock_stopped(dir: &Path) -> Result<fs::File> {
|
||||
Ok(f)
|
||||
}
|
||||
|
||||
pub fn ls(h: &Path) -> Result<()> {
|
||||
println!("{:<28} {:<8} {:<40} {:>8} WORKSPACE", "NAME", "STATUS", "IMAGE", "DISK");
|
||||
let Ok(rd) = fs::read_dir(h.join("sandboxes")) else { return Ok(()) };
|
||||
let mut rows: Vec<_> = rd.flatten().collect();
|
||||
rows.sort_by_key(|e| e.file_name());
|
||||
for e in rows {
|
||||
/// Guest counters, read from /proc through the exec channel.
|
||||
#[derive(Default, Debug, PartialEq)]
|
||||
struct Stats {
|
||||
busy: u64,
|
||||
total: u64,
|
||||
ncpu: u64,
|
||||
mem_used: u64,
|
||||
mem_total: u64,
|
||||
rx: u64,
|
||||
tx: u64,
|
||||
rd: u64,
|
||||
wr: u64,
|
||||
pids: u64,
|
||||
}
|
||||
|
||||
const STATS_SH: &str = "cd /proc; B=/.wsbx/busybox; $B grep ^cpu stat; $B grep -E '^Mem(Total|Available):' meminfo; \
|
||||
$B tail -n +3 net/dev; $B grep ' vda ' diskstats; set -- [0-9]*; echo pids $#";
|
||||
|
||||
fn parse(out: &str) -> Stats {
|
||||
let (mut s, mut avail) = (Stats::default(), 0);
|
||||
for l in out.lines() {
|
||||
let f: Vec<&str> = l.split_whitespace().collect();
|
||||
let num = |i: usize| f.get(i).and_then(|v| v.parse::<u64>().ok()).unwrap_or(0);
|
||||
match f.first().copied() {
|
||||
Some("cpu") => {
|
||||
let v: Vec<u64> = (1..=8).map(num).collect(); // user nice system idle iowait irq softirq steal
|
||||
s.total = v.iter().sum();
|
||||
s.busy = s.total.saturating_sub(v[3] + v[4]);
|
||||
}
|
||||
Some(c) if c.starts_with("cpu") => s.ncpu += 1,
|
||||
Some("MemTotal:") => s.mem_total = num(1) * 1024,
|
||||
Some("MemAvailable:") => avail = num(1) * 1024,
|
||||
Some("pids") => s.pids = num(1),
|
||||
_ if f.get(2) == Some(&"vda") => (s.rd, s.wr) = (num(5) * 512, num(9) * 512),
|
||||
_ => {
|
||||
// net/dev: " eth0:123 ..." (no space once rx bytes outgrow the column)
|
||||
if let Some((ifc, rest)) = l.split_once(':')
|
||||
&& ifc.trim() != "lo"
|
||||
{
|
||||
let n: Vec<u64> = rest.split_whitespace().filter_map(|v| v.parse().ok()).collect();
|
||||
s.rx += n.first().copied().unwrap_or(0);
|
||||
s.tx += n.get(8).copied().unwrap_or(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
s.mem_used = s.mem_total.saturating_sub(avail);
|
||||
s
|
||||
}
|
||||
|
||||
fn human(b: u64) -> String {
|
||||
let (mut v, mut u) = (b as f64, 0);
|
||||
while v >= 1024.0 && u < 4 {
|
||||
v /= 1024.0;
|
||||
u += 1;
|
||||
}
|
||||
if u == 0 { format!("{b}B") } else { format!("{v:.1}{}", ["", "KiB", "MiB", "GiB", "TiB"][u]) }
|
||||
}
|
||||
|
||||
struct Row {
|
||||
name: String,
|
||||
dir: PathBuf,
|
||||
meta: Value,
|
||||
running: bool,
|
||||
}
|
||||
|
||||
fn rows(h: &Path) -> Result<Vec<Row>> {
|
||||
let Ok(rd) = fs::read_dir(h.join("sandboxes")) else { return Ok(vec![]) };
|
||||
let mut rows = vec![];
|
||||
for e in rd.flatten() {
|
||||
let Ok(m) = fs::read(e.path().join("meta.json")) else { continue };
|
||||
let m: Value = serde_json::from_slice(&m)?;
|
||||
let used = fs::metadata(e.path().join("disk.img")).map(|m| m.blocks() * 512).unwrap_or(0);
|
||||
let status = if lock_stopped(&e.path()).is_err() { "running" } else { "stopped" };
|
||||
println!(
|
||||
"{:<28} {:<8} {:<40} {:>6}MB {}",
|
||||
e.file_name().to_string_lossy(),
|
||||
status,
|
||||
m["image"].as_str().unwrap_or("?"),
|
||||
used / 1_000_000,
|
||||
m["workspace"].as_str().unwrap_or("-")
|
||||
);
|
||||
let running = lock_stopped(&e.path()).is_err();
|
||||
rows.push(Row { name: e.file_name().to_string_lossy().into(), dir: e.path(), meta: serde_json::from_slice(&m)?, running });
|
||||
}
|
||||
rows.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Stats of all running sandboxes, sampled in parallel; sandboxes that don't answer are left out.
|
||||
fn sample(rows: &[Row]) -> HashMap<String, Stats> {
|
||||
std::thread::scope(|sc| {
|
||||
let hs: Vec<_> = rows
|
||||
.iter()
|
||||
.filter(|r| r.running)
|
||||
.map(|r| (r.name.clone(), sc.spawn(|| exec::capture(&r.dir.join("exec.sock"), &["/.wsbx/busybox", "sh", "-c", STATS_SH]))))
|
||||
.collect();
|
||||
hs.into_iter().filter_map(|(n, j)| Some((n, parse(&j.join().ok()?.ok()?)))).collect()
|
||||
})
|
||||
}
|
||||
|
||||
fn table(rows: &[Row], prev: &HashMap<String, Stats>, cur: &HashMap<String, Stats>) -> Vec<[String; 11]> {
|
||||
let head = ["NAME", "STATUS", "CPU %", "MEM USAGE / LIMIT", "MEM %", "NET I/O", "BLOCK I/O", "PIDS", "DISK", "IMAGE", "WORKSPACE"];
|
||||
let mut t = vec![head.map(String::from)];
|
||||
for r in rows {
|
||||
let pct = |a: u64, b: u64| format!("{:.1}%", a as f64 * 100.0 / b.max(1) as f64);
|
||||
let st = cur.get(&r.name).map(|c| {
|
||||
// like docker: 100% = one vCPU busy
|
||||
let cpu = prev.get(&r.name).filter(|p| c.total > p.total).map(|p| pct(c.busy.saturating_sub(p.busy) * c.ncpu.max(1), c.total - p.total));
|
||||
[
|
||||
cpu.unwrap_or("-".into()),
|
||||
format!("{} / {}", human(c.mem_used), human(c.mem_total)),
|
||||
pct(c.mem_used, c.mem_total),
|
||||
format!("{} / {}", human(c.rx), human(c.tx)),
|
||||
format!("{} / {}", human(c.rd), human(c.wr)),
|
||||
c.pids.to_string(),
|
||||
]
|
||||
});
|
||||
let [cpu, mem, memp, net, blk, pids] = st.unwrap_or_else(|| std::array::from_fn(|_| "-".into()));
|
||||
let disk = fs::metadata(r.dir.join("disk.img")).map(|m| m.blocks() * 512).unwrap_or(0);
|
||||
let s = |k: &str| r.meta[k].as_str().unwrap_or("-").to_string();
|
||||
let status = if r.running { "running" } else { "stopped" };
|
||||
t.push([r.name.clone(), status.into(), cpu, mem, memp, net, blk, pids, human(disk), s("image"), s("workspace")]);
|
||||
}
|
||||
t
|
||||
}
|
||||
|
||||
/// `wsbx ps`; with `follow`, redraws every second like `docker stats` until Ctrl-C.
|
||||
pub fn ls(h: &Path, follow: bool) -> Result<()> {
|
||||
let mut prev = HashMap::new();
|
||||
let mut first = true;
|
||||
loop {
|
||||
let rows = rows(h)?;
|
||||
let cur = sample(&rows);
|
||||
if first && !cur.is_empty() {
|
||||
// CPU % needs two samples
|
||||
(prev, first) = (cur, false);
|
||||
std::thread::sleep(Duration::from_millis(500));
|
||||
continue;
|
||||
}
|
||||
let t = table(&rows, &prev, &cur);
|
||||
let w: Vec<usize> = (0..11).map(|i| t.iter().map(|r| r[i].chars().count()).max().unwrap_or(0)).collect();
|
||||
let mut out = String::new();
|
||||
for r in &t {
|
||||
let line: String = r
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, c)| if (2..9).contains(&i) { format!("{c:>w$} ", w = w[i]) } else { format!("{c:<w$} ", w = w[i]) })
|
||||
.collect();
|
||||
out += line.trim_end();
|
||||
out += if follow { "\x1b[K\n" } else { "\n" };
|
||||
}
|
||||
if !follow {
|
||||
print!("{out}");
|
||||
return Ok(());
|
||||
}
|
||||
print!("\x1b[H{out}\x1b[J"); // home, overwrite, clear the rest: no flicker
|
||||
std::io::stdout().flush()?;
|
||||
(prev, first) = (cur, false);
|
||||
std::thread::sleep(Duration::from_secs(1));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn rm(h: &Path, names: Vec<String>, disk: bool) -> Result<()> {
|
||||
@@ -60,6 +194,21 @@ pub fn rm(h: &Path, names: Vec<String>, disk: bool) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn parse_proc() {
|
||||
let out = "cpu 100 0 50 800 50 0 0 0 0 0\ncpu0 1 2 3\ncpu1 1 2 3\nMemTotal: 2000 kB\nMemAvailable: 500 kB\n\
|
||||
\x20 lo: 999 0 0 0 0 0 0 0 999 0\n eth0:123456789 1 0 0 0 0 0 0 42 1 0 0 0 0 0 0\n\
|
||||
\x20254 0 vda 10 0 4 0 7 0 8 0 0 0 0\npids 17\n";
|
||||
let s = super::parse(out);
|
||||
assert_eq!((s.total, s.busy, s.ncpu), (1000, 150, 2));
|
||||
assert_eq!((s.mem_total, s.mem_used), (2000 * 1024, 1500 * 1024));
|
||||
assert_eq!((s.rx, s.tx, s.rd, s.wr, s.pids), (123456789, 42, 2048, 4096, 17));
|
||||
assert_eq!(super::human(1536), "1.5KiB");
|
||||
}
|
||||
}
|
||||
|
||||
/// Delete cached images no sandbox was created from.
|
||||
pub fn gc(h: &Path) -> Result<()> {
|
||||
let used: Vec<String> = fs::read_dir(h.join("sandboxes"))
|
||||
|
||||
@@ -97,6 +97,8 @@ pub fn run(h: &Path, o: Opts, image: &str, cmd: Vec<String>, extra: &[String], g
|
||||
.unwrap_or("/".into());
|
||||
let mut s = String::from("B=/.wsbx/busybox\n");
|
||||
s += &format!("$B date -s @{} >/dev/null\n", now());
|
||||
// Some images ship no /etc/hosts: without a localhost entry, Go and others resolve it via DNS and fail.
|
||||
s += "$B grep -qw localhost /etc/hosts 2>/dev/null || printf '127.0.0.1 localhost\\n::1 localhost\\n' >> /etc/hosts\n";
|
||||
s += &format!("$B hostname {name}; echo {name} > /etc/hostname; $B grep -q ' {name}$' /etc/hosts 2>/dev/null || echo '127.0.1.1 {name}' >> /etc/hosts\n");
|
||||
if let Some((rows, cols)) = term_size() {
|
||||
s += &format!("$B stty rows {rows} cols {cols}\n");
|
||||
|
||||
Reference in New Issue
Block a user