// `wsbx exec` server: host -> vfkit unix socket -> vsock -> here. // // Protocol: the host sends one JSON request line, then frames go both ways: kind byte, u32 BE length, payload. // host -> guest: 'd' stdin data, 'e' stdin EOF, 'w' window size (rows, cols: u16 BE) // guest -> host: 'o' stdout, 'r' stderr, 'k' keepalive (ignored), 'x' exit status (u32 BE), always last // With Tty the command gets a pty (stdout carries everything); otherwise plain pipes. // With Connect (`wsbx forward`) there's no command: 'd'/'e' go to the guest's localhost:Connect, its output // comes back as 'o', and 'x' marks its EOF (status 0) or a failed dial (1, after an 'r' message). package main import ( "bufio" "cmp" "encoding/binary" "encoding/json" "errors" "fmt" "io" "net" "os" "os/exec" "strconv" "strings" "sync" "sync/atomic" "syscall" "time" "unsafe" ) type request struct { Argv []string User, Term string Tty bool Rows, Cols uint16 Connect uint16 } func serveExec(port uint32) error { fd, err := syscall.Socket(afVsock, syscall.SOCK_STREAM|syscall.SOCK_CLOEXEC, 0) if err != nil { return err } sa := vsockAddr(port, 0xFFFFFFFF) // VMADDR_CID_ANY if _, _, e := syscall.Syscall(syscall.SYS_BIND, uintptr(fd), uintptr(unsafe.Pointer(&sa[0])), 16); e != 0 { return e } if err := syscall.Listen(fd, 16); err != nil { return err } go func() { for { var peer [16]byte n := uint32(len(peer)) c, _, e := syscall.Syscall6(syscall.SYS_ACCEPT4, uintptr(fd), uintptr(unsafe.Pointer(&peer[0])), uintptr(unsafe.Pointer(&n)), syscall.SOCK_CLOEXEC|syscall.SOCK_NONBLOCK, 0, 0) if e != 0 { time.Sleep(10 * time.Millisecond) continue } // Sessions run as root: only the host may open them, not guest processes via vsock loopback. if binary.LittleEndian.Uint32(peer[8:]) != cidHost { syscall.Close(int(c)) continue } go session(os.NewFile(c, "exec")) // non-blocking fd: Close interrupts pending reads } }() return nil } func session(c *os.File) { defer c.Close() r := bufio.NewReader(c) var req request if line, err := r.ReadBytes('\n'); err != nil || json.Unmarshal(line, &req) != nil { return } var mu sync.Mutex send := func(kind byte, p []byte) error { mu.Lock() defer mu.Unlock() _, err := c.Write(append(binary.BigEndian.AppendUint32([]byte{kind}, uint32(len(p))), p...)) return err } fail := func(err error) { send('r', []byte("wsbx: "+err.Error()+"\r\n")) send('x', binary.BigEndian.AppendUint32(nil, 127)) } if req.Connect != 0 { forward(r, send, req.Connect) return } cmd, uid := command(req) var stdin io.WriteCloser var pty, slave *os.File output := make(chan struct{}) if req.Tty { m, s, err := openPty() if err != nil { fail(err) return } defer m.Close() pty, slave, stdin = m, s, m setWinsize(m, req.Rows, req.Cols) if uid > 0 { s.Chown(uid, -1) // like sshd: tools that reopen the tty by name (gpg, screen) need to own it } cmd.Stdin, cmd.Stdout, cmd.Stderr = s, s, s cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true, Setctty: true} // Ctty 0: the child's stdin defer s.Close() } else { cmd.Stdout, cmd.Stderr = frames{send, 'o'}, frames{send, 'r'} stdin, _ = cmd.StdinPipe() cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} cmd.WaitDelay = time.Second // don't hang on background processes that keep stdout open close(output) // Wait copies it } if err := cmd.Start(); err != nil { fail(err) return } if pty != nil { slave.Close() // our copy of the slave: the master reads EIO once the session's are gone go func() { b := make([]byte, 32<<10) for { n, err := pty.Read(b) if n > 0 { send('o', b[:n]) } if err != nil { close(output) return } } }() } var exited atomic.Bool go func() { for { kind, p, err := readFrame(r) if err != nil { if !exited.Load() { syscall.Kill(-cmd.Process.Pid, syscall.SIGHUP) // host went away } return } switch { case kind == 'd': stdin.Write(p) case kind == 'e' && pty == nil: stdin.Close() case kind == 'w' && pty != nil && len(p) == 4: setWinsize(pty, binary.BigEndian.Uint16(p), binary.BigEndian.Uint16(p[2:])) } } }() // vfkit doesn't pass on the host closing its end; it only closes the vsock once forwarding to the // gone host fails. Keep sending, so a killed `wsbx exec` hangs up the session within ~2 s. go func() { for range time.Tick(2 * time.Second) { if send('k', nil) != nil { return } } }() cmd.Wait() exited.Store(true) select { case <-output: case <-time.After(500 * time.Millisecond): // a background job still holds the pty } code := uint32(cmd.ProcessState.ExitCode()) if ws, ok := cmd.ProcessState.Sys().(syscall.WaitStatus); ok && ws.Signaled() { code = 128 + uint32(ws.Signal()) } send('x', binary.BigEndian.AppendUint32(nil, code)) } // forward pipes a session to the guest's localhost:port until both sides sent EOF or the host is gone. func forward(r *bufio.Reader, send func(byte, []byte) error, port uint16) { t, err := net.Dial("tcp", net.JoinHostPort("localhost", strconv.Itoa(int(port)))) if err != nil { send('r', []byte(err.Error())) send('x', binary.BigEndian.AppendUint32(nil, 1)) return } tc := t.(*net.TCPConn) defer tc.Close() done := make(chan struct{}, 2) go func() { io.Copy(frames{send, 'o'}, tc) send('x', binary.BigEndian.AppendUint32(nil, 0)) done <- struct{}{} }() go func() { defer func() { done <- struct{}{} }() for { kind, p, err := readFrame(r) if err != nil { tc.Close() // host went away return } switch kind { case 'd': tc.Write(p) case 'e': tc.CloseWrite() return } } }() // Same as exec: vfkit doesn't pass on the host's close, so probe; a failed send means it's gone. go func() { for range time.Tick(2 * time.Second) { if send('k', nil) != nil { tc.Close() return } } }() <-done <-done } // command builds the session command: as the requested user, else the image user (like the main command), // with run.sh's environment (image env, -e, HOME) and working directory, which this process inherited. func command(req request) (*exec.Cmd, int) { argv := req.Argv if len(argv) == 0 { argv = []string{"/bin/sh"} if _, err := os.Stat("/bin/bash"); err == nil { argv[0] = "/bin/bash" } } env := os.Environ() if req.Term != "" { env = append(env, "TERM="+req.Term) } user := cmp.Or(req.User, os.Getenv("WSBX_USER"), "root") uid, home := passwd(user) if req.User != "" { env = append(env, "HOME="+home) // run.sh set HOME for the image user } if uid != 0 { argv = append([]string{"/.wsbx/busybox", "chpst", "-u", user}, argv...) } cmd := exec.Command(argv[0], argv[1:]...) cmd.Env = env return cmd, uid } // passwd looks up uid and home of a user (name or uid, optionally ":group"); uid -1 if unknown. func passwd(user string) (uid int, home string) { name, _, _ := strings.Cut(user, ":") uid, home = -1, "/" if n, err := strconv.Atoi(name); err == nil { uid = n } b, _ := os.ReadFile("/etc/passwd") for _, l := range strings.Split(string(b), "\n") { if f := strings.Split(l, ":"); len(f) > 5 && (f[0] == name || f[2] == name) { uid, _ = strconv.Atoi(f[2]) return uid, f[5] } } return } type frames struct { send func(byte, []byte) error kind byte } func (f frames) Write(p []byte) (int, error) { if err := f.send(f.kind, p); err != nil { return 0, err } return len(p), nil } func readFrame(r *bufio.Reader) (byte, []byte, error) { var h [5]byte if _, err := io.ReadFull(r, h[:]); err != nil { return 0, nil, err } n := binary.BigEndian.Uint32(h[1:]) if n > 1<<20 { return 0, nil, errors.New("frame too large") } p := make([]byte, n) _, err := io.ReadFull(r, p) return h[0], p, err } func openPty() (m, s *os.File, err error) { if m, err = os.OpenFile("/dev/ptmx", os.O_RDWR|syscall.O_NOCTTY, 0); err != nil { return } var n, unlock uint32 if err = ioctl(m, syscall.TIOCSPTLCK, unsafe.Pointer(&unlock)); err == nil { if err = ioctl(m, syscall.TIOCGPTN, unsafe.Pointer(&n)); err == nil { s, err = os.OpenFile(fmt.Sprintf("/dev/pts/%d", n), os.O_RDWR|syscall.O_NOCTTY, 0) } } if err != nil { m.Close() } return } func setWinsize(pty *os.File, rows, cols uint16) { ws := [4]uint16{rows, cols, 0, 0} ioctl(pty, syscall.TIOCSWINSZ, unsafe.Pointer(&ws)) } // ioctl via SyscallConn: f.Fd() would switch the fd to blocking mode (Close could then not interrupt Read). func ioctl(f *os.File, req uintptr, arg unsafe.Pointer) error { rc, err := f.SyscallConn() if err != nil { return err } var e syscall.Errno rc.Control(func(fd uintptr) { _, _, e = syscall.Syscall(syscall.SYS_IOCTL, fd, req, uintptr(arg)) }) if e != 0 { return e } return nil }